[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [sbws/master] Use juga's code to cleanup result files too



commit 3a1789bddf7aecd74a9ed784f38ebb1a9f45f2de
Author: Matt Traudt <sirmatt@xxxxxxx>
Date:   Wed Aug 1 19:35:53 2018 -0400

    Use juga's code to cleanup result files too
---
 sbws/core/cleanup.py | 132 +++++++++++++++++----------------------------------
 1 file changed, 43 insertions(+), 89 deletions(-)

diff --git a/sbws/core/cleanup.py b/sbws/core/cleanup.py
index ad5b286..7dc7e78 100644
--- a/sbws/core/cleanup.py
+++ b/sbws/core/cleanup.py
@@ -7,7 +7,6 @@ from sbws.util.timestamp import unixts_to_dt_obj
 from argparse import ArgumentDefaultsHelpFormatter
 from datetime import datetime
 from datetime import timedelta
-import re
 import os
 import gzip
 import shutil
@@ -64,38 +63,6 @@ def _get_files_mtime_older_than(dname, days_delta, extensions):
                 yield fname
 
 
-def _get_older_files_than(dname, num_days_ago, extensions):
-    assert os.path.isdir(dname)
-    assert isinstance(num_days_ago, int)
-    assert isinstance(extensions, list)
-    for ext in extensions:
-        assert isinstance(ext, str)
-        assert ext[0] == '.'
-    # Determine oldest allowed date
-    today = datetime.utcfromtimestamp(time.time())
-    oldest_day = today - timedelta(days=num_days_ago)
-    # Compile a regex that can extract a date from a file name that looks like
-    # /path/to/foo/YYYY-MM-DD*.extension
-    extensions = [re.escape(e) for e in extensions]
-    day_part = '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]'
-    regex = re.compile(r'^.*/({}).*({})$'
-                       .format(day_part, '|'.join(extensions)))
-    # Walk through all files in the given dname, find files that match the
-    # regex, and yield the ones that contain a date in the file name that is
-    # too old.
-    for root, dirs, files in os.walk(dname):
-        for f in files:
-            fname = os.path.join(root, f)
-            match = regex.match(fname)
-            if not match:
-                log.debug('Ignoring %s because it doesn\'t look like '
-                          'YYYY-MM-DD', fname)
-                continue
-            d = datetime(*[int(n) for n in match.group(1).split('-')])
-            if d < oldest_day:
-                yield fname
-
-
 def _delete_files(dname, files, dry_run=True):
     """Delete the files passed as argument."""
     assert os.path.isdir(dname)
@@ -108,20 +75,6 @@ def _delete_files(dname, files, dry_run=True):
                 os.remove(fname)
 
 
-def _remove_rotten_files(datadir, rotten_days, dry_run=True):
-    assert os.path.isdir(datadir)
-    assert isinstance(rotten_days, int)
-    # Hold the lock for basically the entire time just in case someone else
-    # moves files between when we get the list of files and when we try to
-    # delete them.
-    with DirectoryLock(datadir):
-        for fname in _get_older_files_than(datadir, rotten_days,
-                                           ['.txt', '.txt.gz']):
-            log.info('Deleting %s', fname)
-            if not dry_run:
-                os.remove(fname)
-
-
 def _compress_files(dname, files, dry_run=True):
     """Compress the files passed as argument."""
     assert os.path.isdir(dname)
@@ -139,24 +92,6 @@ def _compress_files(dname, files, dry_run=True):
             os.remove(fname)
 
 
-def _compress_stale_files(datadir, stale_days, dry_run=True):
-    assert os.path.isdir(datadir)
-    assert isinstance(stale_days, int)
-    # Hold the lock for basically the entire time just in case someone else
-    # moves files between when we get the list of files and when we try to
-    # compress them.
-    with DirectoryLock(datadir):
-        for fname in _get_older_files_than(datadir, stale_days, ['.txt']):
-            log.info('Compressing %s', fname)
-            if dry_run:
-                continue
-            with open(fname, 'rt') as in_fd:
-                out_fname = fname + '.gz'
-                with gzip.open(out_fname, 'wt') as out_fd:
-                    shutil.copyfileobj(in_fd, out_fd)
-            os.remove(fname)
-
-
 def _check_validity_periods_v3bw(compress_after_days, delete_after_days):
     if 1 <= compress_after_days and compress_after_days < delete_after_days:
         return True
@@ -164,6 +99,26 @@ def _check_validity_periods_v3bw(compress_after_days, delete_after_days):
               "after a bigger number of days.")
 
 
+def _check_validity_periods_results(
+        data_period, compress_after_days, delete_after_days):
+    if compress_after_days - 2 < data_period:
+        fail_hard(
+            'For safetly, cleanup/stale_days (%d) must be at least 2 days '
+            'larger than general/data_period (%d)', compress_after_days,
+            data_period)
+    if delete_after_days < compress_after_days:
+        fail_hard(
+            'cleanup/rotten_days (%d) must be the same or larger than '
+            'cleanup/stale_days (%d)', delete_after_days, compress_after_days)
+    if compress_after_days / 2 < data_period:
+        log.warning(
+            'cleanup/stale_days (%d) is less than twice '
+            'general/data_period (%d). For ease of parsing older results '
+            'if necessary, it is recommended to make stale_days at least '
+            'twice the data_period.', compress_after_days, data_period)
+    return True
+
+
 def _clean_v3bw_files(args, conf):
     v3bw_dname = conf['paths']['v3bw_dname']
     if not os.path.isdir(v3bw_dname):
@@ -186,6 +141,28 @@ def _clean_v3bw_files(args, conf):
     _compress_files(v3bw_dname, files_to_compress, dry_run=args.dry_run)
 
 
+def _clean_result_files(args, conf):
+    datadir = conf['paths']['datadir']
+    if not os.path.isdir(datadir):
+        fail_hard('%s does not exist', datadir)
+    data_period = conf.getint('general', 'data_period')
+    compress_after_days = conf.getint('cleanup', 'stale_days')
+    delete_after_days = conf.getint('cleanup', 'rotten_days')
+    _check_validity_periods_results(
+        data_period, compress_after_days, delete_after_days)
+
+    # first delete so that the files to be deleted are not compressed first
+    files_to_delete = _get_files_mtime_older_than(
+        datadir, delete_after_days, ['.txt', '.gz'])
+    _delete_files(datadir, files_to_delete, dry_run=args.dry_run)
+
+    # when dry_run is true, compress will also show all the files that
+    # would have been deleted, since they are not really deleted
+    files_to_compress = _get_files_mtime_older_than(
+        datadir, compress_after_days, ['.txt'])
+    _compress_files(datadir, files_to_compress, dry_run=args.dry_run)
+
+
 def main(args, conf):
     '''
     Main entry point in to the cleanup command.
@@ -200,30 +177,7 @@ def main(args, conf):
         fail_hard('Nothing to clean.')
 
     if not args.no_results:
-        datadir = conf['paths']['datadir']
-        if not os.path.isdir(datadir):
-            fail_hard('%s does not exist', datadir)
-
-        fresh_days = conf.getint('general', 'data_period')
-        stale_days = conf.getint('cleanup', 'stale_days')
-        rotten_days = conf.getint('cleanup', 'rotten_days')
-        if stale_days - 2 < fresh_days:
-            fail_hard('For safetly, cleanup/stale_days (%d) must be at least '
-                      '2 days larger than general/data_period (%d)',
-                      stale_days, fresh_days)
-        if rotten_days < stale_days:
-            fail_hard('cleanup/rotten_days (%d) must be the same or larger '
-                      'than cleanup/stale_days (%d)', rotten_days, stale_days)
-
-        if stale_days / 2 < fresh_days:
-            log.warning(
-                'cleanup/stale_days (%d) is less than twice '
-                'general/data_period (%d). For ease of parsing older results '
-                'if necessary, it is recommended to make stale_days at least '
-                'twice the data_period.', stale_days, fresh_days)
-
-        _remove_rotten_files(datadir, rotten_days, dry_run=args.dry_run)
-        _compress_stale_files(datadir, stale_days, dry_run=args.dry_run)
+        _clean_result_files(args, conf)
 
     if not args.no_v3bw:
         _clean_v3bw_files(args, conf)



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits