[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [collector/master] Delete files in out/ that are older than 7 weeks.



commit 66ddc4d7d996ad2877aea44ea03982f14f069545
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Wed Nov 25 16:09:14 2020 +0100

    Delete files in out/ that are older than 7 weeks.
    
    Fixes #21219.
---
 CHANGELOG.md                                       |  7 +++
 .../bridgedb/BridgedbMetricsProcessor.java         | 38 ++++----------
 .../bridgedescs/SanitizedBridgesWriter.java        | 37 +++++--------
 .../BridgePoolAssignmentsProcessor.java            | 33 ++++--------
 .../collector/exitlists/ExitListDownloader.java    | 27 ++++------
 .../collector/onionperf/OnionPerfDownloader.java   | 37 ++++++-------
 .../collector/persist/PersistenceUtils.java        | 50 +++++++++++++++---
 .../collector/relaydescs/ArchiveWriter.java        | 61 +++++-----------------
 .../snowflake/SnowflakeStatsDownloader.java        | 33 +++++-------
 .../metrics/collector/sync/SyncPersistence.java    |  7 +--
 .../collector/webstats/SanitizeWeblogs.java        | 12 +++--
 .../collector/persist/PersistUtilsTest.java        | 32 ++++++++++++
 12 files changed, 182 insertions(+), 192 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ff2e9e7..e292f9a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,10 @@
+# Changes in version 1.??.? - 2020-1?-??
+
+ * Medium changes
+  - Clean up descriptors written to the `out/` directory by deleting
+    files that are older than seven weeks.
+
+
 # Changes in version 1.16.1 - 2020-08-16
 
  * Medium changes
diff --git a/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java b/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java
index 0073ee3..d05aa9c 100644
--- a/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java
+++ b/src/main/java/org/torproject/metrics/collector/bridgedb/BridgedbMetricsProcessor.java
@@ -12,6 +12,7 @@ import org.torproject.metrics.collector.conf.ConfigurationException;
 import org.torproject.metrics.collector.conf.Key;
 import org.torproject.metrics.collector.cron.CollecTorMain;
 import org.torproject.metrics.collector.persist.BridgedbMetricsPersistence;
+import org.torproject.metrics.collector.persist.PersistenceUtils;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -23,9 +24,7 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
-import java.util.Arrays;
 import java.util.SortedSet;
-import java.util.Stack;
 import java.util.TreeSet;
 
 public class BridgedbMetricsProcessor extends CollecTorMain {
@@ -127,10 +126,10 @@ public class BridgedbMetricsProcessor extends CollecTorMain {
             descriptor.getClass(), descriptor.getDescriptorFile());
       }
     }
-    logger.info("Cleaning up directory {} containing recent files.",
-        this.recentPathName);
+    logger.info("Cleaning up directories {} and {}.",
+        this.recentPathName, this.outputPathName);
     this.writeProcessedFiles(this.parsedBridgedbMetricsFile, processedFiles);
-    this.cleanUpRsyncDirectory();
+    this.cleanUpDirectories();
     logger.info("Finished processing BridgeDB statistics file(s).");
   }
 
@@ -175,28 +174,13 @@ public class BridgedbMetricsProcessor extends CollecTorMain {
   }
 
   /**
-   * Delete all files from the rsync directory that have not been modified in
-   * the last three days.
+   * Delete all files from the rsync (out) directory that have not been modified
+   * in the last three days (seven weeks).
    */
-  public void cleanUpRsyncDirectory() {
-    Instant cutOff = Instant.now().minus(3L, ChronoUnit.DAYS);
-    Stack<File> allFiles = new Stack<>();
-    allFiles.add(new File(this.recentPathName));
-    while (!allFiles.isEmpty()) {
-      File file = allFiles.pop();
-      if (file.isDirectory()) {
-        File[] filesInDirectory = file.listFiles();
-        if (null != filesInDirectory) {
-          allFiles.addAll(Arrays.asList(filesInDirectory));
-        }
-      } else if (Instant.ofEpochMilli(file.lastModified()).isBefore(cutOff)) {
-        try {
-          Files.deleteIfExists(file.toPath());
-        } catch (IOException e) {
-          logger.warn("Unable to delete file {} that is apparently older than "
-              + "three days.", file, e);
-        }
-      }
-    }
+  private void cleanUpDirectories() {
+    PersistenceUtils.cleanDirectory(Paths.get(this.recentPathName),
+        Instant.now().minus(3, ChronoUnit.DAYS).toEpochMilli());
+    PersistenceUtils.cleanDirectory(Paths.get(this.outputPathName),
+        Instant.now().minus(49, ChronoUnit.DAYS).toEpochMilli());
   }
 }
diff --git a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
index b8e7f2d..62288ad 100644
--- a/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
+++ b/src/main/java/org/torproject/metrics/collector/bridgedescs/SanitizedBridgesWriter.java
@@ -13,6 +13,7 @@ import org.torproject.metrics.collector.conf.Configuration;
 import org.torproject.metrics.collector.conf.ConfigurationException;
 import org.torproject.metrics.collector.conf.Key;
 import org.torproject.metrics.collector.cron.CollecTorMain;
+import org.torproject.metrics.collector.persist.PersistenceUtils;
 
 import org.apache.commons.codec.DecoderException;
 import org.apache.commons.codec.binary.Base64;
@@ -34,15 +35,15 @@ import java.security.GeneralSecurityException;
 import java.security.SecureRandom;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.time.Instant;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.SortedMap;
-import java.util.Stack;
 import java.util.TreeMap;
 
 /**
@@ -228,7 +229,7 @@ public class SanitizedBridgesWriter extends CollecTorMain {
 
     this.checkStaleDescriptors();
 
-    this.cleanUpRsyncDirectory();
+    this.cleanUpDirectories();
   }
 
   private String scrubOrAddress(String orAddress, byte[] fingerprintBytes,
@@ -1388,27 +1389,15 @@ public class SanitizedBridgesWriter extends CollecTorMain {
     }
   }
 
-  /** Delete all files from the rsync directory that have not been modified
-   * in the last three days, and remove the .tmp extension from newly
-   * written files. */
-  public void cleanUpRsyncDirectory() throws ConfigurationException {
-    long cutOffMillis = System.currentTimeMillis()
-        - 3L * 24L * 60L * 60L * 1000L;
-    Stack<File> allFiles = new Stack<>();
-    allFiles.add(new File(config.getPath(Key.RecentPath).toFile(),
-        BRIDGE_DESCRIPTORS));
-    while (!allFiles.isEmpty()) {
-      File file = allFiles.pop();
-      if (file.isDirectory()) {
-        allFiles.addAll(Arrays.asList(file.listFiles()));
-      } else if (file.lastModified() < cutOffMillis) {
-        file.delete();
-      } else if (file.getName().endsWith(".tmp")) {
-        file.renameTo(new File(file.getParentFile(),
-            file.getName().substring(0,
-            file.getName().lastIndexOf(".tmp"))));
-      }
-    }
+  /**
+   * Delete all files from the rsync (out) directory that have not been modified
+   * in the last three days (seven weeks), and remove the .tmp extension from
+   * newly written files. */
+  private void cleanUpDirectories() {
+    PersistenceUtils.cleanDirectory(Paths.get(this.recentPathName),
+        Instant.now().minus(3, ChronoUnit.DAYS).toEpochMilli());
+    PersistenceUtils.cleanDirectory(Paths.get(this.outputPathName),
+        Instant.now().minus(49, ChronoUnit.DAYS).toEpochMilli());
   }
 }
 
diff --git a/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java b/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
index ffae262..9961d4c 100644
--- a/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
+++ b/src/main/java/org/torproject/metrics/collector/bridgepools/BridgePoolAssignmentsProcessor.java
@@ -8,6 +8,7 @@ import org.torproject.metrics.collector.conf.Configuration;
 import org.torproject.metrics.collector.conf.ConfigurationException;
 import org.torproject.metrics.collector.conf.Key;
 import org.torproject.metrics.collector.cron.CollecTorMain;
+import org.torproject.metrics.collector.persist.PersistenceUtils;
 
 import org.apache.commons.codec.DecoderException;
 import org.apache.commons.codec.binary.Hex;
@@ -24,7 +25,6 @@ import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStreamReader;
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.DateTimeException;
@@ -178,7 +178,7 @@ public class BridgePoolAssignmentsProcessor extends CollecTorMain {
     }
     this.writeProcessedFiles(this.parsedBridgePoolAssignmentsFile,
         processedFiles);
-    this.cleanUpRsyncDirectory();
+    this.cleanUpDirectories();
     logger.info("Finished processing bridge pool assignment file(s).");
   }
 
@@ -363,29 +363,14 @@ public class BridgePoolAssignmentsProcessor extends CollecTorMain {
   }
 
   /**
-   * Delete all files from the rsync directory that have not been modified in
-   * the last three days.
+   * Delete all files from the rsync (out) directory that have not been modified
+   * in the last three days (seven weeks).
    */
-  public void cleanUpRsyncDirectory() {
-    Instant cutOff = Instant.now().minus(3L, ChronoUnit.DAYS);
-    Stack<File> allFiles = new Stack<>();
-    allFiles.add(new File(this.recentPathName));
-    while (!allFiles.isEmpty()) {
-      File file = allFiles.pop();
-      if (file.isDirectory()) {
-        File[] filesInDirectory = file.listFiles();
-        if (null != filesInDirectory) {
-          allFiles.addAll(Arrays.asList(filesInDirectory));
-        }
-      } else if (Instant.ofEpochMilli(file.lastModified()).isBefore(cutOff)) {
-        try {
-          Files.deleteIfExists(file.toPath());
-        } catch (IOException e) {
-          logger.warn("Unable to delete file {} that is apparently older than "
-              + "three days.", file, e);
-        }
-      }
-    }
+  public void cleanUpDirectories() {
+    PersistenceUtils.cleanDirectory(Paths.get(this.recentPathName),
+        Instant.now().minus(3, ChronoUnit.DAYS).toEpochMilli());
+    PersistenceUtils.cleanDirectory(Paths.get(this.outputPathName),
+        Instant.now().minus(49, ChronoUnit.DAYS).toEpochMilli());
   }
 }
 
diff --git a/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java
index c6b45da..6b9b791 100644
--- a/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java
+++ b/src/main/java/org/torproject/metrics/collector/exitlists/ExitListDownloader.java
@@ -13,6 +13,7 @@ import org.torproject.metrics.collector.conf.ConfigurationException;
 import org.torproject.metrics.collector.conf.Key;
 import org.torproject.metrics.collector.cron.CollecTorMain;
 import org.torproject.metrics.collector.downloader.Downloader;
+import org.torproject.metrics.collector.persist.PersistenceUtils;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -24,6 +25,8 @@ import java.io.IOException;
 import java.net.URL;
 import java.nio.file.Paths;
 import java.text.SimpleDateFormat;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.SortedSet;
@@ -168,24 +171,16 @@ public class ExitListDownloader extends CollecTorMain {
     }
     logger.info(dumpStats.toString());
 
-    this.cleanUpRsyncDirectory();
+    this.cleanUpDirectories();
   }
 
-  /** Delete all files from the rsync directory that have not been modified
-   * in the last three days. */
-  public void cleanUpRsyncDirectory() {
-    long cutOffMillis = System.currentTimeMillis()
-        - 3L * 24L * 60L * 60L * 1000L;
-    Stack<File> allFiles = new Stack<>();
-    allFiles.add(new File(recentPathName));
-    while (!allFiles.isEmpty()) {
-      File file = allFiles.pop();
-      if (file.isDirectory()) {
-        allFiles.addAll(Arrays.asList(file.listFiles()));
-      } else if (file.lastModified() < cutOffMillis) {
-        file.delete();
-      }
-    }
+  /** Delete all files from the rsync (out) directory that have not been
+   * modified in the last three days (seven weeks). */
+  private void cleanUpDirectories() {
+    PersistenceUtils.cleanDirectory(Paths.get(this.recentPathName),
+        Instant.now().minus(3, ChronoUnit.DAYS).toEpochMilli());
+    PersistenceUtils.cleanDirectory(Paths.get(this.outputPathName),
+        Instant.now().minus(49, ChronoUnit.DAYS).toEpochMilli());
   }
 }
 
diff --git a/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java b/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java
index d22ac0b..f90bdfe 100644
--- a/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java
+++ b/src/main/java/org/torproject/metrics/collector/onionperf/OnionPerfDownloader.java
@@ -12,6 +12,7 @@ import org.torproject.metrics.collector.conf.ConfigurationException;
 import org.torproject.metrics.collector.conf.Key;
 import org.torproject.metrics.collector.cron.CollecTorMain;
 import org.torproject.metrics.collector.downloader.Downloader;
+import org.torproject.metrics.collector.persist.PersistenceUtils;
 
 import org.apache.commons.compress.utils.IOUtils;
 import org.slf4j.Logger;
@@ -31,13 +32,13 @@ import java.nio.file.StandardCopyOption;
 import java.text.DateFormat;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.SortedSet;
-import java.util.Stack;
 import java.util.TreeSet;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -103,7 +104,7 @@ public class OnionPerfDownloader extends CollecTorMain {
       this.downloadFromOnionPerfHost(baseUrl);
     }
     this.writeDownloadedOnionPerfFiles();
-    this.cleanUpRsyncDirectory();
+    this.cleanUpDirectories();
   }
 
   private void readDownloadedOnionPerfFiles() {
@@ -441,21 +442,21 @@ public class OnionPerfDownloader extends CollecTorMain {
     }
   }
 
-  /** Delete all files from the rsync directory that have not been modified
-   * in the last three days. */
-  public void cleanUpRsyncDirectory() throws ConfigurationException {
-    long cutOffMillis = System.currentTimeMillis()
-        - 3L * 24L * 60L * 60L * 1000L;
-    Stack<File> allFiles = new Stack<>();
-    allFiles.add(new File(config.getPath(Key.RecentPath).toFile(), TORPERF));
-    while (!allFiles.isEmpty()) {
-      File file = allFiles.pop();
-      if (file.isDirectory()) {
-        allFiles.addAll(Arrays.asList(file.listFiles()));
-      } else if (file.lastModified() < cutOffMillis) {
-        file.delete();
-      }
-    }
+  /** Delete all files from the rsync (out) directories that have not been
+   * modified in the last three days (seven weeks). */
+  private void cleanUpDirectories() {
+    PersistenceUtils.cleanDirectory(
+        new File(this.recentDirectory, TORPERF).toPath(),
+        Instant.now().minus(3, ChronoUnit.DAYS).toEpochMilli());
+    PersistenceUtils.cleanDirectory(
+        new File(this.recentDirectory, ONIONPERF).toPath(),
+        Instant.now().minus(3, ChronoUnit.DAYS).toEpochMilli());
+    PersistenceUtils.cleanDirectory(
+        new File(this.archiveDirectory, TORPERF).toPath(),
+        Instant.now().minus(49, ChronoUnit.DAYS).toEpochMilli());
+    PersistenceUtils.cleanDirectory(
+        new File(this.archiveDirectory, ONIONPERF).toPath(),
+        Instant.now().minus(49, ChronoUnit.DAYS).toEpochMilli());
   }
 }
 
diff --git a/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java b/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java
index c958aec..e787c39 100644
--- a/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java
+++ b/src/main/java/org/torproject/metrics/collector/persist/PersistenceUtils.java
@@ -18,6 +18,7 @@ import java.nio.file.StandardCopyOption;
 import java.nio.file.StandardOpenOption;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.text.SimpleDateFormat;
+import java.time.Instant;
 import java.util.Date;
 
 public class PersistenceUtils {
@@ -80,33 +81,70 @@ public class PersistenceUtils {
   }
 
   /** Move temporary files to their final location. */
-  public static void cleanDirectory(Path pathToClean) throws IOException {
+  public static void cleanDirectory(Path pathToClean) {
     PersistenceUtils.cleanDirectory(pathToClean, -1L);
   }
 
   /** Clean up the given directory by deleting files that are older than the
    * given cut-off timestamp, and by moving temporary files to their final
    * location. */
-  public static void cleanDirectory(Path pathToClean, long cutOffMillis)
-      throws IOException {
+  public static void cleanDirectory(Path pathToClean, long cutOffMillis) {
+    PersistenceUtils.cleanDirectory(pathToClean, cutOffMillis, null);
+  }
+
+  /** Clean up the given directory, excluding the given subdirectory, by
+   * deleting files that are older than the given cut-off timestamp, and by
+   * moving temporary files to their final location. */
+  public static void cleanDirectory(Path pathToClean, long cutOffMillis,
+      Path pathToExclude) {
+    if (!Files.exists(pathToClean)) {
+      return;
+    }
+    logger.info("Cleaning up directory {} with cut-off time {}.",
+        pathToClean, Instant.ofEpochMilli(cutOffMillis));
     SimpleFileVisitor<Path> sfv = new SimpleFileVisitor<Path>() {
+      @Override
+      public FileVisitResult preVisitDirectory(Path dir,
+          BasicFileAttributes attrs) {
+        if (null == pathToExclude || !pathToExclude.equals(dir)) {
+          return FileVisitResult.CONTINUE;
+        } else {
+          return FileVisitResult.SKIP_SUBTREE;
+        }
+      }
+
       @Override
       public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
           throws IOException {
         String tempName = file.toString();
         if (cutOffMillis >= 0L
             && attrs.lastModifiedTime().toMillis() < cutOffMillis) {
-          file.toFile().delete();
+          Files.delete(file);
         } else if (tempName.endsWith(TEMPFIX)) {
           Path outputPath = Paths
               .get(tempName.substring(0, tempName.length() - TEMPFIX.length()));
           Files.deleteIfExists(outputPath);
-          file.toFile().renameTo(outputPath.toFile());
+          Files.move(file, outputPath);
+        }
+        return FileVisitResult.CONTINUE;
+      }
+
+      @Override
+      public FileVisitResult postVisitDirectory(Path dir, IOException exc)
+          throws IOException {
+        if (!Files.list(dir).findFirst().isPresent()) {
+          Files.delete(dir);
         }
         return FileVisitResult.CONTINUE;
       }
     };
-    Files.walkFileTree(pathToClean, sfv);
+    try {
+      Files.walkFileTree(pathToClean, sfv);
+    } catch (IOException e) {
+      logger.warn("Caught I/O exception while cleaning up directory {} with "
+          + "cut-off time {}. Continuing.",
+          pathToClean, Instant.ofEpochMilli(cutOffMillis), e);
+    }
   }
 
   /** Return all date-time parts as array. */
diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
index 8addd5e..28472f8 100644
--- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
+++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
@@ -18,6 +18,7 @@ import org.torproject.metrics.collector.conf.ConfigurationException;
 import org.torproject.metrics.collector.conf.Key;
 import org.torproject.metrics.collector.conf.SourceType;
 import org.torproject.metrics.collector.cron.CollecTorMain;
+import org.torproject.metrics.collector.persist.PersistenceUtils;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -38,7 +39,7 @@ import java.time.Instant;
 import java.time.LocalDateTime;
 import java.time.ZoneOffset;
 import java.time.format.DateTimeFormatter;
-import java.util.Arrays;
+import java.time.temporal.ChronoUnit;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -46,7 +47,6 @@ import java.util.Map;
 import java.util.Set;
 import java.util.SortedMap;
 import java.util.SortedSet;
-import java.util.Stack;
 import java.util.TreeMap;
 import java.util.TreeSet;
 
@@ -197,7 +197,7 @@ public class ArchiveWriter extends CollecTorMain {
 
     this.checkStaledescriptors();
 
-    this.cleanUpRsyncDirectory();
+    this.cleanUpDirectories();
 
     this.saveDescriptorDigests();
 
@@ -549,51 +549,16 @@ public class ArchiveWriter extends CollecTorMain {
     }
   }
 
-  /** Delete all files from the rsync directory that have not been modified
-   * in the last three days (except for microdescriptors which are kept
-   * for up to thirty days), and remove the .tmp extension from newly
-   * written files. */
-  public void cleanUpRsyncDirectory() {
-    long cutOffMillis = System.currentTimeMillis()
-        - 3L * 24L * 60L * 60L * 1000L;
-    long cutOffMicroMillis = cutOffMillis - 27L * 24L * 60L * 60L * 1000L;
-    Stack<File> allFiles = new Stack<>();
-    allFiles.add(new File(recentPathName, RELAY_DESCRIPTORS));
-    while (!allFiles.isEmpty()) {
-      File file = allFiles.pop();
-      if (file.isDirectory()) {
-        File[] containedFiles = file.listFiles();
-        if (null == containedFiles) {
-          logger.warn("Unable to list files contained in directory {}.", file);
-        } else {
-          allFiles.addAll(Arrays.asList(containedFiles));
-        }
-      } else if (file.getName().endsWith("-micro")) {
-        if (file.lastModified() < cutOffMicroMillis) {
-          if (!file.delete()) {
-            logger.warn("Unable to delete outdated descriptor file {}.", file);
-          }
-        }
-      } else if (file.lastModified() < cutOffMillis) {
-        if (!file.delete()) {
-          logger.warn("Unable to delete outdated descriptor file {}.", file);
-        }
-      } else if (file.getName().endsWith(".tmp")) {
-        File destinationFile = new File(file.getParentFile(),
-            file.getName().substring(0, file.getName().lastIndexOf(".tmp")));
-        if (destinationFile.exists()) {
-          logger.warn("Attempting to rename descriptor file {} to existing "
-              + "file {}.", file, destinationFile);
-        } else {
-          logger.info("Renaming descriptor file {} to non-existing file {}.",
-              file, destinationFile);
-        }
-        if (!file.renameTo(destinationFile)) {
-          logger.warn("Unable to rename descriptor file {} to {}.", file,
-              destinationFile);
-        }
-      }
-    }
+  /** Delete all files from the rsync (out) directory that have not been
+   * modified in the last three days (seven weeks), and remove the .tmp
+   * extension from newly written files. */
+  public void cleanUpDirectories() {
+    PersistenceUtils.cleanDirectory(
+        Paths.get(recentPathName, RELAY_DESCRIPTORS),
+        Instant.now().minus(3, ChronoUnit.DAYS).toEpochMilli());
+    PersistenceUtils.cleanDirectory(Paths.get(outputDirectory),
+        Instant.now().minus(49, ChronoUnit.DAYS).toEpochMilli(),
+        Paths.get(this.outputDirectory, "certs"));
   }
 
   private void saveDescriptorDigests() {
diff --git a/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java b/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java
index cbca74a..93388d5 100644
--- a/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java
+++ b/src/main/java/org/torproject/metrics/collector/snowflake/SnowflakeStatsDownloader.java
@@ -13,6 +13,7 @@ import org.torproject.metrics.collector.conf.ConfigurationException;
 import org.torproject.metrics.collector.conf.Key;
 import org.torproject.metrics.collector.cron.CollecTorMain;
 import org.torproject.metrics.collector.downloader.Downloader;
+import org.torproject.metrics.collector.persist.PersistenceUtils;
 import org.torproject.metrics.collector.persist.SnowflakeStatsPersistence;
 
 import org.slf4j.Logger;
@@ -25,10 +26,10 @@ import java.io.OutputStream;
 import java.net.URL;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.time.Instant;
 import java.time.LocalDateTime;
-import java.util.Arrays;
+import java.time.temporal.ChronoUnit;
 import java.util.SortedSet;
-import java.util.Stack;
 import java.util.TreeSet;
 
 public class SnowflakeStatsDownloader extends CollecTorMain {
@@ -38,6 +39,8 @@ public class SnowflakeStatsDownloader extends CollecTorMain {
 
   private String recentPathName;
 
+  private String outputPathName;
+
   /** Instantiate the snowflake-stats module using the given configuration. */
   public SnowflakeStatsDownloader(Configuration config) {
     super(config);
@@ -81,7 +84,7 @@ public class SnowflakeStatsDownloader extends CollecTorMain {
     DescriptorParser descriptorParser =
         DescriptorSourceFactory.createDescriptorParser();
     SortedSet<LocalDateTime> snowflakeStatsEnds = new TreeSet<>();
-    String outputPathName = config.getPath(Key.OutputPath).toString();
+    this.outputPathName = config.getPath(Key.OutputPath).toString();
     for (Descriptor descriptor : descriptorParser.parseDescriptors(
         downloadedBytes, null, null)) {
       if (descriptor instanceof SnowflakeStats) {
@@ -119,7 +122,7 @@ public class SnowflakeStatsDownloader extends CollecTorMain {
     }
 
     this.writeProcessedFiles(parsedSnowflakeStatsFile, processedFiles);
-    this.cleanUpRsyncDirectory();
+    this.cleanUpDirectories();
   }
 
   /**
@@ -150,21 +153,13 @@ public class SnowflakeStatsDownloader extends CollecTorMain {
     }
   }
 
-  /** Delete all files from the rsync directory that have not been modified
-   * in the last three days. */
-  public void cleanUpRsyncDirectory() {
-    long cutOffMillis = System.currentTimeMillis()
-        - 3L * 24L * 60L * 60L * 1000L;
-    Stack<File> allFiles = new Stack<>();
-    allFiles.add(new File(recentPathName));
-    while (!allFiles.isEmpty()) {
-      File file = allFiles.pop();
-      if (file.isDirectory()) {
-        allFiles.addAll(Arrays.asList(file.listFiles()));
-      } else if (file.lastModified() < cutOffMillis) {
-        file.delete();
-      }
-    }
+  /** Delete all files from the rsync (out) directory that have not been
+   * modified in the last three days (seven weeks). */
+  private void cleanUpDirectories() {
+    PersistenceUtils.cleanDirectory(Paths.get(this.recentPathName),
+        Instant.now().minus(3, ChronoUnit.DAYS).toEpochMilli());
+    PersistenceUtils.cleanDirectory(Paths.get(this.outputPathName),
+        Instant.now().minus(49, ChronoUnit.DAYS).toEpochMilli());
   }
 }
 
diff --git a/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java b/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java
index adffb93..af48b1f 100644
--- a/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java
+++ b/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java
@@ -42,7 +42,6 @@ import org.torproject.metrics.collector.persist.WebServerAccessLogPersistence;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
 import java.nio.file.Path;
 
 /** Provides persistence for descriptors based on the descriptor type. */
@@ -69,11 +68,7 @@ public class SyncPersistence {
    * Cleans the directory in {@code RecentPath} after storing descriptors.
    */
   public void cleanDirectory() {
-    try {
-      PersistenceUtils.cleanDirectory(recentPath);
-    } catch (IOException ioe) {
-      logger.error("Cleaning of {} failed.", recentPath.toString(), ioe);
-    }
+    PersistenceUtils.cleanDirectory(recentPath);
   }
 
   /**
diff --git a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java
index 670f686..e4f427e 100644
--- a/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java
+++ b/src/main/java/org/torproject/metrics/collector/webstats/SanitizeWeblogs.java
@@ -31,8 +31,10 @@ import java.io.OutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.time.Instant;
 import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -106,10 +108,12 @@ public class SanitizeWeblogs extends CollecTorMain {
             = this.findCleanWrite(this.config.getPath(Key.WebstatsLocalOrigins),
             previouslyProcessedWebstats);
         this.writeProcessedWebstats(newlyProcessedWebstats);
-        long cutOffMillis = System.currentTimeMillis()
-            - 3L * 24L * 60L * 60L * 1000L;
-        PersistenceUtils.cleanDirectory(this.config.getPath(Key.RecentPath),
-            cutOffMillis);
+        PersistenceUtils.cleanDirectory(
+            Paths.get(this.recentDirectory.toString(), WEBSTATS),
+            Instant.now().minus(3, ChronoUnit.DAYS).toEpochMilli());
+        PersistenceUtils.cleanDirectory(
+            Paths.get(this.outputDirectory.toString(), WEBSTATS),
+            Instant.now().minus(49, ChronoUnit.DAYS).toEpochMilli());
       }
     } catch (Exception e) {
       logger.error("Cannot sanitize web-logs: {}", e.getMessage(), e);
diff --git a/src/test/java/org/torproject/metrics/collector/persist/PersistUtilsTest.java b/src/test/java/org/torproject/metrics/collector/persist/PersistUtilsTest.java
index a33b94f..054d0e6 100644
--- a/src/test/java/org/torproject/metrics/collector/persist/PersistUtilsTest.java
+++ b/src/test/java/org/torproject/metrics/collector/persist/PersistUtilsTest.java
@@ -16,6 +16,9 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
+import java.nio.file.attribute.FileTime;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.List;
 
 public class PersistUtilsTest {
@@ -107,4 +110,33 @@ public class PersistUtilsTest {
     assertEquals("File contained: " + text, theText2, text.get(3));
   }
 
+  @Test()
+  public void testCleanDirectory() throws Exception {
+    /*
+     * out/
+     *   a/           # empty after deleting x
+     *     x          # too old file, delete
+     *   b/           # keep together with recent file y
+     *     y.tmp      # recent enough, rename to y
+     *   c/           # exclude (empty) subdirectory
+     */
+    Instant now = Instant.now();
+    Path out = tmpf.newFolder().toPath();
+    Path dirA = Files.createDirectory(out.resolve("a"));
+    Path fileX = Files.createFile(dirA.resolve("x"));
+    Files.setLastModifiedTime(fileX,
+        FileTime.from(now.minus(9L, ChronoUnit.DAYS)));
+    Path dirB = Files.createDirectory(out.resolve("b"));
+    Path fileYTmp = Files.createFile(dirB.resolve("y.tmp"));
+    Files.setLastModifiedTime(fileYTmp, FileTime.from(now));
+    Path dirC = Files.createDirectory(out.resolve("c"));
+    PersistenceUtils.cleanDirectory(out,
+        now.minus(3L, ChronoUnit.DAYS).toEpochMilli(), dirC);
+    assertFalse(Files.exists(dirA));
+    assertFalse(Files.exists(fileX));
+    assertTrue(Files.exists(dirB));
+    assertFalse(Files.exists(fileYTmp));
+    assertTrue(Files.exists(dirB.resolve("y")));
+    assertTrue(Files.exists(dirC));
+  }
 }

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits