[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [metrics-db/master] Make each module copy its own files to rsync/.



commit 9766f4c4e9282253090c2e682ee55037c8864383
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Fri Oct 26 14:15:16 2012 -0400

    Make each module copy its own files to rsync/.
---
 .../db/bridgedescs/SanitizedBridgesWriter.java     |   15 ++
 .../BridgePoolAssignmentsProcessor.java            |   10 ++
 .../ernie/db/exitlists/ExitListDownloader.java     |   12 ++-
 src/org/torproject/ernie/db/main/Main.java         |   18 +--
 .../ernie/db/main/RsyncDataProvider.java           |  161 ++++----------------
 .../ernie/db/relaydescs/ArchiveWriter.java         |   20 +++
 .../ernie/db/torperf/TorperfDownloader.java        |    8 +
 7 files changed, 96 insertions(+), 148 deletions(-)

diff --git a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
index e46450f..15cb8f1 100644
--- a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
+++ b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
@@ -28,6 +28,7 @@ import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.codec.binary.Hex;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.torproject.ernie.db.main.Configuration;
+import org.torproject.ernie.db.main.RsyncDataProvider;
 
 /**
  * Sanitizes bridge descriptors, i.e., removes all possibly sensitive
@@ -178,6 +179,20 @@ public class SanitizedBridgesWriter {
 
     // Finish writing sanitized bridge descriptors to disk
     this.finishWriting();
+
+    // Copy sanitized bridge descriptors from the last 3 days to rsync
+    // directory.
+    if (config.getProvideFilesViaRsync()) {
+      RsyncDataProvider rdp = new RsyncDataProvider(
+          new File(config.getRsyncDirectory()));
+      rdp.copyFiles(new File(sanitizedBridgesDirectory, "statuses"),
+          "bridge-descriptors/statuses");
+      rdp.copyFiles(
+          new File(sanitizedBridgesDirectory, "server-descriptor"),
+          "bridge-descriptors/server-descriptors");
+      rdp.copyFiles(new File(sanitizedBridgesDirectory, "extra-info"),
+          "bridge-descriptors/extra-infos");
+    }
   }
 
   private String scrubOrAddress(String orAddress, byte[] fingerprintBytes,
diff --git a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
index 76fa539..63b7c4d 100644
--- a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
+++ b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
@@ -27,6 +27,7 @@ import org.apache.commons.codec.binary.Hex;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
 import org.torproject.ernie.db.main.Configuration;
+import org.torproject.ernie.db.main.RsyncDataProvider;
 
 public class BridgePoolAssignmentsProcessor {
 
@@ -172,6 +173,15 @@ public class BridgePoolAssignmentsProcessor {
       }
     }
 
+    // Copy sanitized bridge pool assignments from the last 3 days to the
+    // rsync directory.
+    if (config.getProvideFilesViaRsync()) {
+      RsyncDataProvider rdp = new RsyncDataProvider(
+          new File(config.getRsyncDirectory()));
+      rdp.copyFiles(sanitizedAssignmentsDirectory,
+          "bridge-pool-assignments");
+    }
+
     logger.info("Finished processing bridge pool assignment file(s).");
   }
 }
diff --git a/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java b/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java
index 64f6a3b..9fb714b 100644
--- a/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java
+++ b/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java
@@ -18,8 +18,11 @@ import java.util.TreeSet;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import org.torproject.ernie.db.main.Configuration;
+import org.torproject.ernie.db.main.RsyncDataProvider;
+
 public class ExitListDownloader {
-  public ExitListDownloader() {
+  public ExitListDownloader(Configuration config) {
     Logger logger = Logger.getLogger(ExitListDownloader.class.getName());
     try {
       logger.fine("Downloading exit list...");
@@ -95,6 +98,13 @@ public class ExitListDownloader {
       dumpStats.append("\n" + f.getName());
     }
     logger.info(dumpStats.toString());
+
+    /* Copy exit lists from the last 3 days to the rsync directory. */
+    if (config.getProvideFilesViaRsync()) {
+      RsyncDataProvider rdp = new RsyncDataProvider(
+          new File(config.getRsyncDirectory()));
+      rdp.copyFiles(new File("exitlist"), "exit-lists");
+    }
   }
 }
 
diff --git a/src/org/torproject/ernie/db/main/Main.java b/src/org/torproject/ernie/db/main/Main.java
index 9109a75..093f002 100644
--- a/src/org/torproject/ernie/db/main/Main.java
+++ b/src/org/torproject/ernie/db/main/Main.java
@@ -51,7 +51,7 @@ public class Main {
 
     // Download exit list and store it to disk
     if (config.getDownloadExitList()) {
-      new ExitListDownloader();
+      new ExitListDownloader(config);
     }
 
     // Process bridge pool assignments
@@ -64,22 +64,6 @@ public class Main {
       new TorperfDownloader(config);
     }
 
-    // Copy recently published files to a local directory that can then
-    // be served via rsync.
-    if (config.getProvideFilesViaRsync()) {
-      new RsyncDataProvider(
-          !config.getWriteDirectoryArchives() ? null :
-          new File(config.getDirectoryArchivesOutputDirectory()),
-          !config.getWriteSanitizedBridges() ? null :
-          new File(config.getSanitizedBridgesWriteDirectory()),
-          !config.getProcessBridgePoolAssignments() ? null :
-          new File(config.getSanitizedAssignmentsDirectory()),
-          config.getDownloadExitList(),
-          !config.getProcessTorperfFiles() ? null :
-          new File(config.getTorperfOutputDirectory()),
-          new File(config.getRsyncDirectory()));
-    }
-
     // Remove lock file
     lf.releaseLock();
 
diff --git a/src/org/torproject/ernie/db/main/RsyncDataProvider.java b/src/org/torproject/ernie/db/main/RsyncDataProvider.java
index cd4a6f9..78ea1f3 100644
--- a/src/org/torproject/ernie/db/main/RsyncDataProvider.java
+++ b/src/org/torproject/ernie/db/main/RsyncDataProvider.java
@@ -18,29 +18,38 @@ import java.util.logging.Logger;
  * then be served via rsync.
  */
 public class RsyncDataProvider {
-  public RsyncDataProvider(File directoryArchivesOutputDirectory,
-      File sanitizedBridgesWriteDirectory,
-      File sanitizedAssignmentsDirectory,
-      boolean downloadExitList,
-      File torperfOutputDirectory, File rsyncDirectory) {
+
+  private Logger logger;
+
+  private long cutOffMillis;
+
+  private File rsyncDirectory;
+
+  public RsyncDataProvider(File rsyncDirectory) {
 
     /* Initialize logger. */
-    Logger logger = Logger.getLogger(RsyncDataProvider.class.getName());
+    this.logger = Logger.getLogger(RsyncDataProvider.class.getName());
 
     /* Determine the cut-off time for files in rsync/. */
-    long cutOffMillis = System.currentTimeMillis()
+    this.cutOffMillis = System.currentTimeMillis()
         - 3L * 24L * 60L * 60L * 1000L;
 
     /* Create rsync/ directory if it doesn't exist. */
     if (!rsyncDirectory.exists()) {
       rsyncDirectory.mkdirs();
     }
+    this.rsyncDirectory = rsyncDirectory;
+  }
 
-    /* Make a list of all files in the rsync/ directory to delete those
+  public void copyFiles(File fromDirectory, String toRsyncSubDirectory) {
+
+    File toDirectory = new File(this.rsyncDirectory, toRsyncSubDirectory);
+
+    /* Make a list of all files in the rsync/ subdirectory to delete those
      * that we didn't copy in this run. */
     Set<String> fileNamesInRsync = new HashSet<String>();
     Stack<File> files = new Stack<File>();
-    files.add(rsyncDirectory);
+    files.add(toDirectory);
     while (!files.isEmpty()) {
       File pop = files.pop();
       if (pop.isDirectory()) {
@@ -50,132 +59,24 @@ public class RsyncDataProvider {
       }
     }
     logger.info("Found " + fileNamesInRsync.size() + " files in "
-        + rsyncDirectory.getAbsolutePath() + " that we're either "
+        + toDirectory.getAbsolutePath() + " that we're either "
         + "overwriting or deleting in this execution.");
 
-    /* Copy relay descriptors from the last 3 days. */
-    if (directoryArchivesOutputDirectory != null) {
-      files.add(directoryArchivesOutputDirectory);
-      while (!files.isEmpty()) {
-        File pop = files.pop();
-        if (pop.isDirectory()) {
-          files.addAll(Arrays.asList(pop.listFiles()));
-        } else if (pop.lastModified() >= cutOffMillis) {
-          String fileName = pop.getName();
-          if (pop.getAbsolutePath().contains("/consensus/")) {
-            this.copyFile(pop, new File(rsyncDirectory,
-                "relay-descriptors/consensuses/" + fileName));
-          } else if (pop.getAbsolutePath().contains("/vote/")) {
-            this.copyFile(pop, new File(rsyncDirectory,
-                "relay-descriptors/votes/" + fileName));
-          } else if (pop.getAbsolutePath().contains(
-                "/server-descriptor/")) {
-            this.copyFile(pop, new File(rsyncDirectory,
-                "relay-descriptors/server-descriptors/" + fileName));
-          } else if (pop.getAbsolutePath().contains("/extra-info/")) {
-            this.copyFile(pop, new File(rsyncDirectory,
-                "relay-descriptors/extra-infos/" + fileName));
-          } else {
-            continue;
-          }
-          fileNamesInRsync.remove(pop.getName());
-        }
-      }
-    }
-    logger.info("After copying relay descriptors, there are still "
-        + fileNamesInRsync.size() + " files left in "
-        + rsyncDirectory.getAbsolutePath() + ".");
-
-    /* Copy sanitized bridge descriptors from the last 3 days. */
-    if (sanitizedBridgesWriteDirectory != null) {
-      files.add(sanitizedBridgesWriteDirectory);
-      while (!files.isEmpty()) {
-        File pop = files.pop();
-        if (pop.isDirectory()) {
-          files.addAll(Arrays.asList(pop.listFiles()));
-        } else if (pop.lastModified() >= cutOffMillis) {
-          String fileName = pop.getName();
-          if (pop.getAbsolutePath().contains("/statuses/")) {
-            this.copyFile(pop, new File(rsyncDirectory,
-                "bridge-descriptors/statuses/" + fileName));
-          } else if (pop.getAbsolutePath().contains(
-                "/server-descriptors/")) {
-            this.copyFile(pop, new File(rsyncDirectory,
-                "bridge-descriptors/server-descriptors/" + fileName));
-          } else if (pop.getAbsolutePath().contains("/extra-infos/")) {
-            this.copyFile(pop, new File(rsyncDirectory,
-                "bridge-descriptors/extra-infos/" + fileName));
-          } else {
-            continue;
-          }
-          fileNamesInRsync.remove(pop.getName());
-        }
-      }
-    }
-    logger.info("After copying sanitized bridge descriptors, there are "
-        + "still " + fileNamesInRsync.size() + " files left in "
-        + rsyncDirectory.getAbsolutePath() + ".");
-
-    /* Copy sanitized bridge pool assignments from the last 3 days. */
-    if (sanitizedAssignmentsDirectory != null) {
-      files.add(sanitizedAssignmentsDirectory);
-      while (!files.isEmpty()) {
-        File pop = files.pop();
-        if (pop.isDirectory()) {
-          files.addAll(Arrays.asList(pop.listFiles()));
-        } else if (pop.lastModified() >= cutOffMillis) {
-          String fileName = pop.getName();
-          this.copyFile(pop, new File(rsyncDirectory,
-              "bridge-pool-assignments/" + fileName));
-          fileNamesInRsync.remove(pop.getName());
-        }
-      }
-    }
-    logger.info("After copying sanitized bridge pool assignments, there "
-        + "are still " + fileNamesInRsync.size() + " files left in "
-        + rsyncDirectory.getAbsolutePath() + ".");
-
-    /* Copy exit lists from the last 3 days. */
-    if (downloadExitList) {
-      files.add(new File("exitlist"));
-      while (!files.isEmpty()) {
-        File pop = files.pop();
-        if (pop.isDirectory()) {
-          files.addAll(Arrays.asList(pop.listFiles()));
-        } else if (pop.lastModified() >= cutOffMillis) {
-          String fileName = pop.getName();
-          this.copyFile(pop, new File(rsyncDirectory,
-              "exit-lists/" + fileName));
-          fileNamesInRsync.remove(pop.getName());
-        }
-      }
-    }
-    logger.info("After copying exit lists, there are still "
-        + fileNamesInRsync.size() + " files left in "
-        + rsyncDirectory.getAbsolutePath() + ".");
-
-    /* Copy Torperf files. */
-    if (torperfOutputDirectory != null) {
-      files.add(torperfOutputDirectory);
-      while (!files.isEmpty()) {
-        File pop = files.pop();
-        if (pop.isDirectory()) {
-          files.addAll(Arrays.asList(pop.listFiles()));
-        } else if (pop.getName().endsWith(".tpf") &&
-            pop.lastModified() >= cutOffMillis) {
-          String fileName = pop.getName();
-          this.copyFile(pop, new File(rsyncDirectory,
-              "torperf/" + fileName));
-          fileNamesInRsync.remove(pop.getName());
-        }
+    /* Copy files modified in the last 3 days. */
+    files.add(fromDirectory);
+    while (!files.isEmpty()) {
+      File pop = files.pop();
+      if (pop.isDirectory()) {
+        files.addAll(Arrays.asList(pop.listFiles()));
+      } else if (pop.lastModified() >= this.cutOffMillis) {
+        String fileName = pop.getName();
+        this.copyFile(pop, new File(toDirectory, fileName));
+        fileNamesInRsync.remove(fileName);
       }
     }
-    logger.info("After copying Torperf files, there are still "
-        + fileNamesInRsync.size() + " files left in "
-        + rsyncDirectory.getAbsolutePath() + ".");
 
     /* Delete all files that we didn't (over-)write in this run. */
-    files.add(rsyncDirectory);
+    files.add(toDirectory);
     while (!files.isEmpty()) {
       File pop = files.pop();
       if (pop.isDirectory()) {
@@ -187,7 +88,7 @@ public class RsyncDataProvider {
     }
     logger.info("After deleting files that we didn't overwrite in this "
         + "run, there are " + fileNamesInRsync.size() + " files left in "
-        + rsyncDirectory.getAbsolutePath() + ".");
+        + toDirectory.getAbsolutePath() + ".");
   }
 
   private void copyFile(File from, File to) {
diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
index d53f726..8d7e413 100644
--- a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
+++ b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
@@ -25,6 +25,7 @@ import org.torproject.descriptor.DescriptorParser;
 import org.torproject.descriptor.DescriptorSourceFactory;
 import org.torproject.descriptor.impl.DescriptorParseException;
 import org.torproject.ernie.db.main.Configuration;
+import org.torproject.ernie.db.main.RsyncDataProvider;
 
 public class ArchiveWriter {
   private Logger logger;
@@ -84,6 +85,25 @@ public class ArchiveWriter {
 
     // Write output to disk that only depends on relay descriptors
     this.dumpStats();
+
+    /* Copy relay descriptors from the last 3 days to the rsync
+     * directory. */
+    if (config.getProvideFilesViaRsync()) {
+      RsyncDataProvider rsdp = new RsyncDataProvider(
+          new File(config.getRsyncDirectory()));
+      rsdp.copyFiles(
+          new File(outputDirectory, "consensus"),
+          "relay-descriptors/consensuses");
+      rsdp.copyFiles(
+          new File(outputDirectory, "vote"),
+          "relay-descriptors/votes");
+      rsdp.copyFiles(
+          new File(outputDirectory, "server-descriptor"),
+          "relay-descriptors/server-descriptors");
+      rsdp.copyFiles(
+          new File(outputDirectory, "extra-info"),
+          "relay-descriptors/extra-infos");
+    }
   }
 
   private boolean store(byte[] typeAnnotation, byte[] data,
diff --git a/src/org/torproject/ernie/db/torperf/TorperfDownloader.java b/src/org/torproject/ernie/db/torperf/TorperfDownloader.java
index a931965..07b444e 100644
--- a/src/org/torproject/ernie/db/torperf/TorperfDownloader.java
+++ b/src/org/torproject/ernie/db/torperf/TorperfDownloader.java
@@ -21,6 +21,7 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import org.torproject.ernie.db.main.Configuration;
+import org.torproject.ernie.db.main.RsyncDataProvider;
 
 /* Download possibly truncated Torperf .data and .extradata files from
  * configured sources, append them to the files we already have, and merge
@@ -54,6 +55,13 @@ public class TorperfDownloader {
       this.downloadAndMergeFiles(torperfFilesLine);
     }
     this.writeLastMergedTimestamps();
+
+    /* Copy Torperf files from the last 3 days to the rsync directory. */
+    if (config.getProvideFilesViaRsync()) {
+      RsyncDataProvider rdp = new RsyncDataProvider(
+          new File(config.getRsyncDirectory()));
+      rdp.copyFiles(torperfOutputDirectory, "torperf");
+    }
   }
 
   private File torperfLastMergedFile =



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits