[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [collector/master] Implements #19424: make all main paths configurable, add property instance url.

To: tor-commits@xxxxxxxxxxxxxxxxxxxx
Subject: [tor-commits] [collector/master] Implements #19424: make all main paths configurable, add property instance url.
From: karsten@xxxxxxxxxxxxxx
Date: Wed, 20 Jul 2016 07:38:26 +0000 (UTC)
Delivered-to: archiver@xxxxxxxx
Delivery-date: Wed, 20 Jul 2016 03:38:43 -0400
List-archive: <http://lists.torproject.org/pipermail/tor-commits/>
List-help: <mailto:tor-commits-request@lists.torproject.org?subject=help>
List-id: "auto: code repository commits" <tor-commits.lists.torproject.org>
List-post: <mailto:tor-commits@lists.torproject.org>
List-subscribe: <https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits>, <mailto:tor-commits-request@lists.torproject.org?subject=subscribe>
List-unsubscribe: <https://lists.torproject.org/cgi-bin/mailman/options/tor-commits>, <mailto:tor-commits-request@lists.torproject.org?subject=unsubscribe>
Patch-author: iwakeh <iwakeh@xxxxxxxxxxxxxx>
Sender: "tor-commits" <tor-commits-bounces@xxxxxxxxxxxxxxxxxxxx>
commit 2f725efbf6a527bee8663fca3751da7069787625
Author: iwakeh <iwakeh@xxxxxxxxxxxxxx>
Date:   Tue Jul 19 15:32:31 2016 +0200

    Implements #19424: make all main paths configurable, add property instance url.
---
 .../bridgedescs/SanitizedBridgesWriter.java        | 28 ++++++++++++----------
 .../java/org/torproject/collector/conf/Key.java    |  2 ++
 .../collector/exitlists/ExitListDownloader.java    | 22 ++++++++++-------
 .../collector/index/CreateIndexJson.java           |  7 +++---
 .../collector/torperf/TorperfDownloader.java       | 25 +++++++++++--------
 src/main/resources/collector.properties            |  7 +++---
 .../collector/conf/ConfigurationTest.java          |  2 +-
 7 files changed, 55 insertions(+), 38 deletions(-)

diff --git a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
index 9d5828c..121f8ca 100644
--- a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
+++ b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
@@ -120,7 +120,7 @@ public class SanitizedBridgesWriter extends Thread {
         config.getPath(Key.BridgeSnapshotsDirectory).toFile();
     File sanitizedBridgesDirectory =
         config.getPath(Key.SanitizedBridgesWriteDirectory).toFile();
-    File statsDirectory = new File("stats");
+    File statsDirectory = config.getPath(Key.StatsPath).toFile();
 
     if (bridgeDirectoriesDirectory == null
         || sanitizedBridgesDirectory == null || statsDirectory == null) {
@@ -923,12 +923,12 @@ public class SanitizedBridgesWriter extends Thread {
         + "/" + descriptorDigest.charAt(0) + "/"
         + descriptorDigest.charAt(1) + "/"
         + descriptorDigest);
-    File rsyncCatFile = new File("recent/bridge-descriptors/"
-        + "server-descriptors/" + this.rsyncCatString
-        + "-server-descriptors.tmp");
-    File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
-    boolean[] append = new boolean[] { false, true };
     try {
+      File rsyncCatFile = new File(config.getPath(Key.RecentPath).toFile(),
+          "bridge-descriptors/server-descriptors/" + this.rsyncCatString
+          + "-server-descriptors.tmp");
+      File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
+      boolean[] append = new boolean[] { false, true };
       for (int i = 0; i < outputFiles.length; i++) {
         File outputFile = outputFiles[i];
         boolean appendToFile = append[i];
@@ -950,7 +950,7 @@ public class SanitizedBridgesWriter extends Thread {
             + "\n");
         bw.close();
       }
-    } catch (IOException e) {
+    } catch (ConfigurationException | IOException e) {
       this.logger.warn("Could not write sanitized server "
           + "descriptor to disk.", e);
       return;
@@ -1201,11 +1201,12 @@ public class SanitizedBridgesWriter extends Thread {
         + descriptorDigest.charAt(0) + "/"
         + descriptorDigest.charAt(1) + "/"
         + descriptorDigest);
-    File rsyncCatFile = new File("recent/bridge-descriptors/"
-        + "extra-infos/" + this.rsyncCatString + "-extra-infos.tmp");
-    File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
-    boolean[] append = new boolean[] { false, true };
     try {
+      File rsyncCatFile = new File(config.getPath(Key.RecentPath).toFile(),
+          "bridge-descriptors/extra-infos/" + this.rsyncCatString
+          + "-extra-infos.tmp");
+      File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
+      boolean[] append = new boolean[] { false, true };
       for (int i = 0; i < outputFiles.length; i++) {
         File outputFile = outputFiles[i];
         boolean appendToFile = append[i];
@@ -1313,11 +1314,12 @@ public class SanitizedBridgesWriter extends Thread {
   /** Delete all files from the rsync directory that have not been modified
    * in the last three days, and remove the .tmp extension from newly
    * written files. */
-  public void cleanUpRsyncDirectory() {
+  public void cleanUpRsyncDirectory() throws ConfigurationException {
     long cutOffMillis = System.currentTimeMillis()
         - 3L * 24L * 60L * 60L * 1000L;
     Stack<File> allFiles = new Stack<File>();
-    allFiles.add(new File("recent/bridge-descriptors"));
+    allFiles.add(new File(config.getPath(Key.RecentPath).toFile(),
+        "bridge-descriptors"));
     while (!allFiles.isEmpty()) {
       File file = allFiles.pop();
       if (file.isDirectory()) {
diff --git a/src/main/java/org/torproject/collector/conf/Key.java b/src/main/java/org/torproject/collector/conf/Key.java
index 21e536d..4f5df64 100644
--- a/src/main/java/org/torproject/collector/conf/Key.java
+++ b/src/main/java/org/torproject/collector/conf/Key.java
@@ -8,6 +8,8 @@ import java.nio.file.Path;
  */
 public enum Key {
 
+  ExitlistOutputDirectory(Path.class),
+  InstanceBaseUrl(String.class),
   LockFilePath(Path.class),
   ArchivePath(Path.class),
   RecentPath(Path.class),
diff --git a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java
index 5e17971..70de74f 100644
--- a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java
+++ b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java
@@ -53,7 +53,12 @@ public class ExitListDownloader extends Thread {
     logger.info("Terminating exit-lists module of CollecTor.");
   }
 
-  public ExitListDownloader(Configuration config) {}
+  private Configuration config;
+
+  /** Instanciate the exit-lists module using the given configuration. */
+  public ExitListDownloader(Configuration config) {
+    this.config = config;
+  }
 
   @Override
   public void run() {
@@ -113,8 +118,8 @@ public class ExitListDownloader extends Thread {
     SimpleDateFormat tarballFormat =
         new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
     tarballFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-    File tarballFile = new File("out/exit-lists/" + tarballFormat.format(
-        downloadedDate));
+    File tarballFile = new File(config.getPath(Key.ExitlistOutputDirectory)
+        + tarballFormat.format(downloadedDate));
 
     long maxScanMillis = 0L;
     try {
@@ -146,8 +151,8 @@ public class ExitListDownloader extends Thread {
     }
 
     /* Write to disk. */
-    File rsyncFile = new File("recent/exit-lists/"
-        + tarballFile.getName());
+    File rsyncFile = new File(config.getPath(Key.RecentPath).toFile(),
+        "exit-lists/" + tarballFile.getName());
     File[] outputFiles = new File[] { tarballFile, rsyncFile };
     for (File outputFile : outputFiles) {
       try {
@@ -166,7 +171,7 @@ public class ExitListDownloader extends Thread {
     StringBuilder dumpStats = new StringBuilder("Finished downloading "
         + "exit list.\nLast three exit lists are:");
     Stack<File> filesInInputDir = new Stack<File>();
-    filesInInputDir.add(new File("out/exit-lists"));
+    filesInInputDir.add(config.getPath(Key.ExitlistOutputDirectory).toFile());
     SortedSet<File> lastThreeExitLists = new TreeSet<File>();
     while (!filesInInputDir.isEmpty()) {
       File pop = filesInInputDir.pop();
@@ -198,11 +203,12 @@ public class ExitListDownloader extends Thread {
 
   /** Delete all files from the rsync directory that have not been modified
    * in the last three days. */
-  public void cleanUpRsyncDirectory() {
+  public void cleanUpRsyncDirectory() throws ConfigurationException {
     long cutOffMillis = System.currentTimeMillis()
         - 3L * 24L * 60L * 60L * 1000L;
     Stack<File> allFiles = new Stack<File>();
-    allFiles.add(new File("recent/exit-lists"));
+    allFiles.add(new File(config.getPath(Key.RecentPath).toFile(),
+        "/exit-lists"));
     while (!allFiles.isEmpty()) {
       File file = allFiles.pop();
       if (file.isDirectory()) {
diff --git a/src/main/java/org/torproject/collector/index/CreateIndexJson.java b/src/main/java/org/torproject/collector/index/CreateIndexJson.java
index 821be59..7da2b5e 100644
--- a/src/main/java/org/torproject/collector/index/CreateIndexJson.java
+++ b/src/main/java/org/torproject/collector/index/CreateIndexJson.java
@@ -39,7 +39,7 @@ public class CreateIndexJson {
 
   private static File indexJsonFile;
 
-  private static String basePath = "https://collector.torproject.org";;
+  private static String basePath;
 
   private static File[] indexedDirectories;
 
@@ -54,9 +54,10 @@ public class CreateIndexJson {
   public static void main(Configuration config)
       throws ConfigurationException, IOException {
     indexJsonFile =  new File(config.getPath(Key.IndexPath).toFile(), "index.json");
+    basePath = config.getProperty(Key.InstanceBaseUrl.name());
     indexedDirectories = new File[] {
-        new File(config.getPath(Key.ArchivePath).toFile(), "archive"),
-        new File(config.getPath(Key.RecentPath).toFile(), "recent") };
+        config.getPath(Key.ArchivePath).toFile(),
+        config.getPath(Key.RecentPath).toFile() };
     writeIndex(indexDirectories());
   }
 
diff --git a/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java b/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java
index dd1c541..7616dd8 100644
--- a/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java
+++ b/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java
@@ -62,6 +62,7 @@ public class TorperfDownloader extends Thread {
   private Map<String, String> torperfSources = new HashMap<>();
   private String[] torperfFilesLines = null;
   private SimpleDateFormat dateFormat;
+  private File torperfLastMergedFile;
 
   @Override
   public void run() {
@@ -77,6 +78,8 @@ public class TorperfDownloader extends Thread {
     this.torperfFilesLines = config.getStringArray(Key.TorperfFilesLines);
     this.torperfOutputDirectory = config.getPath(Key.TorperfOutputDirectory)
         .toFile();
+    this.torperfLastMergedFile = new File(config.getPath(Key.StatsPath).toFile(),
+       "torperf-last-merged");
     if (!this.torperfOutputDirectory.exists()) {
       this.torperfOutputDirectory.mkdirs();
     }
@@ -94,9 +97,6 @@ public class TorperfDownloader extends Thread {
     this.cleanUpRsyncDirectory();
   }
 
-  private File torperfLastMergedFile =
-      new File("stats/torperf-last-merged");
-
   SortedMap<String, String> lastMergedTimestamps =
       new TreeMap<String, String>();
 
@@ -160,7 +160,8 @@ public class TorperfDownloader extends Thread {
     }
   }
 
-  private void downloadAndMergeFiles(String torperfFilesLine) {
+  private void downloadAndMergeFiles(String torperfFilesLine)
+      throws ConfigurationException {
     String[] parts = torperfFilesLine.split(" ");
     String sourceName = parts[0];
     int fileSize = -1;
@@ -306,7 +307,8 @@ public class TorperfDownloader extends Thread {
   }
 
   private String mergeFiles(File dataFile, File extradataFile,
-      String source, int fileSize, String skipUntil) throws IOException {
+      String source, int fileSize, String skipUntil) throws IOException,
+      ConfigurationException {
     SortedMap<String, String> config = new TreeMap<String, String>();
     config.put("SOURCE", source);
     config.put("FILESIZE", String.valueOf(fileSize));
@@ -548,7 +550,8 @@ public class TorperfDownloader extends Thread {
   private SortedMap<String, String> cachedTpfLines;
 
   private void writeTpfLine(String source, int fileSize,
-      SortedMap<String, String> keysAndValues) throws IOException {
+      SortedMap<String, String> keysAndValues) throws ConfigurationException,
+      IOException {
     StringBuilder sb = new StringBuilder();
     int written = 0;
     for (Map.Entry<String, String> keyAndValue :
@@ -602,7 +605,8 @@ public class TorperfDownloader extends Thread {
     br.close();
   }
 
-  private void writeCachedTpfLines() throws IOException {
+  private void writeCachedTpfLines() throws ConfigurationException,
+      IOException {
     if (this.cachedSource == null || this.cachedFileSize == 0
         || this.cachedStartDate == null || this.cachedTpfLines == null) {
       return;
@@ -612,7 +616,8 @@ public class TorperfDownloader extends Thread {
         + "/" + this.cachedSource + "-"
         + String.valueOf(this.cachedFileSize) + "-"
         + this.cachedStartDate + ".tpf");
-    File rsyncFile = new File("recent/torperf/" + tarballFile.getName());
+    File rsyncFile = new File(config.getPath(Key.RecentPath).toFile(),
+        "torperf/" + tarballFile.getName());
     File[] outputFiles = new File[] { tarballFile, rsyncFile };
     for (File outputFile : outputFiles) {
       outputFile.getParentFile().mkdirs();
@@ -631,11 +636,11 @@ public class TorperfDownloader extends Thread {
 
   /** Delete all files from the rsync directory that have not been modified
    * in the last three days. */
-  public void cleanUpRsyncDirectory() {
+  public void cleanUpRsyncDirectory() throws ConfigurationException {
     long cutOffMillis = System.currentTimeMillis()
         - 3L * 24L * 60L * 60L * 1000L;
     Stack<File> allFiles = new Stack<File>();
-    allFiles.add(new File("recent/torperf"));
+    allFiles.add(new File(config.getPath(Key.RecentPath).toFile(), "torperf"));
     while (!allFiles.isEmpty()) {
       File file = allFiles.pop();
       if (file.isDirectory()) {
diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties
index c7df21f..fb9e1b4 100644
--- a/src/main/resources/collector.properties
+++ b/src/main/resources/collector.properties
@@ -1,6 +1,7 @@
 ######## Collector Properties
 #
 ######## General Properties ########
+InstanceBaseUrl = "https://collector.torproject.org";
 LockFilePath = lock
 IndexPath = out/index
 ArchivePath = out/archive
@@ -95,9 +96,9 @@ SanitizedBridgesWriteDirectory = out/bridge-descriptors/
 
 ######## Exit lists ########
 #
-## (No options available)
-#
-#
+##
+ExitlistOutputDirectory = out/exit-lists/
+
 ######## Torperf downloader ########
 #
 ## Path to the directory to store Torperf files in.
diff --git a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java
index f966b43..57dda75 100644
--- a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java
+++ b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java
@@ -22,7 +22,7 @@ public class ConfigurationTest {
   public void testKeyCount() throws Exception {
     assertEquals("The number of properties keys in enum Key changed."
         + "\n This test class should be adapted.",
-        30, Key.values().length);
+        32, Key.values().length);
   }
 
   @Test()

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
Prev by Author: [tor-commits] [collector/master] Made directories configurable; added copyright, description, and path checks.
Next by Author: [tor-commits] [exonerator/master] Add checkstyle task.
Previous by thread: [tor-commits] [collector/master] Use the latest describtor release 1.3.0
Next by thread: [tor-commits] [tor-browser-bundle/master] Fold in stable changelog
Index(es):
- Author
- Thread