[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [collector/master] Added property OutputPath and removed four obsolete path properties.

To: tor-commits@xxxxxxxxxxxxxxxxxxxx
Subject: [tor-commits] [collector/master] Added property OutputPath and removed four obsolete path properties.
From: karsten@xxxxxxxxxxxxxx
Date: Wed, 12 Oct 2016 15:30:58 +0000 (UTC)
Delivered-to: archiver@xxxxxxxx
Delivery-date: Wed, 12 Oct 2016 11:31:11 -0400
List-archive: <http://lists.torproject.org/pipermail/tor-commits/>
List-help: <mailto:tor-commits-request@lists.torproject.org?subject=help>
List-id: "auto: code repository commits" <tor-commits.lists.torproject.org>
List-post: <mailto:tor-commits@lists.torproject.org>
List-subscribe: <https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits>, <mailto:tor-commits-request@lists.torproject.org?subject=subscribe>
List-unsubscribe: <https://lists.torproject.org/cgi-bin/mailman/options/tor-commits>, <mailto:tor-commits-request@lists.torproject.org?subject=unsubscribe>
Patch-author: iwakeh <iwakeh@xxxxxxxxxxxxxx>
Sender: "tor-commits" <tor-commits-bounces@xxxxxxxxxxxxxxxxxxxx>
commit 5003567318b8cb694cb46462c4819b8cbcf2eaf4
Author: iwakeh <iwakeh@xxxxxxxxxxxxxx>
Date:   Wed Oct 12 14:31:01 2016 +0200

    Added property OutputPath and removed four obsolete path properties.
    Added new methods to Configuration.
    Implements part of task-20162 and prepares task-18910.
---
 CHANGELOG.md                                       |  7 ++++++
 .../bridgedescs/SanitizedBridgesWriter.java        | 19 +++++++++++----
 .../torproject/collector/conf/Configuration.java   | 27 +++++++++++++++++++++-
 .../java/org/torproject/collector/conf/Key.java    |  8 +++----
 .../collector/exitlists/ExitListDownloader.java    | 25 ++++++++++++--------
 .../collector/relaydescs/ArchiveWriter.java        |  5 ++--
 .../collector/torperf/TorperfDownloader.java       | 10 ++++----
 src/main/resources/collector.properties            | 15 ++++--------
 .../bridgedescs/SanitizedBridgesWriterTest.java    |  8 +++----
 .../collector/conf/ConfigurationTest.java          | 14 ++++++-----
 10 files changed, 92 insertions(+), 46 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 33e77f6..fffad5e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,10 @@
+# Changes in version 1.1.0 - 2016-10-XX
+
+ * Medium changes
+   - Replace four properties for configuring where to write
+     descriptors by a single 'OutPath' property.
+
+
 # Changes in version 1.0.2 - 2016-10-07
 
  * Medium changes
diff --git a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
index 6d82fc6..00aa54b 100644
--- a/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
+++ b/src/main/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriter.java
@@ -23,6 +23,7 @@ import java.io.FileWriter;
 import java.io.IOException;
 import java.io.StringReader;
 import java.io.UnsupportedEncodingException;
+import java.nio.file.Paths;
 import java.security.GeneralSecurityException;
 import java.security.SecureRandom;
 import java.text.ParseException;
@@ -53,6 +54,7 @@ public class SanitizedBridgesWriter extends CollecTorMain {
 
   private static final Logger logger = LoggerFactory.getLogger(
       SanitizedBridgesWriter.class);
+  private static final String BRIDGE_DESCRIPTORS = "bridge-descriptors";
 
   public SanitizedBridgesWriter(Configuration config) {
     super(config);
@@ -81,6 +83,10 @@ public class SanitizedBridgesWriter extends CollecTorMain {
 
   private SecureRandom secureRandom;
 
+  private String outputPathName;
+
+  private String recentPathName;
+
   @Override
   public String module() {
     return "bridgedescs";
@@ -89,10 +95,13 @@ public class SanitizedBridgesWriter extends CollecTorMain {
   @Override
   protected void startProcessing() throws ConfigurationException {
 
+    outputPathName = Paths.get(config.getPath(Key.OutputPath).toString(),
+        BRIDGE_DESCRIPTORS).toString();
+    recentPathName = Paths.get(config.getPath(Key.RecentPath).toString(),
+        BRIDGE_DESCRIPTORS).toString();
     File bridgeDirectoriesDirectory =
         config.getPath(Key.BridgeSnapshotsDirectory).toFile();
-    File sanitizedBridgesDirectory =
-        config.getPath(Key.SanitizedBridgesWriteDirectory).toFile();
+    File sanitizedBridgesDirectory = new File(outputPathName);
     File statsDirectory = config.getPath(Key.StatsPath).toFile();
 
     if (bridgeDirectoriesDirectory == null
@@ -593,8 +602,8 @@ public class SanitizedBridgesWriter extends CollecTorMain {
           this.sanitizedBridgesDirectory.getAbsolutePath() + "/" + syear
           + "/" + smonth + "/statuses/" + sday + "/" + syear + smonth
           + sday + "-" + stime + "-" + authorityFingerprint);
-      File rsyncFile = new File(config.getPath(Key.RecentPath).toFile(),
-          "bridge-descriptors/statuses/" + tarballFile.getName());
+      File rsyncFile = new File(recentPathName, "statuses/"
+          + tarballFile.getName());
       File[] outputFiles = new File[] { tarballFile, rsyncFile };
       for (File outputFile : outputFiles) {
         outputFile.getParentFile().mkdirs();
@@ -1373,7 +1382,7 @@ public class SanitizedBridgesWriter extends CollecTorMain {
         - 3L * 24L * 60L * 60L * 1000L;
     Stack<File> allFiles = new Stack<File>();
     allFiles.add(new File(config.getPath(Key.RecentPath).toFile(),
-        "bridge-descriptors"));
+        BRIDGE_DESCRIPTORS));
     while (!allFiles.isEmpty()) {
       File file = allFiles.pop();
       if (file.isDirectory()) {
diff --git a/src/main/java/org/torproject/collector/conf/Configuration.java b/src/main/java/org/torproject/collector/conf/Configuration.java
index 52e3bad..b0b76ed 100644
--- a/src/main/java/org/torproject/collector/conf/Configuration.java
+++ b/src/main/java/org/torproject/collector/conf/Configuration.java
@@ -113,6 +113,11 @@ public class Configuration extends Observable implements Cloneable {
     return props.getProperty(key);
   }
 
+  /** Retrieves the value for key returning a default for non-existing keys. */
+  public String getProperty(String key, String def) {
+    return props.getProperty(key, def);
+  }
+
   /** Sets the value for key. */
   public void setProperty(String key, String value) {
     props.setProperty(key, value);
@@ -253,10 +258,30 @@ public class Configuration extends Observable implements Cloneable {
     try {
       checkClass(key, URL.class);
       return new URL(props.getProperty(key.name()));
-    } catch (MalformedURLException mue) {
+    } catch (MalformedURLException | RuntimeException mue) {
       throw new ConfigurationException("Corrupt property: " + key
           + " reason: " + mue.getMessage(), mue);
     }
   }
 
+  /**
+   * Returns {@code URL[]} from a property. Commas seperate array elements,
+   * e.g.,
+   * {@code propertyname = a1.example.org, a2.example2.com, a3.example3.net}
+   */
+  public URL[] getUrlArray(Key key) throws ConfigurationException {
+    try {
+      checkClass(key, URL[].class);
+      String[] interim = props.getProperty(key.name()).split(FIELDSEP);
+      URL[] res = new URL[interim.length];
+      for (int i = 0; i < interim.length; i++) {
+        res[i] = new URL(interim[i].trim());
+      }
+      return res;
+    } catch (MalformedURLException | RuntimeException re) {
+      throw new ConfigurationException("Corrupt property: " + key
+          + " reason: " + re.getMessage(), re);
+    }
+  }
+
 }
diff --git a/src/main/java/org/torproject/collector/conf/Key.java b/src/main/java/org/torproject/collector/conf/Key.java
index 5ec1be4..613dc7a 100644
--- a/src/main/java/org/torproject/collector/conf/Key.java
+++ b/src/main/java/org/torproject/collector/conf/Key.java
@@ -1,3 +1,6 @@
+/* Copyright 2016 The Tor Project
+ * See LICENSE for licensing information */
+
 package org.torproject.collector.conf;
 
 import java.net.URL;
@@ -11,11 +14,11 @@ public enum Key {
 
   ShutdownGraceWaitMinutes(Long.class),
   RunOnce(Boolean.class),
-  ExitlistOutputDirectory(Path.class),
   ExitlistUrl(URL.class),
   InstanceBaseUrl(String.class),
   ArchivePath(Path.class),
   RecentPath(Path.class),
+  OutputPath(Path.class),
   IndexPath(Path.class),
   StatsPath(Path.class),
   BridgedescsActivated(Boolean.class),
@@ -37,7 +40,6 @@ public enum Key {
   CachedRelayDescriptorsDirectories(String[].class),
   CompressRelayDescriptorDownloads(Boolean.class),
   DirectoryArchivesDirectory(Path.class),
-  DirectoryArchivesOutputDirectory(Path.class),
   DownloadRelayDescriptors(Boolean.class),
   DirectoryAuthoritiesAddresses(String[].class),
   DirectoryAuthoritiesFingerprintsForVotes(String[].class),
@@ -54,8 +56,6 @@ public enum Key {
   KeepDirectoryArchiveImportHistory(Boolean.class),
   ReplaceIpAddressesWithHashes(Boolean.class),
   BridgeDescriptorMappingsLimit(Integer.class),
-  SanitizedBridgesWriteDirectory(Path.class),
-  TorperfOutputDirectory(Path.class),
   TorperfFilesLines(String[].class),
   TorperfSources(String[][].class);
 
diff --git a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java
index 2f71d2e..5609cea 100644
--- a/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java
+++ b/src/main/java/org/torproject/collector/exitlists/ExitListDownloader.java
@@ -23,6 +23,7 @@ import java.io.FileWriter;
 import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.URL;
+import java.nio.file.Paths;
 import java.text.SimpleDateFormat;
 import java.util.Arrays;
 import java.util.Date;
@@ -37,6 +38,12 @@ public class ExitListDownloader extends CollecTorMain {
   private static final Logger logger = LoggerFactory.getLogger(
       ExitListDownloader.class);
 
+  private static final String EXITLISTS = "exit-lists";
+
+  private String outputPathName;
+
+  private String recentPathName;
+
   /** Instanciate the exit-lists module using the given configuration. */
   public ExitListDownloader(Configuration config) {
     super(config);
@@ -53,7 +60,10 @@ public class ExitListDownloader extends CollecTorMain {
     SimpleDateFormat dateTimeFormat =
         new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
     dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-
+    outputPathName = Paths.get(config.getPath(Key.OutputPath).toString(),
+        EXITLISTS).toString();
+    recentPathName = Paths.get(config.getPath(Key.RecentPath).toString(),
+        EXITLISTS).toString();
     Date downloadedDate = new Date();
     String downloadedExitList = null;
     try {
@@ -94,9 +104,8 @@ public class ExitListDownloader extends CollecTorMain {
     SimpleDateFormat tarballFormat =
         new SimpleDateFormat("yyyy/MM/dd/yyyy-MM-dd-HH-mm-ss");
     tarballFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-    File tarballFile = new File(
-        config.getPath(Key.ExitlistOutputDirectory).toFile(),
-        tarballFormat.format(downloadedDate));
+    File tarballFile = Paths.get(outputPathName,
+        tarballFormat.format(downloadedDate)).toFile();
 
     long maxScanMillis = 0L;
     try {
@@ -128,8 +137,7 @@ public class ExitListDownloader extends CollecTorMain {
     }
 
     /* Write to disk. */
-    File rsyncFile = new File(config.getPath(Key.RecentPath).toFile(),
-        "exit-lists/" + tarballFile.getName());
+    File rsyncFile = new File(recentPathName, tarballFile.getName());
     File[] outputFiles = new File[] { tarballFile, rsyncFile };
     for (File outputFile : outputFiles) {
       try {
@@ -148,7 +156,7 @@ public class ExitListDownloader extends CollecTorMain {
     StringBuilder dumpStats = new StringBuilder("Finished downloading "
         + "exit list.\nLast three exit lists are:");
     Stack<File> filesInInputDir = new Stack<File>();
-    filesInInputDir.add(config.getPath(Key.ExitlistOutputDirectory).toFile());
+    filesInInputDir.add(new File(outputPathName));
     SortedSet<File> lastThreeExitLists = new TreeSet<File>();
     while (!filesInInputDir.isEmpty()) {
       File pop = filesInInputDir.pop();
@@ -184,8 +192,7 @@ public class ExitListDownloader extends CollecTorMain {
     long cutOffMillis = System.currentTimeMillis()
         - 3L * 24L * 60L * 60L * 1000L;
     Stack<File> allFiles = new Stack<File>();
-    allFiles.add(new File(config.getPath(Key.RecentPath).toFile(),
-        "/exit-lists"));
+    allFiles.add(new File(recentPathName, EXITLISTS));
     while (!allFiles.isEmpty()) {
       File file = allFiles.pop();
       if (file.isDirectory()) {
diff --git a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java
index 59b7969..cac5d14 100644
--- a/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java
+++ b/src/main/java/org/torproject/collector/relaydescs/ArchiveWriter.java
@@ -129,8 +129,9 @@ public class ArchiveWriter extends CollecTorMain {
     storedMicrodescriptorsFile
         = new File(statsDir, "stored-microdescriptors");
     File statsDirectory = config.getPath(Key.StatsPath).toFile();
-    this.outputDirectory = config.getPath(Key.DirectoryArchivesOutputDirectory)
-        .toString();
+    this.outputDirectory
+        = Paths.get(config.getPath(Key.OutputPath).toString(),
+              RELAY_DESCRIPTORS).toString();
     SimpleDateFormat rsyncCatFormat = new SimpleDateFormat(
         "yyyy-MM-dd-HH-mm-ss");
     rsyncCatFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
diff --git a/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java b/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java
index 635c5a3..7c9367e 100644
--- a/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java
+++ b/src/main/java/org/torproject/collector/torperf/TorperfDownloader.java
@@ -37,6 +37,8 @@ public class TorperfDownloader extends CollecTorMain {
   private static final Logger logger = LoggerFactory.getLogger(
       TorperfDownloader.class);
 
+  private static final String TORPERF = "torperf";
+
   public TorperfDownloader(Configuration config) {
     super(config);
   }
@@ -49,14 +51,14 @@ public class TorperfDownloader extends CollecTorMain {
 
   @Override
   public String module() {
-    return "torperf";
+    return TORPERF;
   }
 
   @Override
   protected void startProcessing() throws ConfigurationException {
     this.torperfFilesLines = config.getStringArray(Key.TorperfFilesLines);
-    this.torperfOutputDirectory = config.getPath(Key.TorperfOutputDirectory)
-        .toFile();
+    this.torperfOutputDirectory
+        = new File(config.getPath(Key.OutputPath).toString(), TORPERF);
     this.torperfLastMergedFile = new File(config.getPath(Key.StatsPath).toFile(),
        "torperf-last-merged");
     if (!this.torperfOutputDirectory.exists()) {
@@ -617,7 +619,7 @@ public class TorperfDownloader extends CollecTorMain {
     long cutOffMillis = System.currentTimeMillis()
         - 3L * 24L * 60L * 60L * 1000L;
     Stack<File> allFiles = new Stack<File>();
-    allFiles.add(new File(config.getPath(Key.RecentPath).toFile(), "torperf"));
+    allFiles.add(new File(config.getPath(Key.RecentPath).toFile(), TORPERF));
     while (!allFiles.isEmpty()) {
       File file = allFiles.pop();
       if (file.isDirectory()) {
diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties
index 0c93cd0..1c02c92 100644
--- a/src/main/resources/collector.properties
+++ b/src/main/resources/collector.properties
@@ -59,6 +59,9 @@ ArchivePath = archive
 # The top-level directory for the recent descriptors that were
 # published in the last 72 hours.
 RecentPath = recent
+# The top-level directory for the retrieved descriptors that will
+# be archived.
+OutputPath = out
 # Some statistics are stored here.
 StatsPath = stats
 ######## Relay descriptors ########
@@ -126,9 +129,6 @@ DownloadAllExtraInfoDescriptors = false
 ## Compress relay descriptors downloads by adding .z to the URLs
 CompressRelayDescriptorDownloads = false
 #
-## Path to directory to write directory archives to
-DirectoryArchivesOutputDirectory = out/relay-descriptors/
-#
 #
 ######## Bridge descriptors ########
 #
@@ -144,21 +144,14 @@ ReplaceIpAddressesWithHashes = false
 ## of days, or inf for unlimited.
 BridgeDescriptorMappingsLimit = inf
 #
-## Path to directory to write sanitized bridges to
-SanitizedBridgesWriteDirectory = out/bridge-descriptors/
-
+#
 ######## Exit lists ########
 #
-##
-ExitlistOutputDirectory = out/exit-lists/
 ## Where to download exit-lists from.
 ExitlistUrl = https://check.torproject.org/exit-addresses
 
 ######## Torperf downloader ########
 #
-## Path to the directory to store Torperf files in.
-TorperfOutputDirectory = out/torperf/
-
 ## Torperf source names and base URLs
 ## multiple pairs can be specified separated by semi-colon, e.g.
 ## TorperfSourceName = torperf_A, http://some.torproject.org/; another, http://another.torproject.org/
diff --git a/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java b/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java
index 7468b23..299bd92 100644
--- a/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java
+++ b/src/test/java/org/torproject/collector/bridgedescs/SanitizedBridgesWriterTest.java
@@ -101,7 +101,7 @@ public class SanitizedBridgesWriterTest {
     this.recentDirectory = this.temporaryFolder.newFolder("recent");
     this.statsDirectory = this.temporaryFolder.newFolder("stats").toString();
     this.sanitizedBridgesDirectory =
-        this.temporaryFolder.newFolder("out").toPath();
+        this.temporaryFolder.newFolder("out", "bridge-descriptors").toPath();
     this.initializeTestConfiguration();
     this.defaultServerDescriptorBuilder = new ServerDescriptorBuilder();
     this.defaultExtraInfoDescriptorBuilder = new ExtraInfoDescriptorBuilder();
@@ -134,8 +134,8 @@ public class SanitizedBridgesWriterTest {
     this.configuration.setProperty(Key.StatsPath.name(), statsDirectory);
     this.configuration.setProperty(Key.BridgeSnapshotsDirectory.name(),
         bridgeDirectoriesDir);
-    this.configuration.setProperty(Key.SanitizedBridgesWriteDirectory.name(),
-        sanitizedBridgesDirectory.toString());
+    this.configuration.setProperty(Key.OutputPath.name(),
+        sanitizedBridgesDirectory.toFile().getParent().toString());
   }
 
   /** Runs this test by executing all builders, performing the sanitizing
@@ -468,7 +468,7 @@ public class SanitizedBridgesWriterTest {
         "router-digest B026CF0F81712D94BBF1362294882688DF247887");
     assertEquals("Sanitized descriptor does not contain expected lines.",
         expectedLines, this.parsedExtraInfoDescriptors.get(0));
-    assertTrue("Sanitized descriptor file name differs.",
+    assertTrue("Sanitized descriptor file name differs. " + this.parsedFiles,
         this.parsedFiles.containsKey("2016/06/extra-infos/b/0/"
         + "b026cf0f81712d94bbf1362294882688df247887"));
   }
diff --git a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java
index fdba6b5..8e255c6 100644
--- a/src/test/java/org/torproject/collector/conf/ConfigurationTest.java
+++ b/src/test/java/org/torproject/collector/conf/ConfigurationTest.java
@@ -8,6 +8,7 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
+import org.torproject.collector.Main;
 import org.torproject.collector.MainTest;
 import org.torproject.collector.cron.CollecTorMain;
 import org.torproject.collector.cron.Dummy;
@@ -19,6 +20,7 @@ import org.junit.rules.TemporaryFolder;
 import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.lang.reflect.Field;
+import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.Arrays;
@@ -38,16 +40,16 @@ public class ConfigurationTest {
   public void testKeyCount() throws Exception {
     assertEquals("The number of properties keys in enum Key changed."
         + "\n This test class should be adapted.",
-        49, Key.values().length);
+        46, Key.values().length);
   }
 
   @Test()
   public void testConfiguration() throws Exception {
     Configuration conf = new Configuration();
     String val = "xyz";
-    conf.setProperty(Key.TorperfOutputDirectory.name(), val);
+    conf.setProperty(Key.OutputPath.name(), val);
     assertEquals(1, conf.size());
-    assertEquals(val, conf.getProperty(Key.TorperfOutputDirectory.name()));
+    assertEquals(val, conf.getProperty(Key.OutputPath.name()));
   }
 
   private String propLine(Key key, String val) {
@@ -108,9 +110,9 @@ public class ConfigurationTest {
     Configuration conf = new Configuration();
     for (String file : files) {
       conf.clear();
-      conf.setProperty(Key.DirectoryArchivesOutputDirectory.name(), file);
+      conf.setProperty(Key.OutputPath.name(), file);
       assertEquals(new File(file),
-          conf.getPath(Key.DirectoryArchivesOutputDirectory).toFile());
+          conf.getPath(Key.OutputPath).toFile());
     }
   }
 
@@ -131,7 +133,7 @@ public class ConfigurationTest {
   public void testArrayArrayValueException() throws Exception {
     Configuration conf = new Configuration();
     conf.setProperty(Key.CachedRelayDescriptorsDirectories.name(), "");
-    conf.getStringArrayArray(Key.TorperfOutputDirectory);
+    conf.getStringArrayArray(Key.OutputPath);
   }
 
   @Test(expected = ConfigurationException.class)

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits
Prev by Author: [tor-commits] [collector/master] Added tests for file-storage using PersistenceUtils.
Next by Author: [tor-commits] [collector/master] Move index.json* files to index/ subdirectory.
Previous by thread: [tor-commits] [tor/master] Fix a bug in displaying IPv6 addrs in test_op_ipv6_ with --verbose
Next by thread: [tor-commits] [translation/tails-greeter] Update translations for tails-greeter
Index(es):
- Author
- Thread