[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [metrics-web/master] Update to metrics-lib 1.9.0.



commit 9ee4fe35e08428ce3aae7946d01da843e8004f5b
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Sat Jun 17 17:42:00 2017 +0200

    Update to metrics-lib 1.9.0.
    
    Implements #22683.
---
 .../org/torproject/metrics/advbwdist/Main.java     | 170 ++++++++----------
 .../java/org/torproject/metrics/clients/Main.java  |  54 ++----
 .../org/torproject/metrics/connbidirect/Main.java  |  27 +--
 .../java/org/torproject/metrics/hidserv/Main.java  |   8 +-
 .../org/torproject/metrics/hidserv/Parser.java     |  33 ++--
 .../cron/RelayDescriptorDatabaseImporter.java      |  34 ++--
 .../cron/network/ConsensusStatsFileHandler.java    |  30 ++--
 .../ernie/cron/performance/TorperfProcessor.java   |  97 +++++-----
 .../org/torproject/metrics/onionperf/Main.java     | 197 ++++++++++-----------
 shared/build-base.xml                              |   2 +-
 10 files changed, 287 insertions(+), 365 deletions(-)

diff --git a/modules/advbwdist/src/main/java/org/torproject/metrics/advbwdist/Main.java b/modules/advbwdist/src/main/java/org/torproject/metrics/advbwdist/Main.java
index 7193108..8928be2 100644
--- a/modules/advbwdist/src/main/java/org/torproject/metrics/advbwdist/Main.java
+++ b/modules/advbwdist/src/main/java/org/torproject/metrics/advbwdist/Main.java
@@ -4,7 +4,6 @@
 package org.torproject.metrics.advbwdist;
 
 import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
 import org.torproject.descriptor.DescriptorReader;
 import org.torproject.descriptor.DescriptorSourceFactory;
 import org.torproject.descriptor.NetworkStatusEntry;
@@ -19,7 +18,6 @@ import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.TimeZone;
@@ -31,37 +29,27 @@ public class Main {
 
     /* Parse server descriptors, not keeping a parse history, and memorize
      * the advertised bandwidth for every server descriptor. */
+    Map<String, Long> serverDescriptors = new HashMap<>();
     DescriptorReader descriptorReader =
         DescriptorSourceFactory.createDescriptorReader();
-    descriptorReader.addDirectory(
-        new File("../../shared/in/recent/relay-descriptors/"
-        + "server-descriptors"));
-    Iterator<DescriptorFile> descriptorFiles =
-        descriptorReader.readDescriptors();
-    Map<String, Long> serverDescriptors = new HashMap<>();
-    while (descriptorFiles.hasNext()) {
-      DescriptorFile descriptorFile = descriptorFiles.next();
-      for (Descriptor descriptor : descriptorFile.getDescriptors()) {
-        if (!(descriptor instanceof ServerDescriptor)) {
-          continue;
-        }
-        ServerDescriptor serverDescriptor = (ServerDescriptor) descriptor;
-        String digest = serverDescriptor.getDigestSha1Hex();
-        long advertisedBandwidth = Math.min(Math.min(
-            serverDescriptor.getBandwidthRate(),
-            serverDescriptor.getBandwidthBurst()),
-            serverDescriptor.getBandwidthObserved());
-        serverDescriptors.put(digest.toUpperCase(), advertisedBandwidth);
+    for (Descriptor descriptor : descriptorReader.readDescriptors(new File(
+        "../../shared/in/recent/relay-descriptors/server-descriptors"))) {
+      if (!(descriptor instanceof ServerDescriptor)) {
+        continue;
       }
+      ServerDescriptor serverDescriptor = (ServerDescriptor) descriptor;
+      String digest = serverDescriptor.getDigestSha1Hex();
+      long advertisedBandwidth = Math.min(Math.min(
+          serverDescriptor.getBandwidthRate(),
+          serverDescriptor.getBandwidthBurst()),
+          serverDescriptor.getBandwidthObserved());
+      serverDescriptors.put(digest.toUpperCase(), advertisedBandwidth);
     }
 
     /* Parse consensuses, keeping a parse history. */
     descriptorReader = DescriptorSourceFactory.createDescriptorReader();
-    descriptorReader.addDirectory(
-        new File("../../shared/in/recent/relay-descriptors/consensuses"));
     File historyFile = new File("status/parsed-consensuses");
     descriptorReader.setHistoryFile(historyFile);
-    descriptorFiles = descriptorReader.readDescriptors();
     File resultsFile = new File("stats/advbwdist-validafter.csv");
     resultsFile.getParentFile().mkdirs();
     boolean writeHeader = !resultsFile.exists();
@@ -73,82 +61,80 @@ public class Main {
     SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
         "yyyy-MM-dd HH:mm:ss");
     dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-    while (descriptorFiles.hasNext()) {
-      DescriptorFile descriptorFile = descriptorFiles.next();
-      for (Descriptor descriptor : descriptorFile.getDescriptors()) {
-        if (!(descriptor instanceof RelayNetworkStatusConsensus)) {
+    for (Descriptor descriptor : descriptorReader.readDescriptors(new File(
+        "../../shared/in/recent/relay-descriptors/consensuses"))) {
+      if (!(descriptor instanceof RelayNetworkStatusConsensus)) {
+        continue;
+      }
+
+      /* Parse server descriptor digests from consensus and look up
+       * advertised bandwidths. */
+      RelayNetworkStatusConsensus consensus =
+          (RelayNetworkStatusConsensus) descriptor;
+      String validAfter = dateTimeFormat.format(
+          consensus.getValidAfterMillis());
+      List<Long> advertisedBandwidthsAllRelays = new ArrayList<>();
+      List<Long> advertisedBandwidthsExitsOnly = new ArrayList<>();
+      for (NetworkStatusEntry relay
+          : consensus.getStatusEntries().values()) {
+        if (!relay.getFlags().contains("Running")) {
           continue;
         }
-
-        /* Parse server descriptor digests from consensus and look up
-         * advertised bandwidths. */
-        RelayNetworkStatusConsensus consensus =
-            (RelayNetworkStatusConsensus) descriptor;
-        String validAfter = dateTimeFormat.format(
-            consensus.getValidAfterMillis());
-        List<Long> advertisedBandwidthsAllRelays = new ArrayList<>();
-        List<Long> advertisedBandwidthsExitsOnly = new ArrayList<>();
-        for (NetworkStatusEntry relay
-            : consensus.getStatusEntries().values()) {
-          if (!relay.getFlags().contains("Running")) {
-            continue;
-          }
-          String serverDescriptorDigest = relay.getDescriptor()
-              .toUpperCase();
-          if (!serverDescriptors.containsKey(serverDescriptorDigest)) {
-            continue;
-          }
-          long advertisedBandwidth = serverDescriptors.get(
-              serverDescriptorDigest);
-          advertisedBandwidthsAllRelays.add(advertisedBandwidth);
-          if (relay.getFlags().contains("Exit")
-              && !relay.getFlags().contains("BadExit")) {
-            advertisedBandwidthsExitsOnly.add(advertisedBandwidth);
-          }
+        String serverDescriptorDigest = relay.getDescriptor()
+            .toUpperCase();
+        if (!serverDescriptors.containsKey(serverDescriptorDigest)) {
+          continue;
+        }
+        long advertisedBandwidth = serverDescriptors.get(
+            serverDescriptorDigest);
+        advertisedBandwidthsAllRelays.add(advertisedBandwidth);
+        if (relay.getFlags().contains("Exit")
+            && !relay.getFlags().contains("BadExit")) {
+          advertisedBandwidthsExitsOnly.add(advertisedBandwidth);
         }
+      }
 
-        /* Write advertised bandwidths of n-th fastest relays/exits. */
-        Collections.sort(advertisedBandwidthsAllRelays,
-            Collections.reverseOrder());
-        Collections.sort(advertisedBandwidthsExitsOnly,
-            Collections.reverseOrder());
-        int[] fastestRelays = new int[] { 1, 2, 3, 5, 10, 20, 30, 50, 100,
-            200, 300, 500, 1000, 2000, 3000, 5000 };
-        for (int fastestRelay : fastestRelays) {
-          if (advertisedBandwidthsAllRelays.size() >= fastestRelay) {
-            bw.write(String.format("%s,,%d,,%d%n", validAfter,
-                fastestRelay,
-                advertisedBandwidthsAllRelays.get(fastestRelay - 1)));
-          }
+      /* Write advertised bandwidths of n-th fastest relays/exits. */
+      Collections.sort(advertisedBandwidthsAllRelays,
+          Collections.reverseOrder());
+      Collections.sort(advertisedBandwidthsExitsOnly,
+          Collections.reverseOrder());
+      int[] fastestRelays = new int[] { 1, 2, 3, 5, 10, 20, 30, 50, 100,
+          200, 300, 500, 1000, 2000, 3000, 5000 };
+      for (int fastestRelay : fastestRelays) {
+        if (advertisedBandwidthsAllRelays.size() >= fastestRelay) {
+          bw.write(String.format("%s,,%d,,%d%n", validAfter,
+              fastestRelay,
+              advertisedBandwidthsAllRelays.get(fastestRelay - 1)));
         }
-        for (int fastestRelay : fastestRelays) {
-          if (advertisedBandwidthsExitsOnly.size() >= fastestRelay) {
-            bw.write(String.format("%s,TRUE,%d,,%d%n", validAfter,
-                fastestRelay,
-                advertisedBandwidthsExitsOnly.get(fastestRelay - 1)));
-          }
+      }
+      for (int fastestRelay : fastestRelays) {
+        if (advertisedBandwidthsExitsOnly.size() >= fastestRelay) {
+          bw.write(String.format("%s,TRUE,%d,,%d%n", validAfter,
+              fastestRelay,
+              advertisedBandwidthsExitsOnly.get(fastestRelay - 1)));
         }
+      }
 
-        /* Write advertised bandwidth percentiles of relays/exits. */
-        Collections.sort(advertisedBandwidthsAllRelays);
-        Collections.sort(advertisedBandwidthsExitsOnly);
-        int[] percentiles = new int[] { 0, 1, 2, 3, 5, 9, 10, 20, 25, 30,
-            40, 50, 60, 70, 75, 80, 90, 91, 95, 97, 98, 99, 100 };
-        if (!advertisedBandwidthsAllRelays.isEmpty()) {
-          for (int percentile : percentiles) {
-            bw.write(String.format("%s,,,%d,%d%n", validAfter,
-                percentile, advertisedBandwidthsAllRelays.get(
-                ((advertisedBandwidthsAllRelays.size() - 1)
-                * percentile) / 100)));
-          }
+      /* Write advertised bandwidth percentiles of relays/exits. */
+      Collections.sort(advertisedBandwidthsAllRelays);
+      Collections.sort(advertisedBandwidthsExitsOnly);
+      int[] percentiles = new int[] { 0, 1, 2, 3, 5, 9, 10, 20, 25, 30,
+          40, 50, 60, 70, 75, 80, 90, 91, 95, 97, 98, 99, 100 };
+      if (!advertisedBandwidthsAllRelays.isEmpty()) {
+        for (int percentile : percentiles) {
+          bw.write(String.format("%s,,,%d,%d%n", validAfter,
+              percentile, advertisedBandwidthsAllRelays.get(
+              ((advertisedBandwidthsAllRelays.size() - 1)
+              * percentile) / 100)));
         }
-        if (!advertisedBandwidthsExitsOnly.isEmpty()) {
-          for (int percentile : percentiles) {
-            bw.write(String.format("%s,TRUE,,%d,%d%n", validAfter,
-                percentile, advertisedBandwidthsExitsOnly.get(
-                ((advertisedBandwidthsExitsOnly.size() - 1)
-                * percentile) / 100)));
-          }
+      }
+      if (!advertisedBandwidthsExitsOnly.isEmpty()) {
+        for (int percentile : percentiles) {
+          bw.write(String.format("%s,TRUE,,%d,%d%n", validAfter,
+              percentile, advertisedBandwidthsExitsOnly.get(
+              ((advertisedBandwidthsExitsOnly.size() - 1)
+              * percentile) / 100)));
         }
       }
     }
diff --git a/modules/clients/src/main/java/org/torproject/metrics/clients/Main.java b/modules/clients/src/main/java/org/torproject/metrics/clients/Main.java
index dc34d37..384c621 100644
--- a/modules/clients/src/main/java/org/torproject/metrics/clients/Main.java
+++ b/modules/clients/src/main/java/org/torproject/metrics/clients/Main.java
@@ -6,7 +6,6 @@ package org.torproject.metrics.clients;
 import org.torproject.descriptor.BandwidthHistory;
 import org.torproject.descriptor.BridgeNetworkStatus;
 import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
 import org.torproject.descriptor.DescriptorReader;
 import org.torproject.descriptor.DescriptorSourceFactory;
 import org.torproject.descriptor.ExtraInfoDescriptor;
@@ -19,7 +18,6 @@ import java.io.FileWriter;
 import java.io.IOException;
 import java.text.SimpleDateFormat;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.Map;
 import java.util.SortedMap;
 import java.util.TimeZone;
@@ -65,25 +63,16 @@ public class Main {
         DescriptorSourceFactory.createDescriptorReader();
     File historyFile = new File("status/relay-descriptors");
     descriptorReader.setHistoryFile(historyFile);
-    descriptorReader.addDirectory(new File(
-        "../../shared/in/recent/relay-descriptors/consensuses"));
-    descriptorReader.addDirectory(new File(
-        "../../shared/in/recent/relay-descriptors/extra-infos"));
-    descriptorReader.addDirectory(new File(
-        "../../shared/in/archive/relay-descriptors/consensuses"));
-    descriptorReader.addDirectory(new File(
-        "../../shared/in/archive/relay-descriptors/extra-infos"));
-    Iterator<DescriptorFile> descriptorFiles =
-        descriptorReader.readDescriptors();
-    while (descriptorFiles.hasNext()) {
-      DescriptorFile descriptorFile = descriptorFiles.next();
-      for (Descriptor descriptor : descriptorFile.getDescriptors()) {
-        if (descriptor instanceof ExtraInfoDescriptor) {
-          parseRelayExtraInfoDescriptor((ExtraInfoDescriptor) descriptor);
-        } else if (descriptor instanceof RelayNetworkStatusConsensus) {
-          parseRelayNetworkStatusConsensus(
-              (RelayNetworkStatusConsensus) descriptor);
-        }
+    for (Descriptor descriptor : descriptorReader.readDescriptors(
+        new File("../../shared/in/recent/relay-descriptors/consensuses"),
+        new File("../../shared/in/recent/relay-descriptors/extra-infos"),
+        new File("../../shared/in/archive/relay-descriptors/consensuses"),
+        new File("../../shared/in/archive/relay-descriptors/extra-infos"))) {
+      if (descriptor instanceof ExtraInfoDescriptor) {
+        parseRelayExtraInfoDescriptor((ExtraInfoDescriptor) descriptor);
+      } else if (descriptor instanceof RelayNetworkStatusConsensus) {
+        parseRelayNetworkStatusConsensus(
+            (RelayNetworkStatusConsensus) descriptor);
       }
     }
     descriptorReader.saveHistoryFile(historyFile);
@@ -209,21 +198,14 @@ public class Main {
         DescriptorSourceFactory.createDescriptorReader();
     File historyFile = new File("status/bridge-descriptors");
     descriptorReader.setHistoryFile(historyFile);
-    descriptorReader.addDirectory(new File(
-        "../../shared/in/recent/bridge-descriptors"));
-    descriptorReader.addDirectory(new File(
-        "../../shared/in/archive/bridge-descriptors"));
-    Iterator<DescriptorFile> descriptorFiles =
-        descriptorReader.readDescriptors();
-    while (descriptorFiles.hasNext()) {
-      DescriptorFile descriptorFile = descriptorFiles.next();
-      for (Descriptor descriptor : descriptorFile.getDescriptors()) {
-        if (descriptor instanceof ExtraInfoDescriptor) {
-          parseBridgeExtraInfoDescriptor(
-              (ExtraInfoDescriptor) descriptor);
-        } else if (descriptor instanceof BridgeNetworkStatus) {
-          parseBridgeNetworkStatus((BridgeNetworkStatus) descriptor);
-        }
+    for (Descriptor descriptor : descriptorReader.readDescriptors(
+        new File("../../shared/in/recent/bridge-descriptors"),
+        new File("../../shared/in/archive/bridge-descriptors"))) {
+      if (descriptor instanceof ExtraInfoDescriptor) {
+        parseBridgeExtraInfoDescriptor(
+            (ExtraInfoDescriptor) descriptor);
+      } else if (descriptor instanceof BridgeNetworkStatus) {
+        parseBridgeNetworkStatus((BridgeNetworkStatus) descriptor);
       }
     }
     descriptorReader.saveHistoryFile(historyFile);
diff --git a/modules/connbidirect/src/main/java/org/torproject/metrics/connbidirect/Main.java b/modules/connbidirect/src/main/java/org/torproject/metrics/connbidirect/Main.java
index 63dc6be..1de49d3 100644
--- a/modules/connbidirect/src/main/java/org/torproject/metrics/connbidirect/Main.java
+++ b/modules/connbidirect/src/main/java/org/torproject/metrics/connbidirect/Main.java
@@ -4,7 +4,6 @@
 package org.torproject.metrics.connbidirect;
 
 import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
 import org.torproject.descriptor.DescriptorReader;
 import org.torproject.descriptor.DescriptorSourceFactory;
 import org.torproject.descriptor.ExtraInfoDescriptor;
@@ -21,7 +20,6 @@ import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.SortedMap;
@@ -358,23 +356,16 @@ public class Main {
       SortedMap<String, Long> parseHistory) {
     DescriptorReader descriptorReader =
         DescriptorSourceFactory.createDescriptorReader();
-    for (File descriptorsDirectory : descriptorsDirectories) {
-      descriptorReader.addDirectory(descriptorsDirectory);
-    }
     descriptorReader.setExcludedFiles(parseHistory);
-    Iterator<DescriptorFile> descriptorFiles =
-        descriptorReader.readDescriptors();
-    while (descriptorFiles.hasNext()) {
-      DescriptorFile descriptorFile = descriptorFiles.next();
-      for (Descriptor descriptor : descriptorFile.getDescriptors()) {
-        if (!(descriptor instanceof ExtraInfoDescriptor)) {
-          continue;
-        }
-        RawStat rawStat = parseRawStatFromDescriptor(
-            (ExtraInfoDescriptor) descriptor);
-        if (rawStat != null) {
-          rawStats.add(rawStat);
-        }
+    for (Descriptor descriptor : descriptorReader.readDescriptors(
+        descriptorsDirectories)) {
+      if (!(descriptor instanceof ExtraInfoDescriptor)) {
+        continue;
+      }
+      RawStat rawStat = parseRawStatFromDescriptor(
+          (ExtraInfoDescriptor) descriptor);
+      if (rawStat != null) {
+        rawStats.add(rawStat);
       }
     }
     parseHistory.clear();
diff --git a/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Main.java b/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Main.java
index ad0b415..62a06b0 100644
--- a/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Main.java
+++ b/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Main.java
@@ -19,11 +19,9 @@ public class Main {
   public static void main(String[] args) {
 
     /* Initialize directories and file paths. */
-    Set<File> inDirectories = new HashSet<>();
-    inDirectories.add(
-        new File("../../shared/in/recent/relay-descriptors/consensuses"));
-    inDirectories.add(
-        new File("../../shared/in/recent/relay-descriptors/extra-infos"));
+    File[] inDirectories = new File[] {
+        new File("../../shared/in/recent/relay-descriptors/consensuses"),
+        new File("../../shared/in/recent/relay-descriptors/extra-infos") };
     File statusDirectory = new File("status");
 
     /* Initialize parser and read parse history to avoid parsing
diff --git a/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Parser.java b/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Parser.java
index eccb0c0..ebfe869 100644
--- a/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Parser.java
+++ b/modules/hidserv/src/main/java/org/torproject/metrics/hidserv/Parser.java
@@ -4,7 +4,6 @@
 package org.torproject.metrics.hidserv;
 
 import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
 import org.torproject.descriptor.DescriptorReader;
 import org.torproject.descriptor.DescriptorSourceFactory;
 import org.torproject.descriptor.ExtraInfoDescriptor;
@@ -21,7 +20,6 @@ import java.math.BigInteger;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
-import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
 import java.util.SortedMap;
@@ -42,6 +40,9 @@ public class Parser {
    * consensuses. */
   private DescriptorReader descriptorReader;
 
+  /** Input directories containing descriptors to parse. */
+  private File[] inDirectories;
+
   /** Document file containing previously parsed reported hidden-service
    * statistics. */
   private File reportedHidServStatsFile;
@@ -61,7 +62,7 @@ public class Parser {
 
   /** Initializes a new parser object using the given directories and
    * document stores. */
-  public Parser(Set<File> inDirectories, File statusDirectory,
+  public Parser(File[] inDirectories, File statusDirectory,
       DocumentStore<ReportedHidServStats> reportedHidServStatsStore,
       DocumentStore<ComputedNetworkFractions>
       computedNetworkFractionsStore) {
@@ -73,10 +74,8 @@ public class Parser {
      * rather small extra-info descriptors. */
     this.descriptorReader =
         DescriptorSourceFactory.createDescriptorReader();
-    for (File inDirectory : inDirectories) {
-      this.descriptorReader.addDirectory(inDirectory);
-    }
-    this.descriptorReader.setMaxDescriptorFilesInQueue(5);
+    this.inDirectories = inDirectories;
+    this.descriptorReader.setMaxDescriptorsInQueue(5);
 
     /* Create File instances for the files and directories in the provided
      * status directory. */
@@ -168,18 +167,14 @@ public class Parser {
    * handles the resulting parsed descriptors if they are either
    * extra-info descriptors or consensuses. */
   public boolean parseDescriptors() {
-    Iterator<DescriptorFile> descriptorFiles =
-        this.descriptorReader.readDescriptors();
-    while (descriptorFiles.hasNext()) {
-      DescriptorFile descriptorFile = descriptorFiles.next();
-      for (Descriptor descriptor : descriptorFile.getDescriptors()) {
-        if (descriptor instanceof ExtraInfoDescriptor) {
-          this.parseExtraInfoDescriptor((ExtraInfoDescriptor) descriptor);
-        } else if (descriptor instanceof RelayNetworkStatusConsensus) {
-          if (!this.parseRelayNetworkStatusConsensus(
-              (RelayNetworkStatusConsensus) descriptor)) {
-            return false;
-          }
+    for (Descriptor descriptor : descriptorReader.readDescriptors(
+        this.inDirectories)) {
+      if (descriptor instanceof ExtraInfoDescriptor) {
+        this.parseExtraInfoDescriptor((ExtraInfoDescriptor) descriptor);
+      } else if (descriptor instanceof RelayNetworkStatusConsensus) {
+        if (!this.parseRelayNetworkStatusConsensus(
+            (RelayNetworkStatusConsensus) descriptor)) {
+          return false;
         }
       }
     }
diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java b/modules/legacy/src/main/java/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java
index a04f95f..82fb70f 100644
--- a/modules/legacy/src/main/java/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java
+++ b/modules/legacy/src/main/java/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java
@@ -4,7 +4,6 @@
 package org.torproject.ernie.cron;
 
 import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
 import org.torproject.descriptor.DescriptorReader;
 import org.torproject.descriptor.DescriptorSourceFactory;
 import org.torproject.descriptor.ExtraInfoDescriptor;
@@ -836,33 +835,22 @@ public final class RelayDescriptorDatabaseImporter {
     if (!this.archivesDirectories.isEmpty()) {
       DescriptorReader reader =
           DescriptorSourceFactory.createDescriptorReader();
-      reader.setMaxDescriptorFilesInQueue(10);
-      for (String archivesPath : this.archivesDirectories) {
-        File archivesDirectory = new File(archivesPath);
-        if (archivesDirectory.exists()) {
-          reader.addDirectory(archivesDirectory);
-        }
-      }
+      reader.setMaxDescriptorsInQueue(10);
       File historyFile = new File(statsDirectory,
           "database-importer-relay-descriptor-history");
       if (keepImportHistory) {
         reader.setHistoryFile(historyFile);
       }
-      Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors();
-      while (descriptorFiles.hasNext()) {
-        DescriptorFile descriptorFile = descriptorFiles.next();
-        if (descriptorFile.getDescriptors() != null) {
-          for (Descriptor descriptor : descriptorFile.getDescriptors()) {
-            if (descriptor instanceof RelayNetworkStatusConsensus) {
-              this.addRelayNetworkStatusConsensus(
-                  (RelayNetworkStatusConsensus) descriptor);
-            } else if (descriptor instanceof ServerDescriptor) {
-              this.addServerDescriptor((ServerDescriptor) descriptor);
-            } else if (descriptor instanceof ExtraInfoDescriptor) {
-              this.addExtraInfoDescriptor(
-                  (ExtraInfoDescriptor) descriptor);
-            }
-          }
+      for (Descriptor descriptor : reader.readDescriptors(
+          this.archivesDirectories.toArray(
+          new File[this.archivesDirectories.size()]))) {
+        if (descriptor instanceof RelayNetworkStatusConsensus) {
+          this.addRelayNetworkStatusConsensus(
+              (RelayNetworkStatusConsensus) descriptor);
+        } else if (descriptor instanceof ServerDescriptor) {
+          this.addServerDescriptor((ServerDescriptor) descriptor);
+        } else if (descriptor instanceof ExtraInfoDescriptor) {
+          this.addExtraInfoDescriptor((ExtraInfoDescriptor) descriptor);
         }
       }
       if (keepImportHistory) {
diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java b/modules/legacy/src/main/java/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java
index 801794f..c06acc3 100644
--- a/modules/legacy/src/main/java/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java
+++ b/modules/legacy/src/main/java/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java
@@ -5,7 +5,6 @@ package org.torproject.ernie.cron.network;
 
 import org.torproject.descriptor.BridgeNetworkStatus;
 import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
 import org.torproject.descriptor.DescriptorReader;
 import org.torproject.descriptor.DescriptorSourceFactory;
 import org.torproject.descriptor.NetworkStatusEntry;
@@ -191,36 +190,29 @@ public class ConsensusStatsFileHandler {
       logger.fine("Importing files in directory " + bridgesDir + "/...");
       DescriptorReader reader =
           DescriptorSourceFactory.createDescriptorReader();
-      reader.addDirectory(bridgesDir);
       File historyFile = new File(statsDirectory,
           "consensus-stats-bridge-descriptor-history");
       if (keepImportHistory) {
         reader.setHistoryFile(historyFile);
       }
-      Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors();
-      while (descriptorFiles.hasNext()) {
-        DescriptorFile descriptorFile = descriptorFiles.next();
-        if (descriptorFile.getDescriptors() != null) {
+      for (Descriptor descriptor : reader.readDescriptors(bridgesDir)) {
+        if (descriptor instanceof BridgeNetworkStatus) {
+          String descriptorFileName = descriptor.getDescriptorFile().getName();
           String authority = null;
-          if (descriptorFile.getFileName().contains(
+          if (descriptorFileName.contains(
               "4A0CCD2DDC7995083D73F5D667100C8A5831F16D")) {
             authority = "Tonga";
-          } else if (descriptorFile.getFileName().contains(
+          } else if (descriptorFileName.contains(
               "1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1")) {
             authority = "Bifroest";
           }
-          for (Descriptor descriptor : descriptorFile.getDescriptors()) {
-            if (descriptor instanceof BridgeNetworkStatus) {
-              if (authority == null) {
-                this.logger.warning("Did not recognize the bridge authority "
-                    + "that generated " + descriptorFile.getFileName()
-                    + ".  Skipping.");
-                continue;
-              }
-              this.addBridgeNetworkStatus(
-                  (BridgeNetworkStatus) descriptor, authority);
-            }
+          if (authority == null) {
+            this.logger.warning("Did not recognize the bridge authority "
+                + "that generated " + descriptorFileName + ".  Skipping.");
+            continue;
           }
+          this.addBridgeNetworkStatus(
+              (BridgeNetworkStatus) descriptor, authority);
         }
       }
       if (keepImportHistory) {
diff --git a/modules/legacy/src/main/java/org/torproject/ernie/cron/performance/TorperfProcessor.java b/modules/legacy/src/main/java/org/torproject/ernie/cron/performance/TorperfProcessor.java
index 0c2be9d..b5dae12 100644
--- a/modules/legacy/src/main/java/org/torproject/ernie/cron/performance/TorperfProcessor.java
+++ b/modules/legacy/src/main/java/org/torproject/ernie/cron/performance/TorperfProcessor.java
@@ -4,10 +4,10 @@
 package org.torproject.ernie.cron.performance;
 
 import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
 import org.torproject.descriptor.DescriptorReader;
 import org.torproject.descriptor.DescriptorSourceFactory;
 import org.torproject.descriptor.TorperfResult;
+import org.torproject.descriptor.UnparseableDescriptor;
 
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
@@ -83,60 +83,55 @@ public class TorperfProcessor {
         logger.fine("Importing files in " + torperfDirectory + "/...");
         DescriptorReader descriptorReader =
             DescriptorSourceFactory.createDescriptorReader();
-        descriptorReader.addDirectory(torperfDirectory);
         File historyFile = new File(statsDirectory, "torperf-history");
         descriptorReader.setHistoryFile(historyFile);
-        Iterator<DescriptorFile> descriptorFiles =
-            descriptorReader.readDescriptors();
-        while (descriptorFiles.hasNext()) {
-          DescriptorFile descriptorFile = descriptorFiles.next();
-          if (descriptorFile.getException() != null) {
-            logger.log(Level.FINE, "Error parsing file.",
-                descriptorFile.getException());
+        for (Descriptor descriptor : descriptorReader.readDescriptors(
+            torperfDirectory)) {
+          if (descriptor instanceof UnparseableDescriptor) {
+            logger.log(Level.FINE, "Error parsing descriptor.",
+                ((UnparseableDescriptor) descriptor)
+                .getDescriptorParseException());
+            continue;
+          } else if (!(descriptor instanceof TorperfResult)) {
             continue;
           }
-          for (Descriptor descriptor : descriptorFile.getDescriptors()) {
-            if (!(descriptor instanceof TorperfResult)) {
-              continue;
-            }
-            TorperfResult result = (TorperfResult) descriptor;
-            if (null != result.getUnrecognizedKeys()
-                && result.getUnrecognizedKeys().containsKey("ENDPOINTREMOTE")
-                && result.getUnrecognizedKeys().get("ENDPOINTREMOTE")
-                .contains(".onion")) {
-              continue;
-            }
-            String source = result.getSource();
-            long fileSize = result.getFileSize();
-            if (fileSize == 51200) {
-              source += "-50kb";
-            } else if (fileSize == 1048576) {
-              source += "-1mb";
-            } else if (fileSize == 5242880) {
-              source += "-5mb";
-            } else {
-              logger.fine("Unexpected file size '" + fileSize
-                  + "'.  Skipping.");
-              continue;
-            }
-            String dateTime = formatter.format(result.getStartMillis());
-            long completeMillis = result.getDataCompleteMillis()
-                - result.getStartMillis();
-            String key = source + "," + dateTime;
-            String value = key;
-            if ((result.didTimeout() == null
-                && result.getDataCompleteMillis() < 1)
-                || (result.didTimeout() != null && result.didTimeout())) {
-              value += ",-2"; // -2 for timeout
-            } else if (result.getReadBytes() < fileSize) {
-              value += ",-1"; // -1 for failure
-            } else {
-              value += "," + completeMillis;
-            }
-            if (!rawObs.containsKey(key)) {
-              rawObs.put(key, value);
-              addedRawObs++;
-            }
+          TorperfResult result = (TorperfResult) descriptor;
+          if (null != result.getUnrecognizedKeys()
+              && result.getUnrecognizedKeys().containsKey("ENDPOINTREMOTE")
+              && result.getUnrecognizedKeys().get("ENDPOINTREMOTE")
+              .contains(".onion")) {
+            continue;
+          }
+          String source = result.getSource();
+          long fileSize = result.getFileSize();
+          if (fileSize == 51200) {
+            source += "-50kb";
+          } else if (fileSize == 1048576) {
+            source += "-1mb";
+          } else if (fileSize == 5242880) {
+            source += "-5mb";
+          } else {
+            logger.fine("Unexpected file size '" + fileSize
+                + "'.  Skipping.");
+            continue;
+          }
+          String dateTime = formatter.format(result.getStartMillis());
+          long completeMillis = result.getDataCompleteMillis()
+              - result.getStartMillis();
+          String key = source + "," + dateTime;
+          String value = key;
+          if ((result.didTimeout() == null
+              && result.getDataCompleteMillis() < 1)
+              || (result.didTimeout() != null && result.didTimeout())) {
+            value += ",-2"; // -2 for timeout
+          } else if (result.getReadBytes() < fileSize) {
+            value += ",-1"; // -1 for failure
+          } else {
+            value += "," + completeMillis;
+          }
+          if (!rawObs.containsKey(key)) {
+            rawObs.put(key, value);
+            addedRawObs++;
           }
         }
         descriptorReader.saveHistoryFile(historyFile);
diff --git a/modules/onionperf/src/main/java/org/torproject/metrics/onionperf/Main.java b/modules/onionperf/src/main/java/org/torproject/metrics/onionperf/Main.java
index 2af3100..802e4ef 100644
--- a/modules/onionperf/src/main/java/org/torproject/metrics/onionperf/Main.java
+++ b/modules/onionperf/src/main/java/org/torproject/metrics/onionperf/Main.java
@@ -1,7 +1,6 @@
 package org.torproject.metrics.onionperf;
 
 import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
 import org.torproject.descriptor.DescriptorReader;
 import org.torproject.descriptor.DescriptorSourceFactory;
 import org.torproject.descriptor.TorperfResult;
@@ -79,118 +78,114 @@ public class Main {
         + "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, "
         + "?, ?, ?, ?, ?, ?)", Statement.RETURN_GENERATED_KEYS);
 
-    DescriptorReader dr = DescriptorSourceFactory.createDescriptorReader();
-    dr.addDirectory(new File("../../shared/in/archive/torperf"));
-    dr.addDirectory(new File("../../shared/in/recent/torperf"));
-    Iterator<DescriptorFile> dfs = dr.readDescriptors();
     Calendar calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
-    while (dfs.hasNext()) {
-      DescriptorFile df = dfs.next();
-      for (Descriptor d : df.getDescriptors()) {
-        if (!(d instanceof TorperfResult)) {
-          continue;
+    DescriptorReader dr = DescriptorSourceFactory.createDescriptorReader();
+    for (Descriptor d : dr.readDescriptors(
+        new File("../../shared/in/archive/torperf"),
+        new File("../../shared/in/recent/torperf"))) {
+      if (!(d instanceof TorperfResult)) {
+        continue;
+      }
+      TorperfResult tr = (TorperfResult) d;
+      int measurementId = -1;
+      String truncatedSource = truncateString(tr.getSource(), 32);
+      psMeasurementsSelect.clearParameters();
+      psMeasurementsSelect.setString(1, truncatedSource);
+      psMeasurementsSelect.setInt(2, tr.getFileSize());
+      psMeasurementsSelect.setTimestamp(3,
+          new Timestamp(tr.getStartMillis()), calendar);
+      try (ResultSet rs = psMeasurementsSelect.executeQuery()) {
+        if (rs.next()) {
+          measurementId = rs.getInt(1);
         }
-        TorperfResult tr = (TorperfResult) d;
-        int measurementId = -1;
-        String truncatedSource = truncateString(tr.getSource(), 32);
-        psMeasurementsSelect.clearParameters();
-        psMeasurementsSelect.setString(1, truncatedSource);
-        psMeasurementsSelect.setInt(2, tr.getFileSize());
-        psMeasurementsSelect.setTimestamp(3,
+      }
+      if (measurementId < 0) {
+        psMeasurementsInsert.clearParameters();
+        psMeasurementsInsert.setString(1, truncatedSource);
+        psMeasurementsInsert.setInt(2, tr.getFileSize());
+        psMeasurementsInsert.setTimestamp(3,
             new Timestamp(tr.getStartMillis()), calendar);
-        try (ResultSet rs = psMeasurementsSelect.executeQuery()) {
-          if (rs.next()) {
-            measurementId = rs.getInt(1);
-          }
-        }
-        if (measurementId < 0) {
-          psMeasurementsInsert.clearParameters();
-          psMeasurementsInsert.setString(1, truncatedSource);
-          psMeasurementsInsert.setInt(2, tr.getFileSize());
-          psMeasurementsInsert.setTimestamp(3,
-              new Timestamp(tr.getStartMillis()), calendar);
-          long[] timestamps = new long[] { tr.getSocketMillis(),
-              tr.getConnectMillis(), tr.getNegotiateMillis(),
-              tr.getRequestMillis(), tr.getResponseMillis(),
-              tr.getDataRequestMillis(), tr.getDataResponseMillis(),
-              tr.getDataCompleteMillis() };
-          for (int i = 4, j = 0; j < timestamps.length; i++, j++) {
-            if (timestamps[j] == 0L) {
-              psMeasurementsInsert.setNull(i, Types.INTEGER);
-            } else {
-              psMeasurementsInsert.setInt(i,
-                  (int) (timestamps[j] - tr.getStartMillis()));
-            }
-          }
-          psMeasurementsInsert.setInt(12, tr.getWriteBytes());
-          psMeasurementsInsert.setInt(13, tr.getReadBytes());
-          if (null == tr.didTimeout()) {
-            psMeasurementsInsert.setNull(14, Types.BOOLEAN);
-          } else {
-            psMeasurementsInsert.setBoolean(14, tr.didTimeout());
-          }
-          for (int i = 15, p = 0; i <= 25 && p <= 100; i++, p += 10) {
-            if (null == tr.getDataPercentiles()
-                || !tr.getDataPercentiles().containsKey(p)) {
-              psMeasurementsInsert.setNull(i, Types.INTEGER);
-            } else {
-              psMeasurementsInsert.setInt(i,
-                  (int) (tr.getDataPercentiles().get(p) - tr.getStartMillis()));
-            }
-          }
-          if (tr.getLaunchMillis() < 0L) {
-            psMeasurementsInsert.setNull(26, Types.TIMESTAMP);
-          } else {
-            psMeasurementsInsert.setTimestamp(26,
-                new Timestamp(tr.getLaunchMillis()), calendar);
-          }
-          if (tr.getUsedAtMillis() < 0L) {
-            psMeasurementsInsert.setNull(27, Types.TIMESTAMP);
-          } else {
-            psMeasurementsInsert.setTimestamp(27,
-                new Timestamp(tr.getUsedAtMillis()), calendar);
-          }
-          if (tr.getTimeout() < 0L) {
-            psMeasurementsInsert.setNull(28, Types.INTEGER);
+        long[] timestamps = new long[] { tr.getSocketMillis(),
+            tr.getConnectMillis(), tr.getNegotiateMillis(),
+            tr.getRequestMillis(), tr.getResponseMillis(),
+            tr.getDataRequestMillis(), tr.getDataResponseMillis(),
+            tr.getDataCompleteMillis() };
+        for (int i = 4, j = 0; j < timestamps.length; i++, j++) {
+          if (timestamps[j] == 0L) {
+            psMeasurementsInsert.setNull(i, Types.INTEGER);
           } else {
-            psMeasurementsInsert.setInt(28, (int) tr.getTimeout());
+            psMeasurementsInsert.setInt(i,
+                (int) (timestamps[j] - tr.getStartMillis()));
           }
-          if (tr.getQuantile() < 0.0) {
-            psMeasurementsInsert.setNull(29, Types.REAL);
-          } else {
-            psMeasurementsInsert.setDouble(29, tr.getQuantile());
-          }
-          if (tr.getCircId() < 0L) {
-            psMeasurementsInsert.setNull(30, Types.INTEGER);
+        }
+        psMeasurementsInsert.setInt(12, tr.getWriteBytes());
+        psMeasurementsInsert.setInt(13, tr.getReadBytes());
+        if (null == tr.didTimeout()) {
+          psMeasurementsInsert.setNull(14, Types.BOOLEAN);
+        } else {
+          psMeasurementsInsert.setBoolean(14, tr.didTimeout());
+        }
+        for (int i = 15, p = 0; i <= 25 && p <= 100; i++, p += 10) {
+          if (null == tr.getDataPercentiles()
+              || !tr.getDataPercentiles().containsKey(p)) {
+            psMeasurementsInsert.setNull(i, Types.INTEGER);
           } else {
-            psMeasurementsInsert.setInt(30, tr.getCircId());
+            psMeasurementsInsert.setInt(i,
+                (int) (tr.getDataPercentiles().get(p) - tr.getStartMillis()));
           }
-          if (tr.getUsedBy() < 0L) {
-            psMeasurementsInsert.setNull(31, Types.INTEGER);
+        }
+        if (tr.getLaunchMillis() < 0L) {
+          psMeasurementsInsert.setNull(26, Types.TIMESTAMP);
+        } else {
+          psMeasurementsInsert.setTimestamp(26,
+              new Timestamp(tr.getLaunchMillis()), calendar);
+        }
+        if (tr.getUsedAtMillis() < 0L) {
+          psMeasurementsInsert.setNull(27, Types.TIMESTAMP);
+        } else {
+          psMeasurementsInsert.setTimestamp(27,
+              new Timestamp(tr.getUsedAtMillis()), calendar);
+        }
+        if (tr.getTimeout() < 0L) {
+          psMeasurementsInsert.setNull(28, Types.INTEGER);
+        } else {
+          psMeasurementsInsert.setInt(28, (int) tr.getTimeout());
+        }
+        if (tr.getQuantile() < 0.0) {
+          psMeasurementsInsert.setNull(29, Types.REAL);
+        } else {
+          psMeasurementsInsert.setDouble(29, tr.getQuantile());
+        }
+        if (tr.getCircId() < 0L) {
+          psMeasurementsInsert.setNull(30, Types.INTEGER);
+        } else {
+          psMeasurementsInsert.setInt(30, tr.getCircId());
+        }
+        if (tr.getUsedBy() < 0L) {
+          psMeasurementsInsert.setNull(31, Types.INTEGER);
+        } else {
+          psMeasurementsInsert.setInt(31, tr.getUsedBy());
+        }
+        String[] unrecognizedKeys = new String[] { "ENDPOINTLOCAL",
+            "ENDPOINTPROXY", "ENDPOINTREMOTE", "HOSTNAMELOCAL",
+            "HOSTNAMEREMOTE", "SOURCEADDRESS" };
+        for (int i = 32, j = 0; j < unrecognizedKeys.length; i++, j++) {
+          if (null == tr.getUnrecognizedKeys()
+              || !tr.getUnrecognizedKeys().containsKey(unrecognizedKeys[j])) {
+            psMeasurementsInsert.setNull(i, Types.VARCHAR);
           } else {
-            psMeasurementsInsert.setInt(31, tr.getUsedBy());
-          }
-          String[] unrecognizedKeys = new String[] { "ENDPOINTLOCAL",
-              "ENDPOINTPROXY", "ENDPOINTREMOTE", "HOSTNAMELOCAL",
-              "HOSTNAMEREMOTE", "SOURCEADDRESS" };
-          for (int i = 32, j = 0; j < unrecognizedKeys.length; i++, j++) {
-            if (null == tr.getUnrecognizedKeys()
-                || !tr.getUnrecognizedKeys().containsKey(unrecognizedKeys[j])) {
-              psMeasurementsInsert.setNull(i, Types.VARCHAR);
-            } else {
-              psMeasurementsInsert.setString(i, truncateString(
-                  tr.getUnrecognizedKeys().get(unrecognizedKeys[j]), 64));
-            }
+            psMeasurementsInsert.setString(i, truncateString(
+                tr.getUnrecognizedKeys().get(unrecognizedKeys[j]), 64));
           }
-          psMeasurementsInsert.execute();
-          try (ResultSet rs = psMeasurementsInsert.getGeneratedKeys()) {
-            if (rs.next()) {
-              measurementId = rs.getInt(1);
-            }
+        }
+        psMeasurementsInsert.execute();
+        try (ResultSet rs = psMeasurementsInsert.getGeneratedKeys()) {
+          if (rs.next()) {
+            measurementId = rs.getInt(1);
           }
         }
-        /* Could use measurementId to insert path. */
       }
+      /* Could use measurementId to insert path. */
       connection.commit();
     }
   }
diff --git a/shared/build-base.xml b/shared/build-base.xml
index f473b77..c446aa4 100644
--- a/shared/build-base.xml
+++ b/shared/build-base.xml
@@ -8,7 +8,7 @@
   <property name="classes" value="${generated}/classes/"/>
   <property name="testclasses" value="${generated}/test-classes/"/>
   <property name="source-and-target-java-version" value="1.7" />
-  <property name="descriptorversion" value="1.8.2" />
+  <property name="descriptorversion" value="1.9.0" />
 
   <path id="base.classpath">
     <pathelement path="${classes}"/>



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits