[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [metrics-web/master] Process bridge network statuses from two authorities.



commit d1de2a89cd96c0f9dc6615042ac47c984fbbd590
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Thu Sep 1 15:56:56 2016 +0200

    Process bridge network statuses from two authorities.
    
    Fixes #20049.
---
 modules/legacy/build.xml                           |   2 +-
 .../cron/network/ConsensusStatsFileHandler.java    | 160 ++++++++++++---------
 2 files changed, 96 insertions(+), 66 deletions(-)

diff --git a/modules/legacy/build.xml b/modules/legacy/build.xml
index 6cacf50..1e4049b 100644
--- a/modules/legacy/build.xml
+++ b/modules/legacy/build.xml
@@ -12,7 +12,7 @@
       <include name="commons-compress-1.9.jar"/>
       <include name="postgresql-jdbc3-9.2.jar"/>
       <include name="commons-lang-2.6.jar"/>
-      <include name="descriptor-1.2.0.jar"/>
+      <include name="descriptor-1.4.0.jar"/>
     </fileset>
   </path>
 
diff --git a/modules/legacy/src/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java b/modules/legacy/src/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java
index 6222859..824e1eb 100644
--- a/modules/legacy/src/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java
+++ b/modules/legacy/src/org/torproject/ernie/cron/network/ConsensusStatsFileHandler.java
@@ -51,15 +51,16 @@ public class ConsensusStatsFileHandler {
   private File bridgeConsensusStatsRawFile;
 
   /**
-   * Number of running bridges in a given bridge status. Map keys are
-   * bridge status times formatted as "yyyy-MM-dd HH:mm:ss", map values
-   * are lines as read from <code>stats/bridge-consensus-stats-raw</code>.
+   * Number of running bridges in a given bridge status. Map keys are the bridge
+   * status time formatted as "yyyy-MM-dd HH:mm:ss", a comma, and the bridge
+   * authority nickname, map values are lines as read from
+   * <code>stats/bridge-consensus-stats-raw</code>.
    */
   private SortedMap<String, String> bridgesRaw;
 
   /**
    * Average number of running bridges per day. Map keys are dates
-   * formatted as "yyyy-MM-dd", map values are the last column as written
+   * formatted as "yyyy-MM-dd", map values are the remaining columns as written
    * to <code>stats/consensus-stats</code>.
    */
   private SortedMap<String, String> bridgesPerDay;
@@ -132,17 +133,26 @@ public class ConsensusStatsFileHandler {
             continue;
           }
           String[] parts = line.split(",");
-          String dateTime = parts[0];
-          if (parts.length == 2) {
-            this.bridgesRaw.put(dateTime, line + ",0");
-          } else if (parts.length == 3) {
-            this.bridgesRaw.put(dateTime, line);
-          } else {
+          if (parts.length < 2 || parts.length > 4) {
             this.logger.warning("Corrupt line '" + line + "' in file "
                 + this.bridgeConsensusStatsRawFile.getAbsolutePath()
                 + "! Aborting to read this file!");
             break;
           }
+          String key = parts[0] + "," + (parts.length < 4 ? "Tonga" : parts[1]);
+          String value = null;
+          if (parts.length == 2) {
+            value = key + "," + parts[1] + ",0";
+          } else if (parts.length == 3) {
+            value = key + "," + parts[1] + "," + parts[2];
+          } else if (parts.length == 4) {
+            value = key + "," + parts[2] + "," + parts[3];
+          } else {
+            /* Impossible, we already checked the range above. */
+          }
+          /* Assume that all lines without authority nickname are based on
+           * Tonga's network status, not Bifroest's. */
+          this.bridgesRaw.put(key, value);
         }
         br.close();
         this.logger.fine("Finished reading file "
@@ -159,20 +169,21 @@ public class ConsensusStatsFileHandler {
    * Adds the intermediate results of the number of running bridges in a
    * given bridge status to the existing observations.
    */
-  public void addBridgeConsensusResults(long publishedMillis, int running,
-      int runningEc2Bridges) {
-    String published = dateTimeFormat.format(publishedMillis);
-    String line = published + "," + running + "," + runningEc2Bridges;
-    if (!this.bridgesRaw.containsKey(published)) {
+  public void addBridgeConsensusResults(long publishedMillis,
+      String authorityNickname, int running, int runningEc2Bridges) {
+    String publishedAuthority = dateTimeFormat.format(publishedMillis) + ","
+        + authorityNickname;
+    String line = publishedAuthority + "," + running + "," + runningEc2Bridges;
+    if (!this.bridgesRaw.containsKey(publishedAuthority)) {
       this.logger.finer("Adding new bridge numbers: " + line);
-      this.bridgesRaw.put(published, line);
+      this.bridgesRaw.put(publishedAuthority, line);
       this.bridgeResultsAdded++;
-    } else if (!line.equals(this.bridgesRaw.get(published))) {
+    } else if (!line.equals(this.bridgesRaw.get(publishedAuthority))) {
       this.logger.warning("The numbers of running bridges we were just "
           + "given (" + line + ") are different from what we learned "
-          + "before (" + this.bridgesRaw.get(published) + ")! "
+          + "before (" + this.bridgesRaw.get(publishedAuthority) + ")! "
           + "Overwriting!");
-      this.bridgesRaw.put(published, line);
+      this.bridgesRaw.put(publishedAuthority, line);
     }
   }
 
@@ -191,10 +202,24 @@ public class ConsensusStatsFileHandler {
       while (descriptorFiles.hasNext()) {
         DescriptorFile descriptorFile = descriptorFiles.next();
         if (descriptorFile.getDescriptors() != null) {
+          String authority = null;
+          if (descriptorFile.getFileName().contains(
+              "4A0CCD2DDC7995083D73F5D667100C8A5831F16D")) {
+            authority = "Tonga";
+          } else if (descriptorFile.getFileName().contains(
+              "1D8F3A91C37C5D1C4C19B1AD1D0CFBE8BF72D8E1")) {
+            authority = "Bifroest";
+          }
           for (Descriptor descriptor : descriptorFile.getDescriptors()) {
             if (descriptor instanceof BridgeNetworkStatus) {
+              if (authority == null) {
+                this.logger.warning("Did not recognize the bridge authority "
+                    + "that generated " + descriptorFile.getFileName()
+                    + ".  Skipping.");
+                continue;
+              }
               this.addBridgeNetworkStatus(
-                  (BridgeNetworkStatus) descriptor);
+                  (BridgeNetworkStatus) descriptor, authority);
             }
           }
         }
@@ -203,7 +228,8 @@ public class ConsensusStatsFileHandler {
     }
   }
 
-  private void addBridgeNetworkStatus(BridgeNetworkStatus status) {
+  private void addBridgeNetworkStatus(BridgeNetworkStatus status,
+      String authority) {
     int runningBridges = 0;
     int runningEc2Bridges = 0;
     for (NetworkStatusEntry statusEntry
@@ -215,7 +241,7 @@ public class ConsensusStatsFileHandler {
         }
       }
     }
-    this.addBridgeConsensusResults(status.getPublishedMillis(),
+    this.addBridgeConsensusResults(status.getPublishedMillis(), authority,
         runningBridges, runningEc2Bridges);
   }
 
@@ -225,49 +251,51 @@ public class ConsensusStatsFileHandler {
    */
   public void writeFiles() {
 
-    /* Go through raw observations of numbers of running bridges in bridge
-     * statuses, calculate averages per day, and add these averages to
-     * final results. */
-    if (!this.bridgesRaw.isEmpty()) {
-      String tempDate = null;
+    /* Go through raw observations and put everything into nested maps by day
+     * and bridge authority. */
+    Map<String, Map<String, int[]>> bridgesPerDayAndAuthority =
+        new HashMap<String, Map<String, int[]>>();
+    for (String bridgesRawLine : this.bridgesRaw.values()) {
+      String date = bridgesRawLine.substring(0, 10);
+      if (!bridgesPerDayAndAuthority.containsKey(date)) {
+        bridgesPerDayAndAuthority.put(date, new TreeMap<String, int[]>());
+      }
+      String[] parts = bridgesRawLine.split(",");
+      String authority = parts[1];
+      if (!bridgesPerDayAndAuthority.get(date).containsKey(authority)) {
+        bridgesPerDayAndAuthority.get(date).put(authority, new int[3]);
+      }
+      int[] bridges = bridgesPerDayAndAuthority.get(date).get(authority);
+      bridges[0] += Integer.parseInt(parts[2]);
+      bridges[1] += Integer.parseInt(parts[3]);
+      bridges[2]++;
+    }
+
+    /* Sum up average numbers of running bridges per day reported by all bridge
+     * authorities and add these averages to final results. */
+    for (Map.Entry<String, Map<String, int[]>> perDay
+        : bridgesPerDayAndAuthority.entrySet()) {
+      String date = perDay.getKey();
       int brunning = 0;
       int brunningEc2 = 0;
-      int statuses = 0;
-      Iterator<String> it = this.bridgesRaw.values().iterator();
-      boolean haveWrittenFinalLine = false;
-      while (it.hasNext() || !haveWrittenFinalLine) {
-        String next = it.hasNext() ? it.next() : null;
-        /* Finished reading a day or even all lines? */
-        if (tempDate != null && (next == null
-            || !next.substring(0, 10).equals(tempDate))) {
-          /* Only write results if we have seen at least half of all
-           * statuses. */
-          if (statuses >= 24) {
-            String line = "," + (brunning / statuses) + ","
-                + (brunningEc2 / statuses);
-            /* Are our results new? */
-            if (!this.bridgesPerDay.containsKey(tempDate)) {
-              this.logger.finer("Adding new average bridge numbers: "
-                  + tempDate + line);
-              this.bridgesPerDay.put(tempDate, line);
-            } else if (!line.equals(this.bridgesPerDay.get(tempDate))) {
-              this.logger.finer("Replacing existing average bridge "
-                  + "numbers (" + this.bridgesPerDay.get(tempDate)
-                  + " with new numbers: " + line);
-              this.bridgesPerDay.put(tempDate, line);
-            }
-          }
-          brunning = brunningEc2 = statuses = 0;
-          haveWrittenFinalLine = (next == null);
-        }
-        /* Sum up number of running bridges. */
-        if (next != null) {
-          tempDate = next.substring(0, 10);
-          statuses++;
-          String[] parts = next.split(",");
-          brunning += Integer.parseInt(parts[1]);
-          brunningEc2 += Integer.parseInt(parts[2]);
+      for (int[] perAuthority : perDay.getValue().values()) {
+        int statuses = perAuthority[2];
+        if (statuses < 12) {
+          /* Only write results if we have seen at least a dozen statuses. */
+          continue;
         }
+        brunning += perAuthority[0] / statuses;
+        brunningEc2 += perAuthority[1] / statuses;
+      }
+      String line = "," + brunning + "," + brunningEc2;
+      /* Are our results new? */
+      if (!this.bridgesPerDay.containsKey(date)) {
+        this.logger.finer("Adding new average bridge numbers: " + date + line);
+        this.bridgesPerDay.put(date, line);
+      } else if (!line.equals(this.bridgesPerDay.get(date))) {
+        this.logger.finer("Replacing existing average bridge numbers ("
+            + this.bridgesPerDay.get(date) + " with new numbers: " + line);
+        this.bridgesPerDay.put(date, line);
       }
     }
 
@@ -278,9 +306,11 @@ public class ConsensusStatsFileHandler {
       this.bridgeConsensusStatsRawFile.getParentFile().mkdirs();
       BufferedWriter bw = new BufferedWriter(
           new FileWriter(this.bridgeConsensusStatsRawFile));
-      bw.append("datetime,brunning,brunningec2\n");
+      bw.append("datetime,authority,brunning,brunningec2");
+      bw.newLine();
       for (String line : this.bridgesRaw.values()) {
-        bw.append(line + "\n");
+        bw.append(line);
+        bw.newLine();
       }
       bw.close();
       this.logger.fine("Finished writing file "
@@ -376,7 +406,7 @@ public class ConsensusStatsFileHandler {
               + "old: " + this.bridgesRaw.lastKey());
         }
       } catch (ParseException e) {
-         /* Can't parse the timestamp? Whatever. */
+        logger.warning("Can't parse the timestamp? Reason: " + e);
       }
     }
     logger.info(dumpStats.toString());

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits