[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[or-cvs] [metrics-db/master 2/3] Add 'all' column to bridge and dirreq stats.



Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Mon, 16 Aug 2010 09:20:36 +0200
Subject: Add 'all' column to bridge and dirreq stats.
Commit: b7151fa567ad61ad497822a7ff5f7ab666fc5fa4

---
 .../ernie/db/BridgeDescriptorParser.java           |   18 ++++++++++-----
 .../ernie/db/BridgeStatsFileHandler.java           |   22 +++++++++++++++----
 .../ernie/db/DirreqStatsFileHandler.java           |   20 ++++++++++++++---
 src/org/torproject/ernie/db/Main.java              |    1 +
 .../torproject/ernie/db/RelayDescriptorParser.java |    5 ++++
 5 files changed, 51 insertions(+), 15 deletions(-)

diff --git a/src/org/torproject/ernie/db/BridgeDescriptorParser.java b/src/org/torproject/ernie/db/BridgeDescriptorParser.java
index 18b3d51..b4b0888 100644
--- a/src/org/torproject/ernie/db/BridgeDescriptorParser.java
+++ b/src/org/torproject/ernie/db/BridgeDescriptorParser.java
@@ -103,16 +103,20 @@ public class BridgeDescriptorParser {
           long started = timeFormat.parse(geoipStartTimeLine.
               substring("geoip-start-time ".length())).getTime();
           long seconds = (published - started) / 1000L;
+          double allUsers = 0.0D;
           Map<String, String> obs = new HashMap<String, String>();
           String[] parts = line.split(" ")[1].split(",");
           for (String p : parts) {
-            for (String c : countries) {
+            double users = ((double) Long.parseLong(p.substring(3)) - 4L)
+                    * 86400.0D / ((double) seconds);
+            allUsers += users;
+            for (String c : this.countries) {
               if (p.startsWith(c)) {
-                obs.put(c, String.format("%.2f",
-                    ((double) Long.parseLong(p.substring(3)) - 4L)
-                    * 86400.0D / ((double) seconds)));
+                obs.put(c, String.format("%.2f", users));
+                break;
               }
             }
+            obs.put("zy", String.format("%.2f", allUsers));
           }
           String date = publishedLine.split(" ")[1];
           String time = publishedLine.split(" ")[2];
@@ -130,13 +134,15 @@ public class BridgeDescriptorParser {
                 + " bridge descriptor.");
             break;
           }
+          double allUsers = 0.0D;
           Map<String, String> obs = new HashMap<String, String>();
           String[] parts = line.split(" ")[1].split(",");
           for (String p : parts) {
+            double users = (double) Long.parseLong(p.substring(3)) - 4L;
             for (String c : countries) {
               if (p.startsWith(c)) {
-                obs.put(c, String.format("%.2f",
-                    (double) Long.parseLong(p.substring(3)) - 4L));
+                obs.put(c, String.format("%.2f", users));
+                break;
               }
             }
           }
diff --git a/src/org/torproject/ernie/db/BridgeStatsFileHandler.java b/src/org/torproject/ernie/db/BridgeStatsFileHandler.java
index 60da4ad..8c96a4f 100644
--- a/src/org/torproject/ernie/db/BridgeStatsFileHandler.java
+++ b/src/org/torproject/ernie/db/BridgeStatsFileHandler.java
@@ -122,7 +122,11 @@ public class BridgeStatsFileHandler {
           } else {
             String[] headers = line.split(",");
             for (int i = 3; i < headers.length; i++) {
-              this.countries.add(headers[i]);
+              if (headers[i].equals("all")) {
+                this.countries.add("zy");
+              } else {
+                this.countries.add(headers[i]);
+              }
             }
             /* Read in the rest of the file. */
             while ((line = br.readLine()) != null) {
@@ -251,7 +255,7 @@ public class BridgeStatsFileHandler {
       Map<String, String> obs) {
     String key = hashedIdentity + "," + date;
     StringBuilder sb = new StringBuilder(key + "," + time);
-    for (String c : countries) {
+    for (String c : this.countries) {
       sb.append("," + (obs.containsKey(c) && !obs.get(c).startsWith("-")
           ? obs.get(c) : "0.0"));
     }
@@ -323,7 +327,11 @@ public class BridgeStatsFileHandler {
           this.bridgeStatsRawFile));
       bw.append("bridge,date,time");
       for (String c : this.countries) {
-        bw.append("," + c);
+        if (c.equals("zy")) {
+          bw.append(",all");
+        } else {
+          bw.append("," + c);
+        }
       }
       bw.append("\n");
       for (String line : this.bridgeUsersRaw.values()) {
@@ -358,7 +366,7 @@ public class BridgeStatsFileHandler {
           + date + "," + time)) {
         double[] users = bridgeUsersPerDay.get(date);
         if (users == null) {
-          users = new double[countries.size()];
+          users = new double[this.countries.size()];
           bridgeUsersPerDay.put(date, users);
         }
         for (int i = 3; i < parts.length; i++) {
@@ -377,7 +385,11 @@ public class BridgeStatsFileHandler {
           this.bridgeStatsFile));
       bw.append("date");
       for (String c : this.countries) {
-        bw.append("," + c);
+        if (c.equals("zy")) {
+          bw.append(",all");
+        } else {
+          bw.append("," + c);
+        }
       }
       bw.append("\n");
 
diff --git a/src/org/torproject/ernie/db/DirreqStatsFileHandler.java b/src/org/torproject/ernie/db/DirreqStatsFileHandler.java
index 6d71203..3153fd9 100644
--- a/src/org/torproject/ernie/db/DirreqStatsFileHandler.java
+++ b/src/org/torproject/ernie/db/DirreqStatsFileHandler.java
@@ -86,7 +86,11 @@ public class DirreqStatsFileHandler {
           } else {
             String[] headers = line.split(",");
             for (int i = 2; i < headers.length - 1; i++) {
-              this.countries.add(headers[i]);
+              if (headers[i].equals("all")) {
+                this.countries.add("zy");
+              } else {
+                this.countries.add(headers[i]);
+              }
             }
             /* Read in the rest of the file. */
             while ((line = br.readLine()) != null) {
@@ -106,7 +110,11 @@ public class DirreqStatsFileHandler {
               if (!parts[parts.length - 1].equals("NA")) {
                 Map<String, String> obs = new HashMap<String, String>();
                 for (int i = 2; i < parts.length - 1; i++) {
-                  obs.put(headers[i], parts[i]);
+                  if (headers[i].equals("all")) {
+                    obs.put("zy", parts[i]);
+                  } else {
+                    obs.put(headers[i], parts[i]);
+                  }
                 }
                 String share = parts[parts.length - 1];
                 this.addObs(directory, date, obs, share);
@@ -153,7 +161,7 @@ public class DirreqStatsFileHandler {
           + "before (" + this.dirreqs.get(key) + "! Overwriting!");
       this.dirreqs.put(key, value);
       this.dirreqsModified = true;
-    }   
+    }
   }
 
   /**
@@ -173,7 +181,11 @@ public class DirreqStatsFileHandler {
         /* Write header. */
         bw.append("directory,date");
         for (String country : this.countries) {
-          bw.append("," + country);
+          if (country.equals("zy")) {
+            bw.append(",all");
+          } else {
+            bw.append("," + country);
+          }
         }
         bw.append(",share\n");
         /* Memorize last written date and directory to fill missing dates
diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java
index 7fcf1fd..5cf8d91 100644
--- a/src/org/torproject/ernie/db/Main.java
+++ b/src/org/torproject/ernie/db/Main.java
@@ -31,6 +31,7 @@ public class Main {
 
     // Define which stats we are interested in
     SortedSet<String> countries = config.getDirreqBridgeCountries();
+    countries.add("zy"); // Add country zy for 'all users'
     SortedSet<String> directories = config.getDirreqDirectories();
 
     // Prepare stats file handlers (only if we are writing stats)
diff --git a/src/org/torproject/ernie/db/RelayDescriptorParser.java b/src/org/torproject/ernie/db/RelayDescriptorParser.java
index 810bcde..5582cfb 100644
--- a/src/org/torproject/ernie/db/RelayDescriptorParser.java
+++ b/src/org/torproject/ernie/db/RelayDescriptorParser.java
@@ -316,15 +316,20 @@ public class RelayDescriptorParser {
             v3Reqs = line.split(" ")[1];
           } else if (line.startsWith("dirreq-v3-share ")
               && v3Reqs != null && !skip) {
+            int allUsers = 0;
             Map<String, String> obs = new HashMap<String, String>();
             String[] parts = v3Reqs.split(",");
             for (String p : parts) {
+              allUsers += Integer.parseInt(p.substring(3)) - 4;
               for (String c : this.countries) {
                 if (p.startsWith(c)) {
+                  // TODO in theory, we should substract 4 here, too
                   obs.put(c, p.substring(3));
+                  break;
                 }
               }
             }
+            obs.put("zy", "" + allUsers);
             String share = line.substring("dirreq-v3-share ".length(),
                 line.length() - 1);
             if (this.dsfh != null &&
-- 
1.7.1