[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[or-cvs] [metrics/master] Add parsing script for exit port stats.



Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Sat, 25 Jul 2009 20:37:26 +0200
Subject: Add parsing script for exit port stats.
Commit: 510a58a8a0bc1ec311e5596d8ad578bf05df0555

---
 HOWTO                                              |   26 +++
 .../metrics/exit/EvaluateExitPortStats.java        |  190 ++++++++++++++++++++
 2 files changed, 216 insertions(+), 0 deletions(-)
 create mode 100644 src/org/torproject/metrics/exit/EvaluateExitPortStats.java

diff --git a/HOWTO b/HOWTO
index 99d9656..f7fd123 100644
--- a/HOWTO
+++ b/HOWTO
@@ -304,3 +304,29 @@ Run the parsing script:
 $ java -cp bin/:lib/* org.torproject.metrics.entry.ParseEntryStats
   data/entrystats/ out/entrystats/
 
+
+6 Exit port statistics
+======================
+
+Put the exit-stats file coming from one exit node containing one or more
+days of measurements with possibly different exit policies in a directory
+data/exit/exit-stats . Put the network status consensuses covering the
+whole measurement interval in a directory called data/exit/consensuses/ . 
+
+Compile the evluation application:
+
+$ javac -d bin/ -cp src/:lib/* src/org/torproject/metrics/exit/*.java
+
+Run the evaluation application:
+
+$ java -cp bin/:lib/* org.torproject.metrics.exit.EvaluateExitPortStats
+  data/exit/exit-stats data/exit/consensuses/
+  cGyhQdOhBYtSUJtKG2sovzWuWg8,h7m2jUhDHih5WvZe6nGEgDzO1xU out/exit/
+
+The four parameters are:
+
+1. the exit-stats file,
+2. the directory containing consensuses
+3. a comma-separated list of identities of the measuring relay, and
+4. the output directory.
+
diff --git a/src/org/torproject/metrics/exit/EvaluateExitPortStats.java b/src/org/torproject/metrics/exit/EvaluateExitPortStats.java
new file mode 100644
index 0000000..6f968c3
--- /dev/null
+++ b/src/org/torproject/metrics/exit/EvaluateExitPortStats.java
@@ -0,0 +1,190 @@
+/* Copyright 2009 Karsten Loesing
+ * See LICENSE for licensing information */
+package org.torproject.metrics.exit;
+
+import java.io.*;
+import java.text.*;
+import java.util.*;
+
+public final class EvaluateExitPortStats {
+
+  private EvaluateExitPortStats() {
+  }
+
+  public static void main(final String[] args) throws Exception {
+
+    if (args.length != 4) {
+      System.err.println("Usage: java "
+          + EvaluateExitPortStats.class.getSimpleName()
+          + " <exit-stats file> <consensuses directory>\n"
+          + "<base64-encoded node identity> <output directory>");
+      System.exit(1);
+    }
+    File exitStatsFile = new File(args[0]);
+    if (!exitStatsFile.exists() || exitStatsFile.isDirectory()) {
+      System.err.println("Exit stats file '"
+          + exitStatsFile.getAbsolutePath()
+          + "' does not exist or is a directory.");
+      System.exit(1);
+    }
+    File consensusesDirectory = new File(args[1]);
+    if (!consensusesDirectory.exists()
+        || !consensusesDirectory.isDirectory()) {
+      System.err.println("Consensuses directory '"
+          + consensusesDirectory.getAbsolutePath()
+          + "' does not exist or is not a directory.");
+      System.exit(1);
+    }
+    String[] identities = args[2].split(",");
+    File outputDirectory = new File(args[3]);
+    if (outputDirectory.exists() && !outputDirectory.isDirectory()) {
+      System.err.println("Output directory '"
+          + outputDirectory.getAbsolutePath()
+          + "' exists, but is not a directory.");
+      System.exit(1);
+    }
+    outputDirectory.mkdir();
+
+    long started = System.currentTimeMillis();
+    long[] written = null, read = null, streams = null;
+    SortedSet<Integer> measuredPorts = new TreeSet<Integer>();
+    long measurementsBegin = 0, measurementsEnd = 0;
+    BufferedReader brExitStats = new BufferedReader(
+        new FileReader(exitStatsFile));
+    String exitStatsLine = null;
+    String formattedMeasurementsEnd = null;
+    SimpleDateFormat timeFormat = new SimpleDateFormat(
+        "yyyy-MM-dd HH:mm:ss");
+    timeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+    Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
+
+    while ((exitStatsLine = brExitStats.readLine()) != null) {
+      if (exitStatsLine.startsWith("written ")) {
+        formattedMeasurementsEnd = exitStatsLine.split(" ")[1] + " "
+            + exitStatsLine.split(" ")[2];
+        measurementsEnd = timeFormat.parse(
+            formattedMeasurementsEnd).getTime();
+        measurementsBegin = measurementsEnd - 1000L * Long.parseLong(
+            exitStatsLine.split(" ")[3].substring(1));
+        continue;
+      }
+      String[] pairs = exitStatsLine.split(" ")[1].split(",");
+      long[] observations = new long[65536];
+      for (String pair : pairs) {
+        long obs = Integer.parseInt(pair.split("=")[1]);
+        if (!pair.split("=")[0].equals("other")) {
+          int port = Integer.parseInt(pair.split("=")[0]);
+          measuredPorts.add(port);
+          observations[port] = obs;
+        }
+        observations[0] += obs;
+      }
+      String type = exitStatsLine.split(" ")[0];
+      if (type.equals("kibibytes-written")) {
+        written = observations;
+      } else if (type.equals("kibibytes-read")) {
+        read = observations;
+      } else if (type.equals("streams-opened")) {
+        streams = observations;
+        // parse consensuses for this day
+        long[] totalCapacity = new long[65536];
+        long[] ourCapacity = new long[65536];
+        Stack<File> filesLeftToParse = new Stack<File>();
+        filesLeftToParse.push(consensusesDirectory);
+        while (!filesLeftToParse.isEmpty()) {
+          File directoryOrFile = filesLeftToParse.pop();
+          if (directoryOrFile.isDirectory()) {
+            for (File fileInDir : directoryOrFile.listFiles()) {
+              filesLeftToParse.push(fileInDir);
+            }
+            continue;
+          }
+          BufferedReader brConsensus = new BufferedReader(new FileReader(
+              directoryOrFile));
+          boolean ourRelay = false;
+          int advertisedBandwidth = 0;
+          String consensusLine = null;
+          while ((consensusLine = brConsensus.readLine()) != null) {
+            if (consensusLine.startsWith("consensus-method ")) {
+              if (Integer.parseInt(consensusLine.split(" ")[1]) < 5) {
+                // consensus method must be 5 or higher
+                break;
+              }
+            } else if (consensusLine.startsWith("valid-after ")) {
+              long validAfterTime = timeFormat.parse(
+                  consensusLine.split(" ")[1] + " "
+                  + consensusLine.split(" ")[2]).getTime();
+              if (validAfterTime <= measurementsBegin
+                  || measurementsEnd < validAfterTime) {
+                // consensus is not in measurement interval
+                break;
+              }
+              System.out.printf("Parsing consensus with valid-after time "
+                  + "%s %s for exit-stats ending at %s.%n",
+                  consensusLine.split(" ")[1], consensusLine.split(" ")[2],
+                  formattedMeasurementsEnd);
+            } else if (consensusLine.startsWith("r ")) {
+              ourRelay = false;
+              for (String identity : identities) {
+                if (consensusLine.split(" ")[2].equals(identity)) {
+                  ourRelay = true;
+                }
+              }
+            } else if (consensusLine.startsWith("w ")) {
+              advertisedBandwidth = Integer.parseInt(
+                  consensusLine.split("=")[1]);
+            } else if (consensusLine.startsWith("p ")) {
+              String[] parts = consensusLine.split(" ");
+              boolean accept = parts[1].equals("accept");
+              String[] entries = parts[2].split(",");
+              int intervalBegin = 0, intervalEnd = 0;
+              List<String> intervals = new ArrayList<String>();
+              for (int i = 0; i < entries.length; i++) {
+                intervals.add(entries[i]);
+              }
+              for (int port = 1; port < 65536; port++) {
+                if (port > intervalEnd && !intervals.isEmpty()) {
+                  String[] intervalParts = intervals.remove(0)
+                      .split("-");
+                  intervalBegin = Integer.parseInt(intervalParts[0]);
+                  intervalEnd = intervalParts.length > 1
+                      ? Integer.parseInt(intervalParts[1])
+                      : intervalBegin;
+                }
+                boolean portInInterval = intervalBegin <= port
+                      && port <= intervalEnd;
+                if (accept == portInInterval) {
+                  totalCapacity[port] += advertisedBandwidth;
+                  if (ourRelay) {
+                    ourCapacity[port] += advertisedBandwidth;
+                  }
+                }
+              }
+            }
+          }
+          brConsensus.close();
+        }
+
+        // write output
+        BufferedWriter bw = new BufferedWriter(new FileWriter(new File(
+            outputDirectory.getAbsolutePath() + File.separatorChar
+            + formattedMeasurementsEnd.replace(' ', '-') + ".csv"),
+            false));
+        bw.write("port,written,read,streams,share\n");
+        for (Integer port : measuredPorts) {
+          bw.write(String.format("%d,%d,%d,%d,%.3f%n",
+              port, written[port], read[port], streams[port],
+              (100.0D * ((double) ourCapacity[port])
+              / ((double) totalCapacity[port]))));
+          }
+        bw.write(String.format("total,%d,%d,%d,NA%n", written[0],
+            read[0], streams[0]));
+        bw.close();
+      }
+    }
+    brExitStats.close();
+    System.out.println("Evaluation finished after "
+        + (System.currentTimeMillis() - started) / 1000 + " seconds.");
+  }
+}
+
-- 
1.5.6.5