[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[or-cvs] [metrics/master] Parse the new dirreq stats format.



Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Mon, 12 Oct 2009 11:40:50 +0200
Subject: Parse the new dirreq stats format.
Commit: e31189aed07d228d544f3b3fdb452e995aa4d993

---
 .../torproject/metrics/dirreq/ParseGeoipStats.java |  118 +++++++++++++++-----
 1 files changed, 88 insertions(+), 30 deletions(-)

diff --git a/src/org/torproject/metrics/dirreq/ParseGeoipStats.java b/src/org/torproject/metrics/dirreq/ParseGeoipStats.java
index 7f69e7f..2bf8251 100644
--- a/src/org/torproject/metrics/dirreq/ParseGeoipStats.java
+++ b/src/org/torproject/metrics/dirreq/ParseGeoipStats.java
@@ -16,8 +16,14 @@ public final class ParseGeoipStats {
         SortedMap<String, Integer> v3Ips;
         SortedMap<String, Integer> v2Reqs;
         SortedMap<String, Integer> v3Reqs;
+        String v2Resp;
+        String v3Resp;
         int v2Share;
         int v3Share;
+        String v2DirectDl;
+        String v3DirectDl;
+        String v2TunneledDl;
+        String v3TunneledDl;
     }
 
     private static SortedSet<String> allCountries = new TreeSet<String>();
@@ -40,8 +46,24 @@ public final class ParseGeoipStats {
         return result;
     }
 
+    private static String toCommaSeparatedString(String line, int values) {
+        StringBuilder out = new StringBuilder();
+        String[] parts = line.split(" ")[1].split(",");
+        for (String s : parts) {
+            String[] p = s.split("=");
+            out.append("," + p[1]);
+        }
+        for (int i = parts.length; i < values; i++) {
+            out.append(",NA");
+        }
+        return out.toString();
+    }
+
     private static String estimateRequestsAndClients(int localRequests,
             int localIpsInt, int shareAsInt) {
+        if (shareAsInt == 0) {
+            return "NA,NA,NA";
+        }
         double share = ((double) shareAsInt) / 10000 * 5 / 4;
         double totalRequests = (double) localRequests / share;
         double totalClients = 10000.0D;
@@ -109,45 +131,59 @@ public final class ParseGeoipStats {
             DataPoint currentDataPoint = null;
             boolean haveSeenActualNumbers = false;
             while ((line = br.readLine()) != null) {
-                if (line.startsWith("written ")) {
+                if (line.startsWith("dirreq-stats-end ")) {
                     if (haveSeenActualNumbers) {
                         currentDataPoints.put(currentDate, currentDataPoint);
                     }
                     currentDataPoint = new DataPoint();
                     currentDate = line.split(" ")[1];
                     allDates.add(currentDate);
-                } else if (line.startsWith("started-at ")) {
-                    // ignored
-                } else if (line.startsWith("ns-ips ")) {
+                } else if (line.startsWith("dirreq-v3-ips ")) {
                     currentDataPoint.v3Ips = parseCountryLine(line);
                     if (line.split(" ").length > 1) {
                         haveSeenActualNumbers = true;
                     }
-                } else if (line.startsWith("ns-v2-ips ")) {
+                } else if (line.startsWith("dirreq-v2-ips ")) {
                     currentDataPoint.v2Ips = parseCountryLine(line);
                     if (line.split(" ").length > 1) {
                         haveSeenActualNumbers = true;
                     }
-                } else if (line.startsWith("requests-start ")) {
-                    // ignored
-                } else if (line.startsWith("n-ns-reqs ")) {
+                } else if (line.startsWith("dirreq-v3-reqs ")) {
                     currentDataPoint.v3Reqs = parseCountryLine(line);
                     if (line.split(" ").length > 1) {
                         haveSeenActualNumbers = true;
                     }
-                } else if (line.startsWith("n-v2-ns-reqs ")) {
+                } else if (line.startsWith("dirreq-v2-reqs ")) {
                     currentDataPoint.v2Reqs = parseCountryLine(line);
                     if (line.split(" ").length > 1) {
                         haveSeenActualNumbers = true;
                     }
-                } else if (line.startsWith("v2-ns-share ")) {
+                } else if (line.startsWith("dirreq-v3-resp ")) {
+                    currentDataPoint.v3Resp =
+                            toCommaSeparatedString(line, 6);
+                } else if (line.startsWith("dirreq-v2-resp ")) {
+                    currentDataPoint.v2Resp =
+                            toCommaSeparatedString(line, 5);
+                } else if (line.startsWith("dirreq-v2-share ")) {
                     currentDataPoint.v2Share = Integer.parseInt(
                             line.split(" ")[1].replace('.', ';')
                             .replace('%', ';').replaceAll(";", ""));
-                } else if (line.startsWith("v3-ns-share ")) {
+                } else if (line.startsWith("dirreq-v3-share ")) {
                     currentDataPoint.v3Share = Integer.parseInt(
                             line.split(" ")[1].replace('.', ';')
                             .replace('%', ';').replaceAll(";", ""));
+                } else if (line.startsWith("dirreq-v3-direct-dl ")) {
+                    currentDataPoint.v3DirectDl =
+                            toCommaSeparatedString(line, 16);
+                } else if (line.startsWith("dirreq-v2-direct-dl ")) {
+                    currentDataPoint.v2DirectDl =
+                            toCommaSeparatedString(line, 16);
+                } else if (line.startsWith("dirreq-v3-tunneled-dl ")) {
+                    currentDataPoint.v3TunneledDl =
+                            toCommaSeparatedString(line, 16);
+                } else if (line.startsWith("dirreq-v2-tunneled-dl ")) {
+                    currentDataPoint.v2TunneledDl =
+                            toCommaSeparatedString(line, 16);
                 }
             }
             if (haveSeenActualNumbers) {
@@ -160,35 +196,51 @@ public final class ParseGeoipStats {
                 + "directories.%n", allCountries.size(), allDates.size(),
                 allDataPoints.size());
 
+        File outFile = new File(outputDirectory.getAbsolutePath()
+                + File.separatorChar + "dirreq.csv");
+        BufferedWriter out = new BufferedWriter(new FileWriter(
+                outFile, false));
+        out.write("directory,time,");
+        for (String f : allCountries) {
+            out.write(String.format("ip2%s,ip3%<s,ipt%<s,"
+                    + "req2%<s,req3%<s,reqt%<s,", f));
+        }
+        out.write("ip2total,ip3total,ipttotal,"
+                + "req2total,req3total,reqttotal,"
+                + "v2share,v3share,"
+                + "req2estimate,ip2estimate,reqperip2,"
+                + "req3estimate,ip3estimate,reqperip3,"
+                + "v2ok,v2unav,v2nfound,v2nmod,v2busy,"
+                + "v3ok,v3nsigs,v3unav,v3nfound,v3nmod,v3busy,"
+                + "v2dcompl,v2dtimeo,v2drunn,"
+                + "v2dmin,v2dd1,v2dd2,v2dq1,v2dd3,v2dd4,v2dmd,"
+                + "v2dd6,v2dd7,v2dq3,v2dd8,v2dd9,v2dmax,"
+                + "v2tcompl,v2ttimeo,v2trunn,"
+                + "v2tmin,v2td1,v2td2,v2tq1,v2td3,v2td4,v2tmd,"
+                + "v2td6,v2td7,v2tq3,v2td8,v2td9,v2tmax,"
+                + "v3dcompl,v3dtimeo,v3drunn,"
+                + "v3dmin,v3dd1,v3dd2,v3dq1,v3dd3,v3dd4,v3dmd,"
+                + "v3dd6,v3dd7,v3dq3,v3dd8,v3dd9,v3dmax,"
+                + "v3tcompl,v3ttimeo,v3trunn,"
+                + "v3tmin,v3td1,v3td2,v3tq1,v3td3,v3td4,v3tmd,"
+                + "v3td6,v3td7,v3tq3,v3td8,v3td9,v3tmax\n");
         for (Map.Entry<String, SortedMap<String, DataPoint>> e
                 : allDataPoints.entrySet()) {
             String directory = e.getKey();
             SortedMap<String, DataPoint> dataPoints = e.getValue();
-            File outFile = new File(outputDirectory.getAbsolutePath()
-                    + File.separatorChar + directory + ".csv");
-            BufferedWriter out = new BufferedWriter(new FileWriter(
-                    outFile, false));
-            out.write("time,");
-            for (String f : allCountries) {
-                out.write(String.format("ip2%s,ip3%<s,ipt%<s,"
-                        + "req2%<s,req3%<s,reqt%<s,", f));
-            }
-            out.write("ip2total,ip3total,ipttotal,"
-                    + "req2total,req3total,reqttotal,"
-                    + "v2share,v3share,"
-                    + "req2estimate,ip2estimate,reqperip2,"
-                    + "req3estimate,ip3estimate,reqperip3\n");
+
+
             for (String date : allDates) {
                 if (!dataPoints.containsKey(date)) {
-                    out.write(date + ",");
-                    int nas = allCountries.size() * 6 + 7;
+                    out.write(directory + "," + date + ",");
+                    int nas = allCountries.size() * 6 + 13 + 6+5+16*4;
                     for (int i = 0; i < nas; i++) {
                         out.write("NA,");
                     }
                     out.write("NA\n");
                 } else {
                     DataPoint currentDataPoint = dataPoints.get(date);
-                    out.write(date + ",");
+                    out.write(directory + "," + date + ",");
                     int ip2total = 0, ip3total = 0, ipttotal = 0,
                             req2total = 0, req3total = 0, reqttotal = 0;
                     for (String f : allCountries) {
@@ -218,13 +270,19 @@ public final class ParseGeoipStats {
                     out.write(String.format(",%s",
                             estimateRequestsAndClients(req2total, ip2total,
                             currentDataPoint.v2Share)));
-                    out.write(String.format(",%s%n",
+                    out.write(String.format(",%s",
                             estimateRequestsAndClients(req3total, ip3total,
                             currentDataPoint.v3Share)));
+                    out.write(currentDataPoint.v2Resp
+                            + currentDataPoint.v3Resp
+                            + currentDataPoint.v2DirectDl
+                            + currentDataPoint.v2TunneledDl
+                            + currentDataPoint.v3DirectDl
+                            + currentDataPoint.v3TunneledDl + "\n");
                 }
             }
-            out.close();
         }
+        out.close();
 
         System.out.println("Parsing finished after "
             + ((System.currentTimeMillis() - started) / 1000)
-- 
1.5.6.5