[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [doctor/master] Include download statistics in the website report.



commit 812c1ab54f4eb99b2518d505274c5d79a4165d84
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Thu Dec 8 12:55:38 2011 +0100

    Include download statistics in the website report.
---
 src/org/torproject/doctor/Download.java            |    9 ++-
 src/org/torproject/doctor/DownloadStatistics.java  |   91 ++++++++++++++++++++
 src/org/torproject/doctor/Downloader.java          |    2 +-
 src/org/torproject/doctor/Main.java                |    7 ++
 .../torproject/doctor/MetricsWebsiteReport.java    |   62 +++++++++++++
 src/org/torproject/doctor/Report.java              |    4 +
 src/org/torproject/doctor/StatusFileReport.java    |    5 +
 7 files changed, 178 insertions(+), 2 deletions(-)

diff --git a/src/org/torproject/doctor/Download.java b/src/org/torproject/doctor/Download.java
index 35c24f9..1d53cd5 100644
--- a/src/org/torproject/doctor/Download.java
+++ b/src/org/torproject/doctor/Download.java
@@ -22,6 +22,12 @@ public class Download {
     return this.responseString;
   }
 
+  /* Request start timestamp. */
+  private long requestStartMillis;
+  public long getRequestStartMillis() {
+    return this.requestStartMillis;
+  }
+
   /* Fetch time in millis. */
   private long fetchTime;
   public long getFetchTime() {
@@ -29,9 +35,10 @@ public class Download {
   }
 
   public Download(String authority, String url, String responseString,
-      long fetchTime) {
+      long requestStartMillis, long fetchTime) {
     this.authority = authority;
     this.responseString = responseString;
+    this.requestStartMillis = requestStartMillis;
     this.fetchTime = fetchTime;
   }
 }
diff --git a/src/org/torproject/doctor/DownloadStatistics.java b/src/org/torproject/doctor/DownloadStatistics.java
new file mode 100644
index 0000000..9eeffde
--- /dev/null
+++ b/src/org/torproject/doctor/DownloadStatistics.java
@@ -0,0 +1,91 @@
+/* Copyright 2011 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.doctor;
+
+import java.io.*;
+import java.util.*;
+
+public class DownloadStatistics {
+  public void memorizeFetchTimes(List<Download> downloadedConsensuses) {
+    try {
+      BufferedWriter bw = new BufferedWriter(new FileWriter(
+          this.statisticsFile, true));
+      for (Download downloadedConsensus : downloadedConsensuses) {
+        String authority = downloadedConsensus.getAuthority();
+        long requestStartMillis =
+            downloadedConsensus.getRequestStartMillis();
+        long fetchTimeMillis = downloadedConsensus.getFetchTime();
+        String line = authority + ","
+            + String.valueOf(requestStartMillis) + ","
+            + String.valueOf(fetchTimeMillis);
+        bw.write(line + "\n");
+      }
+      bw.close();
+    } catch (IOException e) {
+      System.err.println("Could not write "
+          + this.statisticsFile.getAbsolutePath() + ".  Ignoring.");
+    }
+  }
+  private SortedMap<String, List<Long>> downloadData =
+      new TreeMap<String, List<Long>>();
+  private int maxDownloadsPerAuthority = 0;
+  private File statisticsFile = new File("download-stats.csv");
+  public void prepareStatistics() {
+    if (this.statisticsFile.exists()) {
+      long cutOffMillis = System.currentTimeMillis()
+          - (7L * 24L * 60L * 60L * 1000L);
+      try {
+        BufferedReader br = new BufferedReader(new FileReader(
+            this.statisticsFile));
+        String line;
+        while ((line = br.readLine()) != null) {
+          String[] parts = line.split(",");
+          long requestStartMillis = Long.parseLong(parts[1]);
+          if (requestStartMillis < cutOffMillis) {
+            continue;
+          }
+          String authority = parts[0];
+          if (!this.downloadData.containsKey(authority)) {
+            this.downloadData.put(authority, new ArrayList<Long>());
+          }
+          long fetchTimeMillis = Long.parseLong(parts[2]);
+          this.downloadData.get(authority).add(fetchTimeMillis);
+        }
+        br.close();
+      } catch (IOException e) {
+        System.err.println("Could not read "
+            + this.statisticsFile.getAbsolutePath() + ".  Ignoring.");
+      }
+      for (Map.Entry<String, List<Long>> e :
+          this.downloadData.entrySet()) {
+        Collections.sort(e.getValue());
+        int downloads = e.getValue().size();
+        if (downloads > this.maxDownloadsPerAuthority) {
+          this.maxDownloadsPerAuthority = downloads;
+        }
+      }
+    }
+  }
+  public SortedSet<String> getKnownAuthorities() {
+    return new TreeSet<String>(this.downloadData.keySet());
+  }
+  public String getPercentile(String authority, int percentile) {
+    if (percentile < 0 || percentile > 100 ||
+        !this.downloadData.containsKey(authority)) {
+      return "NA";
+    } else {
+      List<Long> fetchTimes = this.downloadData.get(authority);
+      int index = (percentile * (fetchTimes.size() - 1)) / 100;
+      return String.valueOf(fetchTimes.get(index));
+    }
+  }
+  public String getNAs(String authority) {
+    if (!this.downloadData.containsKey(authority)) {
+      return "NA";
+    } else {
+      return String.valueOf(this.maxDownloadsPerAuthority
+          - this.downloadData.get(authority).size());
+    }
+  }
+}
+
diff --git a/src/org/torproject/doctor/Downloader.java b/src/org/torproject/doctor/Downloader.java
index c929a96..db44b8b 100755
--- a/src/org/torproject/doctor/Downloader.java
+++ b/src/org/torproject/doctor/Downloader.java
@@ -106,7 +106,7 @@ public class Downloader {
       Download result = null;
       if (this.response != null) {
         result = new Download(this.nickname, this.url, this.response,
-          this.requestEnd - this.requestStart);
+          this.requestStart, this.requestEnd - this.requestStart);
       }
       return result;
     }
diff --git a/src/org/torproject/doctor/Main.java b/src/org/torproject/doctor/Main.java
index ec160e9..0de6298 100755
--- a/src/org/torproject/doctor/Main.java
+++ b/src/org/torproject/doctor/Main.java
@@ -22,6 +22,12 @@ public class Main {
     List<Download> downloadedConsensuses = downloader.getConsensuses();
     List<Download> downloadedVotes = downloader.getVotes();
 
+    /* Write fetch times for requesting consensuses to disk and prepare
+     * statistics about fetch times in the last 7 days. */
+    DownloadStatistics fetchStatistics = new DownloadStatistics();
+    fetchStatistics.memorizeFetchTimes(downloadedConsensuses);
+    fetchStatistics.prepareStatistics();
+
     /* Parse consensus and votes. */
     Parser parser = new Parser();
     SortedMap<String, Status> parsedDownloadedConsensuses = parser.parse(
@@ -37,6 +43,7 @@ public class Main {
     for (Report report : reports) {
       report.processWarnings(warnings);
       report.processDownloadedConsensuses(parsedDownloadedConsensuses);
+      report.includeFetchStatistics(fetchStatistics);
       report.writeReport();
     }
 
diff --git a/src/org/torproject/doctor/MetricsWebsiteReport.java b/src/org/torproject/doctor/MetricsWebsiteReport.java
index b03f652..9ae897a 100755
--- a/src/org/torproject/doctor/MetricsWebsiteReport.java
+++ b/src/org/torproject/doctor/MetricsWebsiteReport.java
@@ -51,6 +51,14 @@ public class MetricsWebsiteReport implements Report {
     }
   }
 
+  /* Store the DownloadStatistics reference to request download statistics
+   * when writing the report. */
+  private DownloadStatistics statistics;
+  public void includeFetchStatistics(
+      DownloadStatistics statistics) {
+    this.statistics = statistics;
+  }
+
   /* Writer to write all HTML output to. */
   private BufferedWriter bw;
 
@@ -71,6 +79,7 @@ public class MetricsWebsiteReport implements Report {
         writeAuthorityKeys();
         writeBandwidthScannerStatus();
         writeAuthorityVersions();
+        writeDownloadStatistics();
         writeRelayFlagsTable();
         writeRelayFlagsSummary();
         writePageFooter();
@@ -569,6 +578,59 @@ public class MetricsWebsiteReport implements Report {
     }
   }
 
+
+  /* Write some download statistics. */
+  private void writeDownloadStatistics() throws IOException {
+    SortedSet<String> knownAuthorities =
+        this.statistics.getKnownAuthorities();
+    if (knownAuthorities.isEmpty()) {
+      return;
+    }
+    this.bw.write("        <br>\n"
+         + "        <a name=\"downloadstats\">\n"
+         + "        <h3><a href=\"#downloadstats\" class=\"anchor\">"
+           + "Consensus download statistics</a></h3>\n"
+        + "        <br>\n"
+        + "        <p>The following table contains statistics on "
+          + "consensus download times in milliseconds over the last 7 "
+          + "days:</p>\n"
+        + "        <table border=\"0\" cellpadding=\"4\" "
+        + "cellspacing=\"0\" summary=\"\">\n"
+        + "          <colgroup>\n"
+        + "            <col width=\"160\">\n"
+        + "            <col width=\"100\">\n"
+        + "            <col width=\"100\">\n"
+        + "            <col width=\"100\">\n"
+        + "            <col width=\"100\">\n"
+        + "            <col width=\"100\">\n"
+        + "            <col width=\"100\">\n"
+        + "          </colgroup>\n"
+        + "          <tr><td><b>Authority</b></td>"
+          + "<td><b>Minimum</b></td>"
+          + "<td><b>1st Quartile</b></td>"
+          + "<td><b>Median</b></td>"
+          + "<td><b>3rd Quartile</b></td>"
+          + "<td><b>Maximum</b></td>"
+          + "<td><b>Timeouts</b></td></tr>\n");
+    for (String authority : knownAuthorities) {
+      this.bw.write("          <tr>\n"
+          + "            <td>" + authority + "</td>\n"
+          + "            <td>"
+            + this.statistics.getPercentile(authority, 0) + "</td>"
+          + "            <td>"
+            + this.statistics.getPercentile(authority, 25) + "</td>"
+          + "            <td>"
+            + this.statistics.getPercentile(authority, 50) + "</td>"
+          + "            <td>"
+            + this.statistics.getPercentile(authority, 75) + "</td>"
+          + "            <td>"
+            + this.statistics.getPercentile(authority, 100) + "</td>"
+          + "            <td>"
+            + this.statistics.getNAs(authority) + "</td></tr>\n");
+    }
+    this.bw.write("        </table>\n");
+  }
+
   /* Write the (huge) table containing relay flags contained in votes and
    * the consensus for each relay. */
   private void writeRelayFlagsTable() throws IOException {
diff --git a/src/org/torproject/doctor/Report.java b/src/org/torproject/doctor/Report.java
index 04e2bc1..2e508bc 100755
--- a/src/org/torproject/doctor/Report.java
+++ b/src/org/torproject/doctor/Report.java
@@ -17,6 +17,10 @@ public interface Report {
   public abstract void processWarnings(
       SortedMap<Warning, String> warnings);
 
+  /* Include download statistics. */
+  public abstract void includeFetchStatistics(
+      DownloadStatistics statistics);
+
   /* Finish writing report. */
   public abstract void writeReport();
 }
diff --git a/src/org/torproject/doctor/StatusFileReport.java b/src/org/torproject/doctor/StatusFileReport.java
index 9583132..1fe3b87 100755
--- a/src/org/torproject/doctor/StatusFileReport.java
+++ b/src/org/torproject/doctor/StatusFileReport.java
@@ -33,6 +33,11 @@ public class StatusFileReport implements Report {
     this.warnings = warnings;
   }
 
+  /* Ignore download statistics for this report. */
+  public void includeFetchStatistics(DownloadStatistics statistics) {
+    /* Do nothing. */
+  }
+
   /* Check consensuses and votes for irregularities and write output to
    * stdout. */
   public void writeReport() {

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits