[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[or-cvs] [ernie/master] Add parser and graphs for gettor stats.
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Sun, 14 Feb 2010 23:56:14 +0100
Subject: Add parser and graphs for gettor stats.
Commit: ac4a99b44c7a666e6dc1bc5e3d8d6641294ba90b
---
R/gettor.R | 42 +++++++++++++++++++
R/graphs.R | 1 +
src/GetTorProcessor.java | 97 ++++++++++++++++++++++++++++++++++++++++++++
src/Main.java | 1 +
website/gettor-graphs.html | 45 ++++++++++++++++++++
website/graphs.html | 10 +++++
6 files changed, 196 insertions(+), 0 deletions(-)
create mode 100644 R/gettor.R
create mode 100644 src/GetTorProcessor.java
create mode 100644 website/gettor-graphs.html
diff --git a/R/gettor.R b/R/gettor.R
new file mode 100644
index 0000000..b37b3ed
--- /dev/null
+++ b/R/gettor.R
@@ -0,0 +1,42 @@
+options(warn = -1)
+suppressPackageStartupMessages(library("ggplot2"))
+
+gettor <- read.csv("stats/gettor-stats", header = TRUE,
+ stringsAsFactors = FALSE);
+total <- data.frame(date = gettor$date,
+ packages = rowSums(gettor[2:length(gettor)]) - gettor$none)
+en <- data.frame(date = gettor$date,
+ packages = gettor$tor.browser.bundle_en + gettor$tor.im.browser.bundle_en)
+zh_cn <- data.frame(date = gettor$date,
+ packages = gettor$tor.browser.bundle_zh_cn +
+ gettor$tor.im.browser.bundle_zh_cn)
+fa <- data.frame(date = gettor$date,
+ packages = gettor$tor.browser.bundle_fa + gettor$tor.im.browser.bundle_fa)
+
+write.csv(data.frame(date = gettor$date,
+ total = rowSums(gettor[2:length(gettor)]) - gettor$none,
+ en = gettor$tor.browser.bundle_en + gettor$tor.im.browser.bundle_en,
+ zh_cn = gettor$tor.browser.bundle_zh_cn +
+ gettor$tor.im.browser.bundle_zh_cn,
+ fa = gettor$tor.browser.bundle_fa + gettor$tor.im.browser.bundle_fa),
+ "website/csv/gettor.csv", quote = FALSE, row.names = FALSE)
+
+plot_packages <- function(filename, title, data) {
+ ggplot(data, aes(x = as.Date(date, "%Y-%m-%d"), y = packages)) + geom_line() +
+ scale_x_date(name = "") +
+ scale_y_continuous(name = "",
+ limits = c(0, max(data$packages, na.rm = TRUE))) +
+ opts(title = paste(title, "\n", sep = ""))
+ ggsave(filename = paste("website/graphs/gettor/", filename, sep = ""),
+ width = 8, height = 5, dpi = 72)
+}
+
+plot_packages("gettor-total.png",
+ "Total packages delivered by GetTor per day", total)
+plot_packages("gettor-en.png",
+ "Tor Browser Bundles (en) delivered by GetTor per day", en)
+plot_packages("gettor-zh_cn.png",
+ "Tor Browser Bundles (zh_CN) delivered by GetTor per day", zh_cn)
+plot_packages("gettor-fa.png",
+ "Tor Browser Bundles (fa) delivered by GetTor per day", fa)
+
diff --git a/R/graphs.R b/R/graphs.R
index 8881d6c..1fc819b 100644
--- a/R/graphs.R
+++ b/R/graphs.R
@@ -2,4 +2,5 @@ source("R/consensus-stats.R");
source("R/dirreq-stats.R");
source("R/bridge-stats.R");
source("R/torperf.R");
+source("R/gettor.R");
diff --git a/src/GetTorProcessor.java b/src/GetTorProcessor.java
new file mode 100644
index 0000000..21b9df1
--- /dev/null
+++ b/src/GetTorProcessor.java
@@ -0,0 +1,97 @@
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.logging.*;
+
+public class GetTorProcessor {
+ private final String gettorStatsUrl =
+ "http://gettor.torproject.org:8080/~gettor/gettor_stats.txt";;
+ public GetTorProcessor(String statsDirectory) {
+ Logger logger = Logger.getLogger(TorperfProcessor.class.getName());
+ Calendar now = Calendar.getInstance();
+ now.setTime(new Date());
+ if (now.get(Calendar.HOUR_OF_DAY) % 6 != 0) {
+ // only download every 6th hour
+ logger.info("Skipping downloading gettor stats.");
+ return;
+ }
+ String unparsed = null;
+ try {
+ logger.info("Downloading gettor stats...");
+ URL u = new URL(gettorStatsUrl);
+ HttpURLConnection huc = (HttpURLConnection) u.openConnection();
+ huc.setRequestMethod("GET");
+ huc.connect();
+ int response = huc.getResponseCode();
+ if (response == 200) {
+ BufferedInputStream in = new BufferedInputStream(
+ huc.getInputStream());
+ StringBuilder sb = new StringBuilder();
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = in.read(data, 0, 1024)) >= 0) {
+ sb.append(new String(data, 0, len));
+ }
+ in.close();
+ unparsed = sb.toString();
+ }
+ logger.info("Finished downloading gettor stats.");
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Failed downloading gettor stats", e);
+ return;
+ }
+
+ SortedSet<String> columns = new TreeSet<String>();
+ SortedMap<String, Map<String, Integer>> data =
+ new TreeMap<String, Map<String, Integer>>();
+ try {
+ logger.info("Parsing downloaded gettor stats...");
+ BufferedReader br = new BufferedReader(new StringReader(unparsed));
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ String[] parts = line.split(" ");
+ String date = parts[0];
+ Map<String, Integer> obs = new HashMap<String, Integer>();
+ data.put(date, obs);
+ for (int i = 2; i < parts.length; i++) {
+ String key = parts[i].split(":")[0].toLowerCase();
+ Integer value = new Integer(parts[i].split(":")[1]);
+ columns.add(key);
+ obs.put(key, value);
+ }
+ }
+ br.close();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Failed parsing gettor stats!", e);
+ return;
+ } catch (NumberFormatException e) {
+ logger.log(Level.WARNING, "Failed parsing gettor stats!", e);
+ return;
+ }
+
+ logger.info("Writing file " + statsDirectory + "/gettor-stats...");
+ try {
+ File statsFile = new File(statsDirectory + "/gettor-stats");
+ new File(statsDirectory).mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(statsFile));
+ bw.write("date");
+ for (String column : columns) {
+ bw.write("," + column);
+ }
+ bw.write("\n");
+ for (String date : data.keySet()) {
+ bw.write(date);
+ for (String column : columns) {
+ Integer value = data.get(date).get(column);
+ bw.write("," + (value == null ? "NA" : value));
+ }
+ bw.write("\n");
+ }
+ bw.close();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Failed writing " + statsDirectory
+ + "/gettor-stats!", e);
+ }
+ }
+}
+
diff --git a/src/Main.java b/src/Main.java
index 1c78520..dd11eed 100644
--- a/src/Main.java
+++ b/src/Main.java
@@ -74,6 +74,7 @@ public class Main {
"bridge-directories", statsDirectory, countries);
TorperfProcessor tp = new TorperfProcessor(statsDirectory,
"torperf");
+ GetTorProcessor gtp = new GetTorProcessor(statsDirectory);
logger.info("Finished importing data.");
}
diff --git a/website/gettor-graphs.html b/website/gettor-graphs.html
new file mode 100644
index 0000000..c69a794
--- /dev/null
+++ b/website/gettor-graphs.html
@@ -0,0 +1,45 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
+<html>
+ <head>
+ <title>Tor Metrics Portal: Packages delivered by GetTor</title>
+ <meta http-equiv=Content-Type content="text/html; charset=iso-8859-1">
+ <link href="http://www.torproject.org/stylesheet-ltr.css"; type=text/css rel=stylesheet>
+ <link href="http://www.torproject.org/favicon.ico"; type=image/x-icon rel="shortcut icon">
+ </head>
+ <body>
+ <div class="center">
+ <table class="banner" border="0" cellpadding="0" cellspacing="0" summary="">
+ <tr>
+ <td class="banner-left"><a href="https://www.torproject.org/";><img src="http://www.torproject.org/images/top-left.png"; alt="Click to go to home page" width="193" height="79"></a></td>
+ <td class="banner-middle">
+ <a href="/">Home</a>
+ <a class="current">Graphs</a>
+ <a href="reports.html">Reports</a>
+ <a href="papers.html">Papers</a>
+ <a href="data.html">Data</a>
+ <a href="docs.html">Docs</a>
+ </td>
+ <td class="banner-right"></td>
+ </tr>
+ </table>
+ <div class="main-column">
+ <h2>Tor Metrics Portal: Graphs</h2>
+ <br/>
+ <h3>Packages delivered by GetTor</h3>
+ <br/>
+ <p>GetTor allows users to fetch Tor via email. The following
+ graphs show the number of delivered packages per day.</p>
+ <p><a href="csv/gettor.csv">CSV</a> file containing all data.</p>
+ <img src="graphs/gettor/gettor-total.png"/>
+ <img src="graphs/gettor/gettor-en.png"/>
+ <img src="graphs/gettor/gettor-zh_cn.png"/>
+ <img src="graphs/gettor/gettor-fa.png"/>
+ <br/>
+ </div>
+ </div>
+ <div class="bottom" id="bottom">
+ <p>"Tor" and the "Onion Logo" are <a href="https://www.torproject.org/trademark-faq.html.en";>registered trademarks</a> of The Tor Project, Inc.</p>
+ </div>
+ </body>
+</html>
+
diff --git a/website/graphs.html b/website/graphs.html
index 113b645..219dc2b 100644
--- a/website/graphs.html
+++ b/website/graphs.html
@@ -38,6 +38,7 @@
users</a></li>
<li><a href="#bridgeusers">Tor users via bridges</a></li>
<li><a href="#torperf">Time to complete requests</a></li>
+ <li><a href="#gettor">Packages delivered by GetTor</a></li>
</ul>
<br/>
<a id="relays"/>
@@ -98,6 +99,15 @@
<p>Graphs for other file sizes or time intervals can be found on a
<a href="torperf-graphs.html">separate page</a>.</p>
<br/>
+ <a id="gettor"/>
+ <h3>Packages delivered by GetTor</h3>
+ <br/>
+ <p>GetTor allows users to fetch Tor via email. The following
+ graphs show the number of delivered packages per day.</p>
+ <img src="graphs/gettor/gettor-total.png"/>
+ <p>More graphs about specific packages can be found on a
+ <a href="gettor-graphs.html">separate page</a>.</p>
+ <br/>
</div>
</div>
<div class="bottom" id="bottom">
--
1.6.5