[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [metrics-web/master] Add #8462 graphs.



commit b50aabcced4b363c8fff1c8e8062e471af59735a
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Mon May 6 20:36:34 2013 +0200

    Add #8462 graphs.
---
 etc/web.xml                                        |   12 ++
 rserve/graphs.R                                    |   74 +++++++++
 .../ernie/web/graphs/GraphParameterChecker.java    |   75 +++++-----
 .../ernie/web/graphs/RObjectGenerator.java         |    8 +
 web/WEB-INF/users.jsp                              |  167 ++++++++++++++++++++
 5 files changed, 300 insertions(+), 36 deletions(-)

diff --git a/etc/web.xml b/etc/web.xml
index 79ac4c1..630af3e 100644
--- a/etc/web.xml
+++ b/etc/web.xml
@@ -197,6 +197,18 @@
     <url-pattern>/almost-fast-exits.png</url-pattern>
     <url-pattern>/almost-fast-exits.pdf</url-pattern>
     <url-pattern>/almost-fast-exits.svg</url-pattern>
+    <url-pattern>/userstats-relay-country.png</url-pattern>
+    <url-pattern>/userstats-relay-country.pdf</url-pattern>
+    <url-pattern>/userstats-relay-country.svg</url-pattern>
+    <url-pattern>/userstats-bridge-country.png</url-pattern>
+    <url-pattern>/userstats-bridge-country.pdf</url-pattern>
+    <url-pattern>/userstats-bridge-country.svg</url-pattern>
+    <url-pattern>/userstats-bridge-transport.png</url-pattern>
+    <url-pattern>/userstats-bridge-transport.pdf</url-pattern>
+    <url-pattern>/userstats-bridge-transport.svg</url-pattern>
+    <url-pattern>/userstats-bridge-version.png</url-pattern>
+    <url-pattern>/userstats-bridge-version.pdf</url-pattern>
+    <url-pattern>/userstats-bridge-version.svg</url-pattern>
   </servlet-mapping>
 
   <servlet>
diff --git a/rserve/graphs.R b/rserve/graphs.R
index 72bbc1e..b1ffd86 100644
--- a/rserve/graphs.R
+++ b/rserve/graphs.R
@@ -1022,3 +1022,77 @@ plot_bandwidth_flags <- function(start, end, path) {
   ggsave(filename = path, width = 8, height = 5, dpi = 72)
 }
 
+plot_userstats <- function(start, end, node, variable, value, path) {
+  end <- min(end, as.character(Sys.Date()))
+  u <- read.csv(paste("/srv/metrics.torproject.org/task-8462-graphs/",
+    "task-8462/userstats.csv", sep = ""),
+    stringsAsFactors = FALSE)
+  if (node == 'relay') {
+    if (value != 'all') {
+      u <- u[u$country == value & u$node == 'relay', ]
+      title <- paste("Directly connecting users from ",
+                     countryname(value), " (BETA)\n", sep = "")
+    } else {
+      u <- u[u$country == '' & u$transport == '' & u$version == '' &
+             u$node == 'relay', ]
+      title <- "Directly connecting users (BETA)\n"
+    }
+  } else if (variable == 'transport') {
+    u <- u[u$transport == value & u$node == 'bridge', ]
+    title <- paste("Bridge users using transport ", value, " (BETA)\n",
+                   sep = "")
+  } else if (variable == 'version') {
+    u <- u[u$version== value & u$node == 'bridge', ]
+    title <- paste("Bridge users using IP", value, " (BETA)\n", sep = "")
+  } else {
+    if (value != 'all') {
+      u <- u[u$country == value & u$node == 'bridge', ]
+      title <- paste("Bridge users from ", countryname(value),
+                     " (BETA)\n", sep = "")
+    } else {
+      u <- u[u$country == '' & u$transport == '' & u$version == '' &
+            u$node == 'bridge', ]
+      title <- "Bridge users (BETA)\n"
+    }
+  }
+  u <- data.frame(date = as.Date(u$date, "%Y-%m-%d"), users = u$users)
+  dates <- seq(from = as.Date(start, "%Y-%m-%d"),
+      to = as.Date(end, "%Y-%m-%d"), by="1 day")
+  missing <- setdiff(dates, u$date)
+  if (length(missing) > 0) {
+    u <- rbind(u,
+        data.frame(date = as.Date(missing, origin = "1970-01-01"),
+        users = NA))
+  }
+  formatter <- function(x, ...) { format(x, scientific = FALSE, ...) }
+  date_breaks <- date_breaks(
+    as.numeric(max(u$date) - min(u$date)))
+  ggplot(u, aes(x = date, y = users)) +
+    geom_line(size = 1) +
+    scale_x_date(name = paste("\nThe Tor Project - ",
+        "https://metrics.torproject.org/";, sep = ""),
+        format = date_breaks$format, major = date_breaks$major,
+        minor = date_breaks$minor) +
+    scale_y_continuous(name = "", limits = c(0,
+        ifelse(length(na.omit(u$users)) == 0, 0,
+        max(u$users, na.rm = TRUE))), formatter = formatter) +
+    opts(title = title)
+  ggsave(filename = path, width = 8, height = 5, dpi = 72)
+}
+
+plot_userstats_relay_country <- function(start, end, country, path) {
+  plot_userstats(start, end, 'relay', 'country', country, path)
+}
+
+plot_userstats_bridge_country <- function(start, end, country, path) {
+  plot_userstats(start, end, 'bridge', 'country', country, path)
+}
+
+plot_userstats_bridge_transport <- function(start, end, transport, path) {
+  plot_userstats(start, end, 'bridge', 'transport', transport, path)
+}
+
+plot_userstats_bridge_version <- function(start, end, version, path) {
+  plot_userstats(start, end, 'bridge', 'version', version, path)
+}
+
diff --git a/src/org/torproject/ernie/web/graphs/GraphParameterChecker.java b/src/org/torproject/ernie/web/graphs/GraphParameterChecker.java
index 8300be5..74ca6f9 100644
--- a/src/org/torproject/ernie/web/graphs/GraphParameterChecker.java
+++ b/src/org/torproject/ernie/web/graphs/GraphParameterChecker.java
@@ -56,9 +56,11 @@ public class GraphParameterChecker {
     }
     this.knownParameterValues.put("country", sb.toString());
     this.knownParameterValues.put("events", "on,off,points");
-    this.knownParameterValues.put("language", "all,en,zh_CN,fa");
     this.knownParameterValues.put("source", "all,siv,moria,torperf");
     this.knownParameterValues.put("filesize", "50kb,1mb,5mb");
+    this.knownParameterValues.put("transport",
+        "obfs2,obfs3,websocket,<OR>,<??>");
+    this.knownParameterValues.put("version", "v4,v6");
   }
 
   public void setAvailableGraphs(Map<String, String> availableGraphs) {
@@ -187,26 +189,6 @@ public class GraphParameterChecker {
       recognizedGraphParameters.put("events", eventsParameter);
     }
 
-    /* Parse language if supported by the graph type. Only a single
-     * language can be passed. If no language is passed, use "all" as
-     * default. */
-    if (supportedGraphParameters.contains("language")) {
-      String[] languageParameter = (String[]) requestParameters.get(
-          "language");
-      List<String> knownBundles = Arrays.asList(
-          this.knownParameterValues.get("language").split(","));
-      if (languageParameter != null) {
-        if (languageParameter.length != 1 ||
-            languageParameter[0].length() == 0 ||
-            !knownBundles.contains(languageParameter[0])) {
-          return null;
-        }
-      } else {
-        languageParameter = new String[] { "all" };
-      }
-      recognizedGraphParameters.put("language", languageParameter);
-    }
-
     /* Parse torperf data source if supported by the graph type. Only a
      * single source can be passed. If no source is passed, use "torperf"
      * as default. */
@@ -251,23 +233,44 @@ public class GraphParameterChecker {
       recognizedGraphParameters.put("filesize", filesizeParameter);
     }
 
-    /* Parse fingerprint if supported/required by the graph type. Make
-     * sure the fingerprint contains only hexadecimal characters and is 40
-     * characters long. Fail if no fingerprint is provided! */
-    if (supportedGraphParameters.contains("fingerprint")) {
-      String[] fingerprintParameter = (String[]) requestParameters.get(
-          "fingerprint");
-      if (fingerprintParameter == null ||
-          fingerprintParameter.length != 1 ||
-          fingerprintParameter[0] == null ||
-          !Pattern.matches("[0-9a-f]{40}",
-          fingerprintParameter[0].toLowerCase())) {
-        return null;
+    /* Parse transports if supported by the graph type. If no transports
+     * are passed, use "<OR>" as default. */
+    if (supportedGraphParameters.contains("transport")) {
+      String[] transportParameters = (String[]) requestParameters.get(
+          "transport");
+      List<String> knownTransports = Arrays.asList(
+          this.knownParameterValues.get("transport").split(","));
+      if (transportParameters != null) {
+        for (String transport : transportParameters) {
+          if (transport == null || transport.length() == 0 ||
+              !knownTransports.contains(transport)) {
+            return null;
+          }
+        }
+      } else {
+        transportParameters = new String[] { "<OR>" };
+      }
+      recognizedGraphParameters.put("transport", transportParameters);
+    }
+
+    /* Parse versions if supported by the graph type. If no versions
+     * are passed, use "v4" as default. */
+    if (supportedGraphParameters.contains("version")) {
+      String[] versionParameters = (String[]) requestParameters.get(
+          "version");
+      List<String> knownVersions = Arrays.asList(
+          this.knownParameterValues.get("version").split(","));
+      if (versionParameters != null) {
+        for (String version : versionParameters) {
+          if (version == null || version.length() == 0 ||
+              !knownVersions.contains(version)) {
+            return null;
+          }
+        }
       } else {
-        fingerprintParameter[0] = fingerprintParameter[0].toLowerCase();
-        recognizedGraphParameters.put("fingerprint",
-            fingerprintParameter);
+        versionParameters = new String[] { "v4" };
       }
+      recognizedGraphParameters.put("version", versionParameters);
     }
 
     /* We now have a map with all required graph parameters. Return it. */
diff --git a/src/org/torproject/ernie/web/graphs/RObjectGenerator.java b/src/org/torproject/ernie/web/graphs/RObjectGenerator.java
index 6ed77f2..1fb7183 100644
--- a/src/org/torproject/ernie/web/graphs/RObjectGenerator.java
+++ b/src/org/torproject/ernie/web/graphs/RObjectGenerator.java
@@ -105,6 +105,14 @@ public class RObjectGenerator implements ServletContextListener {
     this.availableGraphs.put("connbidirect", "start,end,filename");
     this.availableGraphs.put("fast-exits", "start,end,filename");
     this.availableGraphs.put("almost-fast-exits", "start,end,filename");
+    this.availableGraphs.put("userstats-relay-country",
+        "start,end,country,filename");
+    this.availableGraphs.put("userstats-bridge-country",
+        "start,end,country,filename");
+    this.availableGraphs.put("userstats-bridge-transport",
+        "start,end,transport,filename");
+    this.availableGraphs.put("userstats-bridge-version",
+        "start,end,version,filename");
     this.availableGraphFileTypes = new HashSet<String>(Arrays.asList(
         "png,pdf,svg".split(",")));
     GraphParameterChecker.getInstance().setAvailableGraphs(
diff --git a/web/WEB-INF/users.jsp b/web/WEB-INF/users.jsp
index d75bf7a..d7d17d5 100644
--- a/web/WEB-INF/users.jsp
+++ b/web/WEB-INF/users.jsp
@@ -207,6 +207,173 @@ Tor users (direct and bridge) per month by country.</p>
 <p><a href="csv/monthly-users-average.csv">CSV</a> file containing average
 daily Tor users (direct and bridge) per month by country.</p>
 <br>
+
+<a name="userstats"></a>
+<h3><a href="#userstats" class="anchor">New approach to estimating daily
+Tor users (BETA)</a></h3>
+<br>
+<p>As of April 2013, we are experimenting with a new approach to estimating
+daily Tor users.
+The new approach works very similar to the existing approach to estimate
+directly connecting users, but can also be applied to bridge users.
+This new approach can break down user numbers by country, pluggable
+transport, and IP version.
+See tech report on
+<a href="https://research.torproject.org/techreports/counting-daily-bridge-users-2012-10-24.pdf";>Counting daily bridge users</a>
+and the
+<a href="https://gitweb.torproject.org/metrics-tasks.git/tree/HEAD:/task-8462";>source code</a>
+for details.
+
+<p><font color="red">Note that this approach should be considered
+experimental and absolute numbers should be taken with care!</font></p>
+
+<a name="userstats-relay-country"></a>
+<p><b>Direct users by country (BETA):</b></p>
+
+<font color="red">In contrast to the graphs above, this graph is based on
+requests to directory mirrors <i>and</i> directory authorities.
+That is why the numbers here are higher.
+It's yet to be decided which approach is more correct.</font>
+
+<img src="userstats-relay-country.png${userstats_relay_country_url}"
+     width="576" height="360" alt="Direct users by country graph (BETA)">
+<form action="users.html#userstats-relay-country">
+  <div class="formrow">
+    <input type="hidden" name="graph" value="userstats-relay-country">
+    <p>
+    <label>Start date (yyyy-mm-dd):</label>
+      <input type="text" name="start" size="10"
+             value="<c:choose><c:when test="${fn:length(userstats_relay_country_start) == 0}">${default_start_date}</c:when><c:otherwise>${userstats_relay_country_start[0]}</c:otherwise></c:choose>">
+    <label>End date (yyyy-mm-dd):</label>
+      <input type="text" name="end" size="10"
+             value="<c:choose><c:when test="${fn:length(userstats_relay_country_end) == 0}">${default_end_date}</c:when><c:otherwise>${userstats_relay_country_end[0]}</c:otherwise></c:choose>">
+    </p><p>
+      Source: <select name="country">
+        <option value="all"<c:if test="${userstats_relay_country_country[0] eq 'all'}"> selected</c:if>>All users</option>
+        <c:forEach var="country" items="${countries}" >
+          <option value="${country[0]}"<c:if test="${userstats_relay_country_country[0] eq country[0]}"> selected</c:if>>${country[1]}</option>
+        </c:forEach>
+      </select>
+    </p><p>
+    <input class="submit" type="submit" value="Update graph">
+    </p>
+  </div>
+</form>
+<p>Download graph as
+<a href="userstats-relay-country.pdf${userstats_relay_country_url}">PDF</a> or
+<a href="userstats-relay-country.svg${userstats_relay_country_url}">SVG</a>.</p>
+<hr>
+
+<a name="userstats-bridge-country"></a>
+<p><b>Bridge users by country (BETA):</b></p>
+
+<font color="red">In contrast to the bridge-user graph above, this graph
+uses directory requests to estimate user numbers, not unique IP address sets.
+It's yet to be decided which approach is more correct.</font>
+
+<img src="userstats-bridge-country.png${userstats_bridge_country_url}"
+     width="576" height="360" alt="Bridge users by country graph (BETA)">
+<form action="users.html#userstats-bridge-country">
+  <div class="formrow">
+    <input type="hidden" name="graph" value="userstats-bridge-country">
+    <p>
+    <label>Start date (yyyy-mm-dd):</label>
+      <input type="text" name="start" size="10"
+             value="<c:choose><c:when test="${fn:length(userstats_bridge_country_start) == 0}">${default_start_date}</c:when><c:otherwise>${userstats_bridge_country_start[0]}</c:otherwise></c:choose>">
+    <label>End date (yyyy-mm-dd):</label>
+      <input type="text" name="end" size="10"
+             value="<c:choose><c:when test="${fn:length(userstats_bridge_country_end) == 0}">${default_end_date}</c:when><c:otherwise>${userstats_bridge_country_end[0]}</c:otherwise></c:choose>">
+    </p><p>
+      Source: <select name="country">
+        <option value="all"<c:if test="${userstats_bridge_country_country[0] eq 'all'}"> selected</c:if>>All users</option>
+        <c:forEach var="country" items="${countries}" >
+          <option value="${country[0]}"<c:if test="${userstats_bridge_country_country[0] eq country[0]}"> selected</c:if>>${country[1]}</option>
+        </c:forEach>
+      </select>
+    </p><p>
+    <input class="submit" type="submit" value="Update graph">
+    </p>
+  </div>
+</form>
+<p>Download graph as
+<a href="userstats-bridge-country.pdf${userstats_bridge_country_url}">PDF</a> or
+<a href="userstats-bridge-country.svg${userstats_bridge_country_url}">SVG</a>.</p>
+<hr>
+
+<a name="userstats-bridge-transport"></a>
+<p><b>Bridge users by transport (BETA):</b></p>
+
+<font color="red">Almost none of the currently running bridges report the
+transport name of connecting users, which is why non-OR transport usage is
+so low.
+By default, we consider all users of a bridge OR transport users, unless told
+otherwise.
+Non-OR transport numbers will become more accurate over time.</font>
+
+<img src="userstats-bridge-transport.png${userstats_bridge_transport_url}"
+     width="576" height="360" alt="Bridge users by transport graph (BETA)">
+<form action="users.html#userstats-bridge-transport">
+  <div class="formrow">
+    <input type="hidden" name="graph" value="userstats-bridge-transport">
+    <p>
+    <label>Start date (yyyy-mm-dd):</label>
+      <input type="text" name="start" size="10"
+             value="<c:choose><c:when test="${fn:length(userstats_bridge_transport_start) == 0}">${default_start_date}</c:when><c:otherwise>${userstats_bridge_transport_start[0]}</c:otherwise></c:choose>">
+    <label>End date (yyyy-mm-dd):</label>
+      <input type="text" name="end" size="10"
+             value="<c:choose><c:when test="${fn:length(userstats_bridge_transport_end) == 0}">${default_end_date}</c:when><c:otherwise>${userstats_bridge_transport_end[0]}</c:otherwise></c:choose>">
+    </p><p>
+      Source: <select name="transport">
+        <option value="<OR>"<c:if test="${userstats_bridge_transport_transport[0] eq '<OR>'}"> selected</c:if>>Default OR protocol</option>
+        <option value="obfs2"<c:if test="${userstats_bridge_transport_transport[0] eq 'obfs2'}"> selected</c:if>>obfs2</option>
+        <option value="obfs3"<c:if test="${userstats_bridge_transport_transport[0] eq 'obfs3'}"> selected</c:if>>obfs3</option>
+        <option value="websocket"<c:if test="${userstats_bridge_transport_transport[0] eq 'websocket'}"> selected</c:if>>Flash proxy/websocket</option>
+        <option value="<??>"<c:if test="${userstats_bridge_transport_transport[0] eq '<??>'}"> selected</c:if>>Unknown transport</option>
+      </select>
+    </p><p>
+    <input class="submit" type="submit" value="Update graph">
+    </p>
+  </div>
+</form>
+<p>Download graph as
+<a href="userstats-bridge-transport.pdf${userstats_bridge_transport_url}">PDF</a> or
+<a href="userstats-bridge-transport.svg${userstats_bridge_transport_url}">SVG</a>.</p>
+<hr>
+
+<a name="userstats-bridge-version"></a>
+<p><b>Bridge users by IP version (BETA):</b></p>
+
+<font color="red">Not all of the currently running bridges report the
+IP version of connecting users.
+By default, we consider all users of a bridge IPv4 users, unless told
+otherwise.
+IPv6 numbers will become more accurate over time.</font>
+
+<img src="userstats-bridge-version.png${userstats_bridge_version_url}"
+     width="576" height="360" alt="Bridge users by IP version graph (BETA)">
+<form action="users.html#userstats-bridge-version">
+  <div class="formrow">
+    <input type="hidden" name="graph" value="userstats-bridge-version">
+    <p>
+    <label>Start date (yyyy-mm-dd):</label>
+      <input type="text" name="start" size="10"
+             value="<c:choose><c:when test="${fn:length(userstats_bridge_version_start) == 0}">${default_start_date}</c:when><c:otherwise>${userstats_bridge_version_start[0]}</c:otherwise></c:choose>">
+    <label>End date (yyyy-mm-dd):</label>
+      <input type="text" name="end" size="10"
+             value="<c:choose><c:when test="${fn:length(userstats_bridge_version_end) == 0}">${default_end_date}</c:when><c:otherwise>${userstats_bridge_version_end[0]}</c:otherwise></c:choose>">
+    </p><p>
+      Source: <select name="version">
+        <option value="v4"<c:if test="${userstats_bridge_version_version[0] eq 'v4'}"> selected</c:if>>IPv4</option>
+        <option value="v6"<c:if test="${userstats_bridge_version_version[0] eq 'v6'}"> selected</c:if>>IPv6</option>
+      </select>
+    </p><p>
+    <input class="submit" type="submit" value="Update graph">
+    </p>
+  </div>
+</form>
+<p>Download graph as
+<a href="userstats-bridge-version.pdf${userstats_bridge_version_url}">PDF</a> or
+<a href="userstats-bridge-version.svg${userstats_bridge_version_url}">SVG</a>.</p>
     </div>
   </div>
   <div class="bottom" id="bottom">

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits