[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [metrics-web/master] Add table with top-10 countries to users.html.



commit d2078417458e63ded2bdbb28e061d3b28605f95d
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Thu Jul 28 21:37:49 2011 -0400

    Add table with top-10 countries to users.html.
    
    Implements #3624.
---
 rserve/rserve-init.R                               |    2 +
 rserve/tables.R                                    |   32 ++++++
 .../ernie/web/GraphsSubpagesServlet.java           |   91 +++++++++++++++-
 src/org/torproject/ernie/web/RObjectGenerator.java |   70 +++++++++++-
 .../ernie/web/TableParameterChecker.java           |  117 ++++++++++++++++++++
 web/WEB-INF/users.jsp                              |   30 +++++
 6 files changed, 338 insertions(+), 4 deletions(-)

diff --git a/rserve/rserve-init.R b/rserve/rserve-init.R
index bafb5ba..8344705 100644
--- a/rserve/rserve-init.R
+++ b/rserve/rserve-init.R
@@ -15,3 +15,5 @@ dbpassword= ""
 
 source('graphs.R')
 source('csv.R')
+source('tables.R')
+
diff --git a/rserve/tables.R b/rserve/tables.R
new file mode 100644
index 0000000..725bc31
--- /dev/null
+++ b/rserve/tables.R
@@ -0,0 +1,32 @@
+countrynames <- function(countries) {
+  sapply(countries, countryname)
+}
+
+write_direct_users <- function(start, end, path) {
+  drv <- dbDriver("PostgreSQL")
+  con <- dbConnect(drv, user = dbuser, password = dbpassword, dbname = db)
+  q <- paste("SELECT date, country, r, bwp, brn, bwn, brp, bwr, brr ",
+      "FROM user_stats WHERE date >= '", start, "' AND date <= '", end,
+      "' AND date < (SELECT MAX(date) FROM user_stats) - 1 ",
+      "ORDER BY date, country", sep = "")
+  rs <- dbSendQuery(con, q)
+  u <- fetch(rs, n = -1)
+  dbDisconnect(con)
+  dbUnloadDriver(drv)
+  d <- data.frame(date = u$date, country = u$country,
+       directusers = floor(u$r * (u$bwp * u$brn / u$bwn - u$brp) /
+               (u$bwr * u$brn / u$bwn - u$brr) / 10))
+  d <- aggregate(d$directusers, by = list(country = d$country), mean)
+  total <- d[d$country == "zy", "x"]
+  d <- d[d$country != "zy", ]
+  d <- data.frame(country = d$country, directusers = d$x)
+  d <- d[order(d$directusers, decreasing = TRUE), ]
+  d <- d[1:10, ]
+  d <- data.frame(
+    cc = as.character(d$country),
+    country = sub('the ', '', countrynames(as.character(d$country))),
+    abs = round(d$directusers),
+    rel = round(100 * d$directusers / total, 2))
+  write.csv(d, path, quote = FALSE, row.names = FALSE)
+}
+
diff --git a/src/org/torproject/ernie/web/GraphsSubpagesServlet.java b/src/org/torproject/ernie/web/GraphsSubpagesServlet.java
index f8dfba4..e3845cf 100644
--- a/src/org/torproject/ernie/web/GraphsSubpagesServlet.java
+++ b/src/org/torproject/ernie/web/GraphsSubpagesServlet.java
@@ -12,9 +12,15 @@ public class GraphsSubpagesServlet extends HttpServlet {
    * are forwarded. */
   private Map<String, String> availableGraphsSubpages;
 
+  /* Available tables on graphs subpages. */
+  private Map<String, Set<String>> availableGraphsSubpageTables;
+
   /* Country codes and names for per-country graphs. */
   private List<String[]> knownCountries;
 
+  /* R object generator for generating table data. */
+  private RObjectGenerator rObjectGenerator;
+
   public GraphsSubpagesServlet() {
     this.availableGraphsSubpages = new HashMap<String, String>();
     this.availableGraphsSubpages.put("network.html",
@@ -25,9 +31,21 @@ public class GraphsSubpagesServlet extends HttpServlet {
     this.availableGraphsSubpages.put("performance.html",
         "WEB-INF/performance.jsp");
 
+    this.availableGraphsSubpageTables =
+        new HashMap<String, Set<String>>();
+    this.availableGraphsSubpageTables.put("users.html",
+        new HashSet<String>(Arrays.asList("direct-users".split(","))));
+
     this.knownCountries = Countries.getInstance().getCountryList();
   }
 
+  public void init() {
+    /* Get a reference to the R object generator that we need to generate
+     * table data. */
+    this.rObjectGenerator = (RObjectGenerator) getServletContext().
+        getAttribute("RObjectGenerator");
+  }
+
   public void doGet(HttpServletRequest request,
       HttpServletResponse response) throws IOException, ServletException {
 
@@ -48,8 +66,9 @@ public class GraphsSubpagesServlet extends HttpServlet {
     }
     String jsp = availableGraphsSubpages.get(requestedPage);
 
-    /* Find out which graph type was requested, if any. */
+    /* Find out which graph or table type was requested, if any. */
     String requestedGraph = request.getParameter("graph");
+    String requestedTable = request.getParameter("table");
     if (requestedGraph != null) {
 
       /* Check if the passed parameters are valid. */
@@ -74,6 +93,76 @@ public class GraphsSubpagesServlet extends HttpServlet {
             url);
       }
     }
+    if (requestedTable != null) {
+
+      /* Check if the passed parameters are valid. */
+      Map<String, String[]> checkedParameters = TableParameterChecker.
+          getInstance().checkParameters(requestedTable,
+          request.getParameterMap());
+      if (checkedParameters != null) {
+
+        /* Set the table's attributes to the appropriate values, so that
+         * we can prepopulate the form. */
+        for (Map.Entry<String, String[]> param :
+            checkedParameters.entrySet()) {
+          request.setAttribute(requestedTable.replaceAll("-", "_") + "_"
+              + param.getKey(), param.getValue());
+        }
+      }
+    }
+
+    /* Trigger generation of table data if the graphs subpage has any
+     * tables, regardless of whether a table update was requested. */
+    if (this.availableGraphsSubpageTables.containsKey(requestedPage)) {
+      for (String tableName :
+          this.availableGraphsSubpageTables.get(requestedPage)) {
+
+        Map<String, String[]> checkedParameters = null;
+        if (tableName.equals(requestedTable)) {
+          checkedParameters = TableParameterChecker.
+              getInstance().checkParameters(requestedTable,
+              request.getParameterMap());
+        } else {
+          checkedParameters = TableParameterChecker.
+              getInstance().checkParameters(tableName, null);
+        }
+
+        /* Prepare filename and R query string. */
+        StringBuilder rQueryBuilder = new StringBuilder("write_"
+            + tableName.replaceAll("-", "_") + "("),
+            tableFilenameBuilder = new StringBuilder(tableName);
+
+        for (Map.Entry<String, String[]> parameter :
+            checkedParameters.entrySet()) {
+          String parameterName = parameter.getKey();
+          String[] parameterValues = parameter.getValue();
+          for (String param : parameterValues) {
+            tableFilenameBuilder.append("-" + param);
+          }
+          if (parameterValues.length < 2) {
+            rQueryBuilder.append(parameterName + " = '"
+                + parameterValues[0] + "', ");
+          } else {
+            rQueryBuilder.append(parameterName + " = c(");
+            for (int i = 0; i < parameterValues.length - 1; i++) {
+              rQueryBuilder.append("'" + parameterValues[i] + "', ");
+            }
+            rQueryBuilder.append("'" + parameterValues[
+                parameterValues.length - 1] + "'), ");
+          }
+        }
+        tableFilenameBuilder.append(".tbl");
+        String tableFilename = tableFilenameBuilder.toString();
+        rQueryBuilder.append("path = '%s')");
+        String rQuery = rQueryBuilder.toString();
+
+        /* Generate table data and add it as request attribute. */
+        List<Map<String, String>> tableData = rObjectGenerator.
+            generateTable(rQuery, tableFilename);
+        request.setAttribute(tableName.replaceAll("-", "_")
+              + "_tabledata", tableData);
+      }
+    }
 
     /* Pass list of known countries in case we want to display them. */
     request.setAttribute("countries", this.knownCountries);
diff --git a/src/org/torproject/ernie/web/RObjectGenerator.java b/src/org/torproject/ernie/web/RObjectGenerator.java
index 7f152dd..10aa3b8 100644
--- a/src/org/torproject/ernie/web/RObjectGenerator.java
+++ b/src/org/torproject/ernie/web/RObjectGenerator.java
@@ -55,9 +55,10 @@ public class RObjectGenerator implements ServletContextListener {
     if (!imageFile.exists() || imageFile.lastModified() < now
         - this.maxCacheAge * 1000L) {
 
-      /* We do. Update the R query to contain the absolute path to the file
-       * to be generated, create a connection to Rserve, run the R query,
-       * and close the connection. The generated graph will be on disk. */
+      /* We do. Update the R query to contain the absolute path to the
+       * file to be generated, create a connection to Rserve, run the R
+       * query, and close the connection. The generated graph will be on
+       * disk. */
       rQuery = String.format(rQuery, imageFile.getAbsolutePath());
       try {
         RConnection rc = new RConnection(rserveHost, rservePort);
@@ -135,5 +136,68 @@ public class RObjectGenerator implements ServletContextListener {
     /* Return the csv file. */
     return result;
   }
+
+  /* Generate table data using the given R query and filename or read
+   * previously generated table data from disk if it's not too old and
+   * return table data. */
+  public List<Map<String, String>> generateTable(String rQuery,
+      String tableFilename) {
+
+    /* See if we need to generate this table. */
+    File tableFile = new File(this.cachedGraphsDirectory + "/"
+        + tableFilename);
+    long now = System.currentTimeMillis();
+    if (!tableFile.exists() || tableFile.lastModified() < now
+        - this.maxCacheAge * 1000L) {
+
+      /* We do. Update the R query to contain the absolute path to the
+       * file to be generated, create a connection to Rserve, run the R
+       * query, and close the connection. The generated csv file will be
+       * on disk in the same directory as the generated graphs. */
+      rQuery = String.format(rQuery, tableFile.getAbsolutePath());
+      try {
+        RConnection rc = new RConnection(rserveHost, rservePort);
+        rc.eval(rQuery);
+        rc.close();
+      } catch (RserveException e) {
+        return null;
+      }
+
+      /* Check that we really just generated the file */
+      if (!tableFile.exists() || tableFile.lastModified() < now
+          - this.maxCacheAge * 1000L) {
+        return null;
+      }
+    }
+
+    /* Read the text file from disk and write the table content to a
+     * map. */
+    List<Map<String, String>> result = null;
+    try {
+      result = new ArrayList<Map<String, String>>();
+      BufferedReader br = new BufferedReader(new FileReader(tableFile));
+      String line = br.readLine();
+      if (line != null) {
+        List<String> headers = new ArrayList<String>(Arrays.asList(
+            line.split(",")));
+        while ((line = br.readLine()) != null) {
+          String[] parts = line.split(",");
+          if (headers.size() != parts.length) {
+            return null;
+          }
+          Map<String, String> row = new HashMap<String, String>();
+          for (int i = 0; i < headers.size(); i++) {
+            row.put(headers.get(i), parts[i]);
+          }
+          result.add(row);
+        }
+      }
+    } catch (IOException e) {
+      return null;
+    }
+
+    /* Return table values. */
+    return result;
+  }
 }
 
diff --git a/src/org/torproject/ernie/web/TableParameterChecker.java b/src/org/torproject/ernie/web/TableParameterChecker.java
new file mode 100644
index 0000000..9bf1c33
--- /dev/null
+++ b/src/org/torproject/ernie/web/TableParameterChecker.java
@@ -0,0 +1,117 @@
+package org.torproject.ernie.web;
+
+import java.text.*;
+import java.util.*;
+import java.util.regex.*;
+
+/**
+ * Checks request parameters passed to generate tables.
+ */
+public class TableParameterChecker {
+
+  /**
+   * Singleton instance of this class.
+   */
+  private static TableParameterChecker instance =
+      new TableParameterChecker();
+
+  /**
+   * Returns the singleton instance of this class.
+   */
+  public static TableParameterChecker getInstance() {
+    return instance;
+  }
+
+  /* Date format for parsing start and end dates. */
+  private SimpleDateFormat dateFormat;
+
+  /* Available tables with corresponding parameter lists. */
+  private Map<String, String> availableTables;
+
+  /* Known parameters and parameter values. */
+  private Map<String, String> knownParameterValues;
+
+  /**
+   * Initializes map with valid parameters for each of the graphs.
+   */
+  public TableParameterChecker() {
+    this.dateFormat = new SimpleDateFormat("yyyy-MM-dd");
+    this.dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+
+    this.availableTables = new HashMap<String, String>();
+    this.availableTables.put("direct-users", "start,end,filename");
+
+    this.knownParameterValues = new HashMap<String, String>();
+  }
+
+  /**
+   * Checks request parameters for the given table type and returns a map
+   * of recognized parameters, or null if the table type doesn't exist or
+   * the parameters are invalid.
+   */
+  public Map<String, String[]> checkParameters(String tableType,
+      Map requestParameters) {
+
+    /* Check if the graph type exists. */
+    if (tableType == null ||
+        !this.availableTables.containsKey(tableType)) {
+      return null;
+    }
+
+    /* Find out which other parameters are supported by this table type
+     * and parse them if they are given. */
+    Set<String> supportedTableParameters = new HashSet<String>(Arrays.
+        asList(this.availableTables.get(tableType).split(",")));
+    Map<String, String[]> recognizedTableParameters =
+        new HashMap<String, String[]>();
+
+    /* Parse start and end dates if supported by the table type. If no end
+     * date is provided, set it to today. If no start date is provided,
+     * set it to 90 days before the end date. Make sure that start date
+     * precedes end date. */
+    if (supportedTableParameters.contains("start") ||
+        supportedTableParameters.contains("end")) {
+      String[] startParameter = null;
+      String[] endParameter = null;
+      if (requestParameters != null) {
+        startParameter = (String[]) requestParameters.get("start");
+        endParameter = (String[]) requestParameters.get("end");
+      }
+      long endTimestamp = System.currentTimeMillis();
+      if (endParameter != null && endParameter.length > 0 &&
+          endParameter[0].length() > 0) {
+        try {
+          endTimestamp = dateFormat.parse(endParameter[0]).getTime();
+        } catch (ParseException e)  {
+          return null;
+        }
+        if (!endParameter[0].startsWith("20")) {
+          return null;
+        }
+      }
+      endParameter = new String[] { dateFormat.format(endTimestamp) };
+      long startTimestamp = endTimestamp - 90L * 24L * 60L * 60L * 1000L;
+      if (startParameter != null && startParameter.length > 0 &&
+          startParameter[0].length() > 0) {
+        try {
+          startTimestamp = dateFormat.parse(startParameter[0]).getTime();
+        } catch (ParseException e)  {
+          return null;
+        }
+        if (!startParameter[0].startsWith("20")) {
+          return null;
+        }
+      }
+      startParameter = new String[] { dateFormat.format(startTimestamp) };
+      if (startTimestamp > endTimestamp) {
+       return null;
+      }
+      recognizedTableParameters.put("start", startParameter);
+      recognizedTableParameters.put("end", endParameter);
+    }
+
+    /* We now have a map with all required table parameters. Return it. */
+    return recognizedTableParameters;
+  }
+}
+
diff --git a/web/WEB-INF/users.jsp b/web/WEB-INF/users.jsp
index 814c15c..b378af1 100644
--- a/web/WEB-INF/users.jsp
+++ b/web/WEB-INF/users.jsp
@@ -1,5 +1,6 @@
 <%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core"; %>
 <%@ taglib prefix="fn" uri="http://java.sun.com/jsp/jstl/functions"; %>
+<%@ taglib prefix="fmt" uri="http://java.sun.com/jsp/jstl/fmt"; %>
 <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
 <html>
 <head>
@@ -58,6 +59,35 @@ based on the requests seen by a few dozen directory mirrors.</p>
     </p>
   </div>
 </form>
+<a name="direct-users-table"></a>
+<table>
+  <tr>
+    <th>Country</th>
+    <th>Mean daily users</th>
+  </tr>
+  <c:forEach var="row" items="${direct_users_tabledata}">
+    <tr>
+      <td><a href="users.html?graph=direct-users&country=${row['cc']}#direct-users">${row['country']}</a>&emsp;</td>
+      <td>${row['abs']} (<fmt:formatNumber type="number" minFractionDigits="2" value="${row['rel']}" /> %)</td>
+    </tr>
+  </c:forEach>
+</table>
+<br>
+<form action="users.html#direct-users-table">
+  <div class="formrow">
+    <input type="hidden" name="table" value="direct-users">
+    <p>
+    <label>Start date (yyyy-mm-dd):</label>
+      <input type="text" name="start" size="10"
+             value="<c:choose><c:when test="${fn:length(direct_users_start) == 0}">${default_start_date}</c:when><c:otherwise>${direct_users_start[0]}</c:otherwise></c:choose>">
+    <label>End date (yyyy-mm-dd):</label>
+      <input type="text" name="end" size="10"
+             value="<c:choose><c:when test="${fn:length(direct_users_end) == 0}">${default_end_date}</c:when><c:otherwise>${direct_users_end[0]}</c:otherwise></c:choose>">
+    </p><p>
+    <input class="submit" type="submit" value="Update table">
+    </p>
+  </div>
+</form>
 <p><a href="csv/direct-users.csv">CSV</a> file containing all data.</p>
 <p><a href="csv/monthly-users-peak.csv">CSV</a> file containing peak daily
 Tor users (direct and bridge) per month by country.</p>

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits