[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [metrics-web/master] Import GetTor stats into database.



commit e0934bf15e83c8020351c890b6efc7c78f4e3492
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Thu Mar 3 18:24:19 2011 +0100

    Import GetTor stats into database.
---
 config.template                                    |    6 +
 src/org/torproject/ernie/cron/Configuration.java   |   13 ++
 src/org/torproject/ernie/cron/GetTorProcessor.java |  124 ++++++++++++++++++++
 src/org/torproject/ernie/cron/Main.java            |    7 +
 4 files changed, 150 insertions(+), 0 deletions(-)

diff --git a/config.template b/config.template
index 4a5588a..fd37f4d 100644
--- a/config.template
+++ b/config.template
@@ -48,4 +48,10 @@
 #
 ## Relative path to directory to import torperf results from
 #TorperfDirectory torperf/
+#
+## Process GetTor stats and import them into the database
+#ProcessGetTorStats 0
+#
+## Relative path to directory where to find GetTor stats
+#GetTorDirectory gettor/
 
diff --git a/src/org/torproject/ernie/cron/Configuration.java b/src/org/torproject/ernie/cron/Configuration.java
index dd09bae..e2c630b 100644
--- a/src/org/torproject/ernie/cron/Configuration.java
+++ b/src/org/torproject/ernie/cron/Configuration.java
@@ -28,6 +28,8 @@ public class Configuration {
   private boolean writeBridgeStats = false;
   private boolean importWriteTorperfStats = false;
   private String torperfDirectory = "torperf/";
+  private boolean processGetTorStats = false;
+  private String getTorDirectory = "gettor/";
   public Configuration() {
 
     /* Initialize logger. */
@@ -82,6 +84,11 @@ public class Configuration {
               line.split(" ")[1]) != 0;
         } else if (line.startsWith("TorperfDirectory")) {
           this.torperfDirectory = line.split(" ")[1];
+        } else if (line.startsWith("ProcessGetTorStats")) {
+          this.processGetTorStats = Integer.parseInt(
+              line.split(" ")[1]) != 0;
+        } else if (line.startsWith("GetTorDirectory")) {
+          this.getTorDirectory = line.split(" ")[1];
         } else {
           logger.severe("Configuration file contains unrecognized "
               + "configuration key in line '" + line + "'! Exiting!");
@@ -149,5 +156,11 @@ public class Configuration {
   public String getTorperfDirectory() {
     return this.torperfDirectory;
   }
+  public boolean getProcessGetTorStats() {
+    return this.processGetTorStats;
+  }
+  public String getGetTorDirectory() {
+    return this.getTorDirectory;
+  }
 }
 
diff --git a/src/org/torproject/ernie/cron/GetTorProcessor.java b/src/org/torproject/ernie/cron/GetTorProcessor.java
new file mode 100644
index 0000000..a6ddf11
--- /dev/null
+++ b/src/org/torproject/ernie/cron/GetTorProcessor.java
@@ -0,0 +1,124 @@
+/* Copyright 2011 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.cron;
+
+import java.io.*;
+import java.sql.*;
+import java.util.*;
+import java.util.logging.*;
+
+public class GetTorProcessor {
+  public GetTorProcessor(File getTorDirectory, String connectionURL) {
+
+    Logger logger = Logger.getLogger(GetTorProcessor.class.getName());
+
+    /* Parse stats file. */
+    File getTorFile = new File(getTorDirectory, "gettor_stats.txt");
+    if (!getTorFile.exists() || getTorFile.isDirectory()) {
+      logger.warning("Could not read GetTor stats");
+      return;
+    }
+    SortedSet<String> columns = new TreeSet<String>();
+    SortedMap<String, Map<String, Integer>> data =
+        new TreeMap<String, Map<String, Integer>>();
+    try {
+      logger.fine("Parsing GetTor stats...");
+      BufferedReader br = new BufferedReader(new FileReader(getTorFile));
+      String line = null;
+      while ((line = br.readLine()) != null) {
+        String[] parts = line.split(" ");
+        String date = parts[0];
+        Map<String, Integer> obs = new HashMap<String, Integer>();
+        data.put(date, obs);
+        for (int i = 2; i < parts.length; i++) {
+          String key = parts[i].split(":")[0].toLowerCase();
+          Integer value = new Integer(parts[i].split(":")[1]);
+          columns.add(key);
+          obs.put(key, value);
+        }
+      }
+      br.close();
+    } catch (IOException e) {
+      logger.log(Level.WARNING, "Failed parsing GetTor stats!", e);
+      return;
+    } catch (NumberFormatException e) {
+      logger.log(Level.WARNING, "Failed parsing GetTor stats!", e);
+      return;
+    }
+
+    /* Write results to database. */
+    if (connectionURL != null) {
+      try {
+        Map<String, Integer> updateRows = new HashMap<String, Integer>(),
+            insertRows = new HashMap<String, Integer>();
+        for (Map.Entry<String, Map<String, Integer>> e :
+            data.entrySet()) {
+          String date = e.getKey();
+          Map<String, Integer> obs = e.getValue();
+          for (String column : columns) {
+            if (obs.containsKey(column)) {
+              Integer value = obs.get(column);
+              String key = date + "," + column;
+              insertRows.put(key, value);
+            }
+          }
+        }
+        Connection conn = DriverManager.getConnection(connectionURL);
+        PreparedStatement psI = conn.prepareStatement(
+            "INSERT INTO gettor_stats (downloads, date, bundle) "
+            + "VALUES (?, ?, ?)");
+        PreparedStatement psU = conn.prepareStatement(
+            "UPDATE gettor_stats SET downloads = ? "
+            + "WHERE date = ? AND bundle = ?");
+        conn.setAutoCommit(false);
+        Statement statement = conn.createStatement();
+        ResultSet rs = statement.executeQuery(
+            "SELECT date, bundle, downloads FROM gettor_stats");
+        while (rs.next()) {
+          String date = rs.getDate(1).toString();
+          String bundle = rs.getString(2);
+          String key = date + "," + bundle;
+          if (insertRows.containsKey(key)) {
+            int insertRow = insertRows.remove(key);
+            int oldCount = rs.getInt(3);
+            if (insertRow != oldCount) {
+              updateRows.put(key, insertRow);
+            }
+          }
+        }
+        for (Map.Entry<String, Integer> e : updateRows.entrySet()) {
+          String[] keyParts = e.getKey().split(",");
+          java.sql.Date date = java.sql.Date.valueOf(keyParts[0]);
+          String bundle = keyParts[1];
+          int downloads = e.getValue();
+          psU.clearParameters();
+          psU.setLong(1, downloads);
+          psU.setDate(2, date);
+          psU.setString(3, bundle);
+          psU.executeUpdate();
+        }
+        for (Map.Entry<String, Integer> e : insertRows.entrySet()) {
+          String[] keyParts = e.getKey().split(",");
+          java.sql.Date date = java.sql.Date.valueOf(keyParts[0]);
+          String bundle = keyParts[1];
+          int downloads = e.getValue();
+          psI.clearParameters();
+          psI.setLong(1, downloads);
+          psI.setDate(2, date);
+          psI.setString(3, bundle);
+          psI.executeUpdate();
+        }
+        conn.commit();
+        conn.close();
+      } catch (SQLException e) {
+        logger.log(Level.WARNING, "Failed to add GetTor stats to "
+            + "database.", e);
+      }
+    }
+
+    logger.info("Finished processing statistics on Tor packages "
+        + "delivered by GetTor.\nLast date in statistics is "
+        + data.lastKey() + ".");
+  }
+}
+
diff --git a/src/org/torproject/ernie/cron/Main.java b/src/org/torproject/ernie/cron/Main.java
index 943d326..ec02130 100644
--- a/src/org/torproject/ernie/cron/Main.java
+++ b/src/org/torproject/ernie/cron/Main.java
@@ -111,6 +111,13 @@ public class Main {
           statsDirectory, config.getRelayDescriptorDatabaseJDBC());
     }
 
+    // Download and process GetTor stats
+    if (config.getProcessGetTorStats()) {
+      new GetTorProcessor(
+          new File(config.getGetTorDirectory()),
+          config.getRelayDescriptorDatabaseJDBC());
+    }
+
     // Remove lock file
     lf.releaseLock();
 

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits