[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [metrics-db/master] Remove GetTor stats database import.
commit f67c7db477740b4f13a949b32c1fec0b5a72c464
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Thu Mar 3 18:25:31 2011 +0100
Remove GetTor stats database import.
---
config.template | 15 +--
src/org/torproject/ernie/db/Configuration.java | 39 ++----
src/org/torproject/ernie/db/GetTorDownloader.java | 101 +++++++++++++++
src/org/torproject/ernie/db/GetTorProcessor.java | 143 ---------------------
src/org/torproject/ernie/db/Main.java | 7 +-
5 files changed, 122 insertions(+), 183 deletions(-)
diff --git a/config.template b/config.template
index 72cc411..0d6743e 100644
--- a/config.template
+++ b/config.template
@@ -32,12 +32,15 @@
## Relative path to directory to import bridge descriptor snapshots from
#BridgeSnapshotsDirectory bridge-directories/
#
-## Download and process GetTor stats
-#DownloadProcessGetTorStats 0
+## Download GetTor stats
+#DownloadGetTorStats 0
#
## URL to download GetTor stats from
#GetTorStatsURL http://gettor.torproject.org:8080/~gettor/gettor_stats.txt
#
+## Relative path to directory to store GetTor stats in
+#GetTorDirectory gettor/
+#
## Download exit list and store it to disk
#DownloadExitList 0
#
@@ -49,14 +52,6 @@
## Relative path to directory to write directory archives to
#DirectoryArchivesOutputDirectory directory-archive/
#
-## Write aggregate statistics (bridges and bridge users per day, directory
-## clients per day, torperf results, packages requested from GetTor, etc.)
-## to database for later evaluation
-#WriteAggregateStatsDatabase 0
-#
-## JDBC string for relay descriptor database
-#RelayDescriptorDatabaseJDBC jdbc:postgresql://localhost/tordir?user=ernie&password=password
-#
## Write sanitized bridges to disk
#WriteSanitizedBridges 0
#
diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java
index 4d75b59..c1cdbea 100644
--- a/src/org/torproject/ernie/db/Configuration.java
+++ b/src/org/torproject/ernie/db/Configuration.java
@@ -21,7 +21,6 @@ public class Configuration {
private boolean importDirectoryArchives = false;
private String directoryArchivesDirectory = "archives/";
private boolean keepDirectoryArchiveImportHistory = false;
- private boolean writeAggregateStatsDatabase = false;
private String relayDescriptorDatabaseJdbc =
"jdbc:postgresql://localhost/tordir?user=ernie&password=password";
private boolean writeSanitizedBridges = false;
@@ -33,9 +32,10 @@ public class Configuration {
private boolean downloadRelayDescriptors = false;
private List<String> downloadFromDirectoryAuthorities = Arrays.asList(
"86.59.21.38,194.109.206.212,80.190.246.100:8180".split(","));
- private boolean downloadProcessGetTorStats = false;
+ private boolean downloadGetTorStats = false;
private String getTorStatsUrl = "http://gettor.torproject.org:8080/";
+ "~gettor/gettor_stats.txt";
+ private String getTorDirectory = "gettor/";
private boolean downloadExitList = false;
public Configuration() {
@@ -82,11 +82,6 @@ public class Configuration {
} else if (line.startsWith("KeepDirectoryArchiveImportHistory")) {
this.keepDirectoryArchiveImportHistory = Integer.parseInt(
line.split(" ")[1]) != 0;
- } else if (line.startsWith("WriteAggregateStatsDatabase")) {
- this.writeAggregateStatsDatabase = Integer.parseInt(
- line.split(" ")[1]) != 0;
- } else if (line.startsWith("RelayDescriptorDatabaseJDBC")) {
- this.relayDescriptorDatabaseJdbc = line.split(" ")[1];
} else if (line.startsWith("WriteSanitizedBridges")) {
this.writeSanitizedBridges = Integer.parseInt(
line.split(" ")[1]) != 0;
@@ -119,14 +114,16 @@ public class Configuration {
new URL("http://"; + dir + "/");
this.downloadFromDirectoryAuthorities.add(dir);
}
- } else if (line.startsWith("DownloadProcessGetTorStats")) {
- this.downloadProcessGetTorStats = Integer.parseInt(
+ } else if (line.startsWith("DownloadGetTorStats")) {
+ this.downloadGetTorStats = Integer.parseInt(
line.split(" ")[1]) != 0;
} else if (line.startsWith("GetTorStatsURL")) {
String newUrl = line.split(" ")[1];
/* Test if URL has correct format. */
new URL(newUrl);
this.getTorStatsUrl = newUrl;
+ } else if (line.startsWith("GetTorDirectory")) {
+ this.getTorDirectory = line.split(" ")[1];
} else if (line.startsWith("DownloadExitList")) {
this.downloadExitList = Integer.parseInt(
line.split(" ")[1]) != 0;
@@ -158,9 +155,8 @@ public class Configuration {
/** Make some checks if configuration is valid. */
if (!this.importCachedRelayDescriptors &&
!this.importDirectoryArchives && !this.downloadRelayDescriptors &&
- !this.importBridgeSnapshots && !this.downloadProcessGetTorStats &&
+ !this.importBridgeSnapshots && !this.downloadGetTorStats &&
!this.downloadExitList && !this.writeDirectoryArchives &&
- !this.writeAggregateStatsDatabase &&
!this.writeSanitizedBridges) {
logger.warning("We have not been configured to read data from any "
+ "data source or write data to any data sink. You need to "
@@ -182,8 +178,7 @@ public class Configuration {
+ "least one data sink, but we don't have a single data source "
+ "containing relay descriptors.");
}
- if (this.importBridgeSnapshots && !(this.writeSanitizedBridges ||
- this.writeAggregateStatsDatabase)) {
+ if (this.importBridgeSnapshots && !this.writeSanitizedBridges) {
logger.warning("We are configured to import/download bridge "
+ "descriptors, but we don't have a single data sink to write "
+ "bridge descriptors to.");
@@ -193,11 +188,6 @@ public class Configuration {
+ "least one data sink, but we don't have a single data source "
+ "containing bridge descriptors.");
}
- if (this.downloadProcessGetTorStats &&
- !this.writeAggregateStatsDatabase) {
- logger.warning("We are configured to download GetTor statistics, "
- + "but not to import them into the database.");
- }
}
public boolean getWriteDirectoryArchives() {
return this.writeDirectoryArchives;
@@ -220,12 +210,6 @@ public class Configuration {
public boolean getKeepDirectoryArchiveImportHistory() {
return this.keepDirectoryArchiveImportHistory;
}
- public boolean getWriteAggregateStatsDatabase() {
- return this.writeAggregateStatsDatabase;
- }
- public String getRelayDescriptorDatabaseJDBC() {
- return this.relayDescriptorDatabaseJdbc;
- }
public boolean getWriteSanitizedBridges() {
return this.writeSanitizedBridges;
}
@@ -250,12 +234,15 @@ public class Configuration {
public List<String> getDownloadFromDirectoryAuthorities() {
return this.downloadFromDirectoryAuthorities;
}
- public boolean getDownloadProcessGetTorStats() {
- return this.downloadProcessGetTorStats;
+ public boolean getDownloadGetTorStats() {
+ return this.downloadGetTorStats;
}
public String getGetTorStatsUrl() {
return this.getTorStatsUrl;
}
+ public String getGetTorDirectory() {
+ return this.getTorDirectory;
+ }
public boolean getDownloadExitList() {
return this.downloadExitList;
}
diff --git a/src/org/torproject/ernie/db/GetTorDownloader.java b/src/org/torproject/ernie/db/GetTorDownloader.java
new file mode 100644
index 0000000..91e9f5f
--- /dev/null
+++ b/src/org/torproject/ernie/db/GetTorDownloader.java
@@ -0,0 +1,101 @@
+/* Copyright 2010 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.db;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.logging.*;
+
+public class GetTorDownloader {
+
+ public GetTorDownloader(String gettorStatsUrl, File getTorDirectory) {
+
+ Logger logger = Logger.getLogger(GetTorDownloader.class.getName());
+
+ File getTorFile = new File(getTorDirectory, "gettor_stats.txt");
+ SortedMap<String, String> getTorStats = new TreeMap<String, String>();
+
+ if (getTorFile.exists() && !getTorFile.isDirectory()) {
+ try {
+ logger.fine("Reading local gettor_stats.txt file...");
+ BufferedReader br = new BufferedReader(new FileReader(
+ getTorFile));
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ String date = line.split(" ")[0];
+ getTorStats.put(date, line);
+ }
+ br.close();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Failed parsing local GetTor stats!",
+ e);
+ return;
+ }
+ }
+
+ String unparsed = null;
+ try {
+ logger.fine("Downloading GetTor stats...");
+ URL u = new URL(gettorStatsUrl);
+ HttpURLConnection huc = (HttpURLConnection) u.openConnection();
+ huc.setRequestMethod("GET");
+ huc.connect();
+ int response = huc.getResponseCode();
+ if (response == 200) {
+ BufferedInputStream in = new BufferedInputStream(
+ huc.getInputStream());
+ StringBuilder sb = new StringBuilder();
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = in.read(data, 0, 1024)) >= 0) {
+ sb.append(new String(data, 0, len));
+ }
+ in.close();
+ unparsed = sb.toString();
+ }
+ logger.fine("Finished downloading GetTor stats.");
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Failed downloading GetTor stats", e);
+ return;
+ }
+
+ try {
+ logger.fine("Parsing downloaded GetTor stats...");
+ BufferedReader br = new BufferedReader(new StringReader(unparsed));
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ String date = line.split(" ")[0];
+ getTorStats.put(date, line);
+ }
+ br.close();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Failed parsing downloaded GetTor stats!",
+ e);
+ return;
+ }
+
+ try {
+ logger.fine("Writing GetTor stats to local gettor_stats.txt "
+ + "file...");
+ if (!getTorDirectory.exists()) {
+ getTorDirectory.mkdirs();
+ }
+ BufferedWriter bw = new BufferedWriter(new FileWriter(getTorFile));
+ for (String line : getTorStats.values()) {
+ bw.write(line + "\n");
+ }
+ bw.close();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Failed writing GetTor stats to local "
+ + "gettor_stats.txt file", e);
+ return;
+ }
+
+ logger.info("Finished downloading and processing statistics on Tor "
+ + "packages delivered by GetTor.\nDownloaded " + unparsed.length()
+ + " bytes. Last date in statistics is " + getTorStats.lastKey()
+ + ".");
+ }
+}
+
diff --git a/src/org/torproject/ernie/db/GetTorProcessor.java b/src/org/torproject/ernie/db/GetTorProcessor.java
deleted file mode 100644
index 6d9a935..0000000
--- a/src/org/torproject/ernie/db/GetTorProcessor.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/* Copyright 2010 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.ernie.db;
-
-import java.io.*;
-import java.net.*;
-import java.sql.*;
-import java.util.*;
-import java.util.logging.*;
-
-public class GetTorProcessor {
- public GetTorProcessor(String gettorStatsUrl, String connectionURL) {
- Logger logger = Logger.getLogger(GetTorProcessor.class.getName());
- String unparsed = null;
- try {
- logger.fine("Downloading gettor stats...");
- URL u = new URL(gettorStatsUrl);
- HttpURLConnection huc = (HttpURLConnection) u.openConnection();
- huc.setRequestMethod("GET");
- huc.connect();
- int response = huc.getResponseCode();
- if (response == 200) {
- BufferedInputStream in = new BufferedInputStream(
- huc.getInputStream());
- StringBuilder sb = new StringBuilder();
- int len;
- byte[] data = new byte[1024];
- while ((len = in.read(data, 0, 1024)) >= 0) {
- sb.append(new String(data, 0, len));
- }
- in.close();
- unparsed = sb.toString();
- }
- logger.fine("Finished downloading gettor stats.");
- } catch (IOException e) {
- logger.log(Level.WARNING, "Failed downloading gettor stats", e);
- return;
- }
-
- SortedSet<String> columns = new TreeSet<String>();
- SortedMap<String, Map<String, Integer>> data =
- new TreeMap<String, Map<String, Integer>>();
- try {
- logger.fine("Parsing downloaded gettor stats...");
- BufferedReader br = new BufferedReader(new StringReader(unparsed));
- String line = null;
- while ((line = br.readLine()) != null) {
- String[] parts = line.split(" ");
- String date = parts[0];
- Map<String, Integer> obs = new HashMap<String, Integer>();
- data.put(date, obs);
- for (int i = 2; i < parts.length; i++) {
- String key = parts[i].split(":")[0].toLowerCase();
- Integer value = new Integer(parts[i].split(":")[1]);
- columns.add(key);
- obs.put(key, value);
- }
- }
- br.close();
- } catch (IOException e) {
- logger.log(Level.WARNING, "Failed parsing gettor stats!", e);
- return;
- } catch (NumberFormatException e) {
- logger.log(Level.WARNING, "Failed parsing gettor stats!", e);
- return;
- }
-
- /* Write results to database. */
- if (connectionURL != null) {
- try {
- Map<String, Integer> updateRows = new HashMap<String, Integer>(),
- insertRows = new HashMap<String, Integer>();
- for (Map.Entry<String, Map<String, Integer>> e :
- data.entrySet()) {
- String date = e.getKey();
- Map<String, Integer> obs = e.getValue();
- for (String column : columns) {
- if (obs.containsKey(column)) {
- Integer value = obs.get(column);
- String key = date + "," + column;
- insertRows.put(key, value);
- }
- }
- }
- Connection conn = DriverManager.getConnection(connectionURL);
- PreparedStatement psI = conn.prepareStatement(
- "INSERT INTO gettor_stats (downloads, date, bundle) "
- + "VALUES (?, ?, ?)");
- PreparedStatement psU = conn.prepareStatement(
- "UPDATE gettor_stats SET downloads = ? "
- + "WHERE date = ? AND bundle = ?");
- conn.setAutoCommit(false);
- Statement statement = conn.createStatement();
- ResultSet rs = statement.executeQuery(
- "SELECT date, bundle, downloads FROM gettor_stats");
- while (rs.next()) {
- String date = rs.getDate(1).toString();
- String bundle = rs.getString(2);
- String key = date + "," + bundle;
- if (insertRows.containsKey(key)) {
- int insertRow = insertRows.remove(key);
- int oldCount = rs.getInt(3);
- if (insertRow != oldCount) {
- updateRows.put(key, insertRow);
- }
- }
- }
- for (Map.Entry<String, Integer> e : updateRows.entrySet()) {
- String[] keyParts = e.getKey().split(",");
- java.sql.Date date = java.sql.Date.valueOf(keyParts[0]);
- String bundle = keyParts[1];
- int downloads = e.getValue();
- psU.clearParameters();
- psU.setLong(1, downloads);
- psU.setDate(2, date);
- psU.setString(3, bundle);
- psU.executeUpdate();
- }
- for (Map.Entry<String, Integer> e : insertRows.entrySet()) {
- String[] keyParts = e.getKey().split(",");
- java.sql.Date date = java.sql.Date.valueOf(keyParts[0]);
- String bundle = keyParts[1];
- int downloads = e.getValue();
- psI.clearParameters();
- psI.setLong(1, downloads);
- psI.setDate(2, date);
- psI.setString(3, bundle);
- psI.executeUpdate();
- }
- conn.commit();
- conn.close();
- } catch (SQLException e) {
- logger.log(Level.WARNING, "Failed to add GetTor stats to "
- + "database.", e);
- }
- }
-
- logger.info("Finished downloading and processing statistics on Tor "
- + "packages delivered by GetTor.\nDownloaded " + unparsed.length()
- + " bytes. Last date in statistics is " + data.lastKey() + ".");
- }
-}
-
diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java
index e10b6c7..657cdfc 100644
--- a/src/org/torproject/ernie/db/Main.java
+++ b/src/org/torproject/ernie/db/Main.java
@@ -118,10 +118,9 @@ public class Main {
}
// Download and process GetTor stats
- if (config.getDownloadProcessGetTorStats()) {
- new GetTorProcessor(config.getGetTorStatsUrl(),
- config.getWriteAggregateStatsDatabase() ?
- config.getRelayDescriptorDatabaseJDBC() : null);
+ if (config.getDownloadGetTorStats()) {
+ new GetTorDownloader(config.getGetTorStatsUrl(),
+ new File(config.getGetTorDirectory()));
}
// Download exit list and store it to disk
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits