[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [metrics-db/master] Download and merge possibly truncated Torperf files.



commit 138b3397ec300ba64ac48a13175435f7673d65fd
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Tue Mar 6 11:53:39 2012 +0100

    Download and merge possibly truncated Torperf files.
---
 .gitignore                                         |    3 +
 config.template                                    |   21 ++++
 src/org/torproject/ernie/db/Configuration.java     |   58 ++++++++++-
 src/org/torproject/ernie/db/Main.java              |    7 +
 src/org/torproject/ernie/db/TorperfDownloader.java |  116 ++++++++++++++++++++
 5 files changed, 204 insertions(+), 1 deletions(-)

diff --git a/.gitignore b/.gitignore
index d5a5c14..e119284 100644
--- a/.gitignore
+++ b/.gitignore
@@ -47,6 +47,9 @@ config
 # Gettor files
 gettor/
 
+# Torperf files
+torperf/
+
 # Files provided via rsync
 rsync/
 
diff --git a/config.template b/config.template
index 6cfdce5..62ff828 100644
--- a/config.template
+++ b/config.template
@@ -122,4 +122,25 @@
 #
 ## Relative path to the directory that shall be provided via rsync
 #RsyncDirectory rsync/
+#
+#
+######## Torperf downloader ########
+#
+## Download and merge Torperf .data and .extradata files
+#ProcessTorperfFiles 0
+#
+## Relative path to the directory to store Torperf files in
+#TorperfOutputDirectory torperf/
+#
+## Torperf source names and base URLs (option can be contained multiple
+## times)
+#TorperfSource torperf http://torperf.torproject.org/
+#
+## Torperf .data files available on a given source (option can be
+## contained multiple times)
+#TorperfDataFiles torperf 50kb.data 1mb.data 5mb.data
+#
+## Torperf .extradata files available on a given source (option can be
+## contained multiple times)
+#TorperfExtradataFiles torperf 50kb.extradata 1mb.extradata 5mb.extradata
 
diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java
index 7197c7d..91f00b1 100644
--- a/src/org/torproject/ernie/db/Configuration.java
+++ b/src/org/torproject/ernie/db/Configuration.java
@@ -49,6 +49,11 @@ public class Configuration {
   private boolean processBridgePoolAssignments = false;
   private String assignmentsDirectory = "assignments/";
   private String sanitizedAssignmentsDirectory = "sanitized-assignments/";
+  private boolean processTorperfFiles = false;
+  private String torperfOutputDirectory = "torperf/";
+  private SortedMap<String, String> torperfSources = null;
+  private SortedMap<String, List<String>> torperfDataFiles = null;
+  private SortedMap<String, List<String>> torperfExtradataFiles = null;
   private boolean provideFilesViaRsync = false;
   private String rsyncDirectory = "rsync";
   public Configuration() {
@@ -170,6 +175,41 @@ public class Configuration {
           this.assignmentsDirectory = line.split(" ")[1];
         } else if (line.startsWith("SanitizedAssignmentsDirectory")) {
           this.sanitizedAssignmentsDirectory = line.split(" ")[1];
+        } else if (line.startsWith("ProcessTorperfFiles")) {
+          this.processTorperfFiles = Integer.parseInt(line.split(" ")[1])
+              != 0;
+        } else if (line.startsWith("TorperfOutputDirectory")) {
+        } else if (line.startsWith("TorperfSource")) {
+          if (this.torperfSources == null) {
+            this.torperfSources = new TreeMap<String, String>();
+          }
+          String[] parts = line.split(" ");
+          String sourceName = parts[1];
+          String baseUrl = parts[2];
+          this.torperfSources.put(sourceName, baseUrl);
+        } else if (line.startsWith("TorperfDataFiles")) {
+          if (this.torperfDataFiles == null) {
+            this.torperfDataFiles = new TreeMap<String, List<String>>();
+          }
+          String[] parts = line.split(" ");
+          String sourceName = parts[1];
+          List<String> dataFiles = new ArrayList<String>();
+          for (int i = 2; i < parts.length; i++) {
+            dataFiles.add(parts[i]);
+          }
+          this.torperfDataFiles.put(sourceName, dataFiles);
+        } else if (line.startsWith("TorperfExtradataFiles")) {
+          if (this.torperfExtradataFiles == null) {
+            this.torperfExtradataFiles =
+                new TreeMap<String, List<String>>();
+          }
+          String[] parts = line.split(" ");
+          String sourceName = parts[1];
+          List<String> extradataFiles = new ArrayList<String>();
+          for (int i = 2; i < parts.length; i++) {
+            extradataFiles.add(parts[i]);
+          }
+          this.torperfExtradataFiles.put(sourceName, extradataFiles);
         } else if (line.startsWith("ProvideFilesViaRsync")) {
           this.provideFilesViaRsync = Integer.parseInt(
               line.split(" ")[1]) != 0;
@@ -205,7 +245,8 @@ public class Configuration {
         !this.importDirectoryArchives && !this.downloadRelayDescriptors &&
         !this.importBridgeSnapshots && !this.downloadGetTorStats &&
         !this.downloadExitList && !this.processBridgePoolAssignments &&
-        !this.writeDirectoryArchives && !this.writeSanitizedBridges) {
+        !this.writeDirectoryArchives && !this.writeSanitizedBridges &&
+        !this.processTorperfFiles) {
       logger.warning("We have not been configured to read data from any "
           + "data source or write data to any data sink. You need to "
           + "edit your config file (" + configFile.getAbsolutePath()
@@ -324,6 +365,21 @@ public class Configuration {
   public String getSanitizedAssignmentsDirectory() {
     return sanitizedAssignmentsDirectory;
   }
+  public boolean getProcessTorperfFiles() {
+    return this.processTorperfFiles;
+  }
+  public String getTorperfOutputDirectory() {
+    return this.torperfOutputDirectory;
+  }
+  public SortedMap<String, String> getTorperfSources() {
+    return this.torperfSources;
+  }
+  public SortedMap<String, List<String>> getTorperfDataFiles() {
+    return this.torperfDataFiles;
+  }
+  public SortedMap<String, List<String>> getTorperfExtradataFiles() {
+    return this.torperfExtradataFiles;
+  }
   public boolean getProvideFilesViaRsync() {
     return this.provideFilesViaRsync;
   }
diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java
index ab79174..a5b5080 100644
--- a/src/org/torproject/ernie/db/Main.java
+++ b/src/org/torproject/ernie/db/Main.java
@@ -136,6 +136,13 @@ public class Main {
           new File(config.getSanitizedAssignmentsDirectory()));
     }
 
+    // Process Torperf files
+    if (config.getProcessTorperfFiles()) {
+      new TorperfDownloader(new File(config.getTorperfOutputDirectory()),
+          config.getTorperfSources(), config.getTorperfDataFiles(),
+          config.getTorperfExtradataFiles());
+    }
+
     // Copy recently published files to a local directory that can then
     // be served via rsync.
     if (config.getProvideFilesViaRsync()) {
diff --git a/src/org/torproject/ernie/db/TorperfDownloader.java b/src/org/torproject/ernie/db/TorperfDownloader.java
new file mode 100644
index 0000000..95d2e30
--- /dev/null
+++ b/src/org/torproject/ernie/db/TorperfDownloader.java
@@ -0,0 +1,116 @@
+/* Copyright 2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.db;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+import java.util.logging.*;
+
+/* Download possibly truncated Torperf .data and .extradata files from
+ * configured sources and append them to the files we already have. */
+public class TorperfDownloader {
+
+  private File torperfOutputDirectory = null;
+  private SortedMap<String, String> torperfSources = null;
+  private SortedMap<String, List<String>> torperfDataFiles = null;
+  private SortedMap<String, List<String>> torperfExtradataFiles = null;
+  private Logger logger = null;
+
+  public TorperfDownloader(File torperfOutputDirectory,
+      SortedMap<String, String> torperfSources,
+      SortedMap<String, List<String>> torperfDataFiles,
+      SortedMap<String, List<String>> torperfExtradataFiles) {
+    if (torperfOutputDirectory == null) {
+      throw new IllegalArgumentException();
+    }
+    this.torperfOutputDirectory = torperfOutputDirectory;
+    this.torperfSources = torperfSources;
+    this.torperfDataFiles = torperfDataFiles;
+    this.torperfExtradataFiles = torperfExtradataFiles;
+    if (!this.torperfOutputDirectory.exists()) {
+      this.torperfOutputDirectory.mkdirs();
+    }
+    this.logger = Logger.getLogger(TorperfDownloader.class.getName());
+    this.downloadAndMergeFiles(this.torperfDataFiles, true);
+    this.downloadAndMergeFiles(this.torperfExtradataFiles, false);
+  }
+
+  private void downloadAndMergeFiles(
+      SortedMap<String, List<String>> dataOrExtradataFiles,
+      boolean isDataFile) {
+    for (Map.Entry<String, List<String>> e :
+        dataOrExtradataFiles.entrySet()) {
+      String sourceName = e.getKey();
+      String sourceBaseUrl = torperfSources.get(sourceName);
+      List<String> files = e.getValue();
+      for (String file : files) {
+        String url = sourceBaseUrl + file;
+        File outputFile = new File(torperfOutputDirectory,
+            sourceName + "-" + file);
+        this.downloadAndMergeFile(url, outputFile, isDataFile);
+      }
+    }
+  }
+
+  private void downloadAndMergeFile(String url, File outputFile,
+      boolean isDataFile) {
+    String lastTimestampLine = null;
+    int linesAfterLastTimestampLine = 0;
+    if (outputFile.exists() && outputFile.lastModified() >
+        System.currentTimeMillis() - 6L * 60L * 60L * 1000L) {
+      return;
+    } else if (outputFile.exists()) {
+      try {
+        BufferedReader br = new BufferedReader(new FileReader(
+            outputFile));
+        String line;
+        while ((line = br.readLine()) != null) {
+          if (isDataFile || line.contains(" LAUNCH")) {
+            lastTimestampLine = line;
+            linesAfterLastTimestampLine = 0;
+          } else {
+            linesAfterLastTimestampLine++;
+          }
+        }
+        br.close();
+      } catch (IOException e) {
+        logger.log(Level.WARNING, "Failed reading '"
+            + outputFile.getAbsolutePath() + "' to find the last line to "
+            + "append to.", e);
+        return;
+      }
+    }
+    try {
+      this.logger.fine("Downloading " + (isDataFile ? ".data" :
+          ".extradata") + " file from '" + url + "' and merging it into "
+          + "'" + outputFile.getAbsolutePath() + "'.");
+      URL u = new URL(url);
+      HttpURLConnection huc = (HttpURLConnection) u.openConnection();
+      huc.setRequestMethod("GET");
+      huc.connect();
+      BufferedReader br = new BufferedReader(new InputStreamReader(
+          huc.getInputStream()));
+      String line;
+      BufferedWriter bw = new BufferedWriter(new FileWriter(outputFile,
+          true));
+      boolean copyLines = lastTimestampLine == null;
+      while ((line = br.readLine()) != null) {
+        if (copyLines && linesAfterLastTimestampLine == 0) {
+          bw.write(line + "\n");
+        } else if (copyLines && linesAfterLastTimestampLine > 0) {
+          linesAfterLastTimestampLine--;
+        } else if (line.equals(lastTimestampLine)) {
+          copyLines = true;
+        }
+      }
+      bw.close();
+      br.close();
+    } catch (IOException e) {
+      logger.log(Level.WARNING, "Failed downloading and merging '" + url
+          + "'.", e);
+      return;
+    }
+  }
+}
+

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits