[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [metrics-db/master] Run five data-processing modules in parallel.
commit 7db75dbedf2fc77a3855e4fdff051bcbdcccac5e
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Fri Oct 26 15:13:21 2012 -0400
Run five data-processing modules in parallel.
---
.../db/bridgedescs/SanitizedBridgesWriter.java | 19 ++++++++++++-------
.../BridgePoolAssignmentsProcessor.java | 8 +++++++-
.../ernie/db/exitlists/ExitListDownloader.java | 6 +++++-
src/org/torproject/ernie/db/main/Main.java | 14 +++++---------
.../ernie/db/relaydescs/ArchiveWriter.java | 12 ++++++++++--
.../ernie/db/torperf/TorperfDownloader.java | 10 ++++++++--
6 files changed, 47 insertions(+), 22 deletions(-)
diff --git a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
index 10d2baa..cb08df1 100644
--- a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
+++ b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
@@ -43,7 +43,16 @@ import org.torproject.ernie.db.main.RsyncDataProvider;
* by the bridge to advertise their capabilities), and extra-info
* descriptors (published by the bridge, mainly for statistical analysis).
*/
-public class SanitizedBridgesWriter {
+public class SanitizedBridgesWriter extends Thread {
+
+ private Configuration config;
+
+ /**
+ * Initializes this class.
+ */
+ public SanitizedBridgesWriter(Configuration config) {
+ this.config = config;
+ }
/**
* Logger for this class.
@@ -71,12 +80,7 @@ public class SanitizedBridgesWriter {
private SecureRandom secureRandom;
- /**
- * Initializes this class.
- */
- public SanitizedBridgesWriter(Configuration config,
- File statsDirectory) {
-
+ public void run() {
File bridgeDirectoriesDirectory =
new File(config.getBridgeSnapshotsDirectory());
File sanitizedBridgesDirectory =
@@ -85,6 +89,7 @@ public class SanitizedBridgesWriter {
config.getReplaceIPAddressesWithHashes();
long limitBridgeSanitizingInterval =
config.getLimitBridgeDescriptorMappings();
+ File statsDirectory = new File("stats");
if (bridgeDirectoriesDirectory == null ||
sanitizedBridgesDirectory == null || statsDirectory == null) {
diff --git a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
index 3893495..d822d11 100644
--- a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
+++ b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
@@ -29,9 +29,15 @@ import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.torproject.ernie.db.main.Configuration;
import org.torproject.ernie.db.main.RsyncDataProvider;
-public class BridgePoolAssignmentsProcessor {
+public class BridgePoolAssignmentsProcessor extends Thread {
+
+ private Configuration config;
public BridgePoolAssignmentsProcessor(Configuration config) {
+ this.config = config;
+ }
+
+ public void run() {
File assignmentsDirectory =
new File(config.getAssignmentsDirectory());
File sanitizedAssignmentsDirectory =
diff --git a/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java b/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java
index f6f2865..bf67bb7 100644
--- a/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java
+++ b/src/org/torproject/ernie/db/exitlists/ExitListDownloader.java
@@ -21,8 +21,12 @@ import java.util.logging.Logger;
import org.torproject.ernie.db.main.Configuration;
import org.torproject.ernie.db.main.RsyncDataProvider;
-public class ExitListDownloader {
+public class ExitListDownloader extends Thread {
+
public ExitListDownloader(Configuration config) {
+ }
+
+ public void run() {
Logger logger = Logger.getLogger(ExitListDownloader.class.getName());
try {
logger.fine("Downloading exit list...");
diff --git a/src/org/torproject/ernie/db/main/Main.java b/src/org/torproject/ernie/db/main/Main.java
index 093f002..88f2372 100644
--- a/src/org/torproject/ernie/db/main/Main.java
+++ b/src/org/torproject/ernie/db/main/Main.java
@@ -2,7 +2,6 @@
* See LICENSE for licensing information */
package org.torproject.ernie.db.main;
-import java.io.File;
import java.util.logging.Logger;
import org.torproject.ernie.db.bridgedescs.SanitizedBridgesWriter;
@@ -35,33 +34,30 @@ public class Main {
System.exit(1);
}
- // Define stats directory for temporary files
- File statsDirectory = new File("stats");
-
// Import/download relay descriptors from the various sources
if (config.getWriteDirectoryArchives()) {
- new ArchiveWriter(config, statsDirectory);
+ new ArchiveWriter(config).start();
}
// Sanitize bridge descriptors
if (config.getImportBridgeSnapshots() &&
config.getWriteSanitizedBridges()) {
- new SanitizedBridgesWriter(config, statsDirectory);
+ new SanitizedBridgesWriter(config).start();
}
// Download exit list and store it to disk
if (config.getDownloadExitList()) {
- new ExitListDownloader(config);
+ new ExitListDownloader(config).start();
}
// Process bridge pool assignments
if (config.getProcessBridgePoolAssignments()) {
- new BridgePoolAssignmentsProcessor(config);
+ new BridgePoolAssignmentsProcessor(config).start();
}
// Process Torperf files
if (config.getProcessTorperfFiles()) {
- new TorperfDownloader(config);
+ new TorperfDownloader(config).start();
}
// Remove lock file
diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
index a9cb604..5c3d2e9 100644
--- a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
+++ b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
@@ -27,17 +27,25 @@ import org.torproject.descriptor.impl.DescriptorParseException;
import org.torproject.ernie.db.main.Configuration;
import org.torproject.ernie.db.main.RsyncDataProvider;
-public class ArchiveWriter {
+public class ArchiveWriter extends Thread {
+
+ private Configuration config;
+
+ public ArchiveWriter(Configuration config) {
+ this.config = config;
+ }
+
private Logger logger;
private File outputDirectory;
private DescriptorParser descriptorParser;
private int storedConsensuses = 0, storedVotes = 0, storedCerts = 0,
storedServerDescriptors = 0, storedExtraInfoDescriptors = 0;
- public ArchiveWriter(Configuration config, File statsDirectory) {
+ public void run() {
File outputDirectory =
new File(config.getDirectoryArchivesOutputDirectory());
+ File statsDirectory = new File("stats");
this.logger = Logger.getLogger(ArchiveWriter.class.getName());
this.outputDirectory = outputDirectory;
diff --git a/src/org/torproject/ernie/db/torperf/TorperfDownloader.java b/src/org/torproject/ernie/db/torperf/TorperfDownloader.java
index 5f09038..ef67ae7 100644
--- a/src/org/torproject/ernie/db/torperf/TorperfDownloader.java
+++ b/src/org/torproject/ernie/db/torperf/TorperfDownloader.java
@@ -26,7 +26,13 @@ import org.torproject.ernie.db.main.RsyncDataProvider;
/* Download possibly truncated Torperf .data and .extradata files from
* configured sources, append them to the files we already have, and merge
* the two files into the .tpf format. */
-public class TorperfDownloader {
+public class TorperfDownloader extends Thread {
+
+ private Configuration config;
+
+ public TorperfDownloader(Configuration config) {
+ this.config = config;
+ }
private File torperfOutputDirectory = null;
private SortedMap<String, String> torperfSources = null;
@@ -34,7 +40,7 @@ public class TorperfDownloader {
private Logger logger = null;
private SimpleDateFormat dateFormat;
- public TorperfDownloader(Configuration config) {
+ public void run() {
File torperfOutputDirectory =
new File(config.getTorperfOutputDirectory());
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits