[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [metrics-db/master] Move all processing logic to sub modules.
commit 0df849df625f749342f0c6ae333928d4a92c37c7
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Fri Oct 26 13:24:39 2012 -0400
Move all processing logic to sub modules.
---
.../db/bridgedescs/SanitizedBridgesWriter.java | 33 ++++++-
.../BridgePoolAssignmentsProcessor.java | 8 +-
src/org/torproject/ernie/db/main/Main.java | 100 ++------------------
.../ernie/db/relaydescs/ArchiveWriter.java | 51 +++++++++-
.../ernie/db/torperf/TorperfDownloader.java | 15 ++-
5 files changed, 98 insertions(+), 109 deletions(-)
diff --git a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
index 85b0883..e46450f 100644
--- a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
+++ b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
@@ -27,6 +27,7 @@ import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.codec.digest.DigestUtils;
+import org.torproject.ernie.db.main.Configuration;
/**
* Sanitizes bridge descriptors, i.e., removes all possibly sensitive
@@ -48,6 +49,8 @@ public class SanitizedBridgesWriter {
*/
private Logger logger;
+ private File bridgeDirectoriesDirectory;
+
/**
* Output directory for writing sanitized bridge descriptors.
*/
@@ -70,15 +73,25 @@ public class SanitizedBridgesWriter {
/**
* Initializes this class.
*/
- public SanitizedBridgesWriter(File sanitizedBridgesDirectory,
- File statsDirectory, boolean replaceIPAddressesWithHashes,
- long limitBridgeSanitizingInterval) {
-
- if (sanitizedBridgesDirectory == null || statsDirectory == null) {
+ public SanitizedBridgesWriter(Configuration config,
+ File statsDirectory) {
+
+ File bridgeDirectoriesDirectory =
+ new File(config.getBridgeSnapshotsDirectory());
+ File sanitizedBridgesDirectory =
+ new File(config.getSanitizedBridgesWriteDirectory());
+ boolean replaceIPAddressesWithHashes =
+ config.getReplaceIPAddressesWithHashes();
+ long limitBridgeSanitizingInterval =
+ config.getLimitBridgeDescriptorMappings();
+
+ if (bridgeDirectoriesDirectory == null ||
+ sanitizedBridgesDirectory == null || statsDirectory == null) {
throw new IllegalArgumentException();
}
/* Memorize argument values. */
+ this.bridgeDirectoriesDirectory = bridgeDirectoriesDirectory;
this.sanitizedBridgesDirectory = sanitizedBridgesDirectory;
this.replaceIPAddressesWithHashes = replaceIPAddressesWithHashes;
@@ -155,6 +168,16 @@ public class SanitizedBridgesWriter {
} else {
this.bridgeSanitizingCutOffTimestamp = "1999-12-31 23:59:59";
}
+
+ // Prepare bridge descriptor parser
+ BridgeDescriptorParser bdp = new BridgeDescriptorParser(this);
+
+ // Import bridge descriptors
+ new BridgeSnapshotReader(bdp, this.bridgeDirectoriesDirectory,
+ statsDirectory);
+
+ // Finish writing sanitized bridge descriptors to disk
+ this.finishWriting();
}
private String scrubOrAddress(String orAddress, byte[] fingerprintBytes,
diff --git a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
index 470f6ab..76fa539 100644
--- a/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
+++ b/src/org/torproject/ernie/db/bridgepools/BridgePoolAssignmentsProcessor.java
@@ -26,11 +26,15 @@ import org.apache.commons.codec.DecoderException;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
+import org.torproject.ernie.db.main.Configuration;
public class BridgePoolAssignmentsProcessor {
- public BridgePoolAssignmentsProcessor(File assignmentsDirectory,
- File sanitizedAssignmentsDirectory) {
+ public BridgePoolAssignmentsProcessor(Configuration config) {
+ File assignmentsDirectory =
+ new File(config.getAssignmentsDirectory());
+ File sanitizedAssignmentsDirectory =
+ new File(config.getSanitizedAssignmentsDirectory());
Logger logger =
Logger.getLogger(BridgePoolAssignmentsProcessor.class.getName());
diff --git a/src/org/torproject/ernie/db/main/Main.java b/src/org/torproject/ernie/db/main/Main.java
index e008eca..9109a75 100644
--- a/src/org/torproject/ernie/db/main/Main.java
+++ b/src/org/torproject/ernie/db/main/Main.java
@@ -3,19 +3,12 @@
package org.torproject.ernie.db.main;
import java.io.File;
-import java.util.List;
import java.util.logging.Logger;
-import org.torproject.ernie.db.bridgedescs.BridgeDescriptorParser;
-import org.torproject.ernie.db.bridgedescs.BridgeSnapshotReader;
import org.torproject.ernie.db.bridgedescs.SanitizedBridgesWriter;
import org.torproject.ernie.db.bridgepools.BridgePoolAssignmentsProcessor;
import org.torproject.ernie.db.exitlists.ExitListDownloader;
-import org.torproject.ernie.db.relaydescs.ArchiveReader;
import org.torproject.ernie.db.relaydescs.ArchiveWriter;
-import org.torproject.ernie.db.relaydescs.CachedRelayDescriptorReader;
-import org.torproject.ernie.db.relaydescs.RelayDescriptorDownloader;
-import org.torproject.ernie.db.relaydescs.RelayDescriptorParser;
import org.torproject.ernie.db.torperf.TorperfDownloader;
/**
@@ -45,89 +38,15 @@ public class Main {
// Define stats directory for temporary files
File statsDirectory = new File("stats");
- // Prepare writing relay descriptor archive to disk
- ArchiveWriter aw = config.getWriteDirectoryArchives() ?
- new ArchiveWriter(
- new File(config.getDirectoryArchivesOutputDirectory())) : null;
-
- // Prepare relay descriptor parser (only if we are writing stats or
- // directory archives to disk)
- RelayDescriptorParser rdp = aw != null ?
- new RelayDescriptorParser(aw) : null;
-
// Import/download relay descriptors from the various sources
- if (rdp != null) {
- RelayDescriptorDownloader rdd = null;
- if (config.getDownloadRelayDescriptors()) {
- List<String> dirSources =
- config.getDownloadFromDirectoryAuthorities();
- rdd = new RelayDescriptorDownloader(rdp, dirSources,
- config.getDownloadCurrentConsensus(),
- config.getDownloadCurrentVotes(),
- config.getDownloadMissingServerDescriptors(),
- config.getDownloadMissingExtraInfoDescriptors(),
- config.getDownloadAllServerDescriptors(),
- config.getDownloadAllExtraInfoDescriptors(),
- config.getCompressRelayDescriptorDownloads());
- rdp.setRelayDescriptorDownloader(rdd);
- }
- if (config.getImportCachedRelayDescriptors()) {
- new CachedRelayDescriptorReader(rdp,
- config.getCachedRelayDescriptorDirectory(), statsDirectory);
- if (aw != null) {
- aw.intermediateStats("importing relay descriptors from local "
- + "Tor data directories");
- }
- }
- if (config.getImportDirectoryArchives()) {
- new ArchiveReader(rdp,
- new File(config.getDirectoryArchivesDirectory()),
- statsDirectory,
- config.getKeepDirectoryArchiveImportHistory());
- if (aw != null) {
- aw.intermediateStats("importing relay descriptors from local "
- + "directory");
- }
- }
- if (rdd != null) {
- rdd.downloadDescriptors();
- rdd.writeFile();
- rdd = null;
- if (aw != null) {
- aw.intermediateStats("downloading relay descriptors from the "
- + "directory authorities");
- }
- }
- }
-
- // Write output to disk that only depends on relay descriptors
- if (aw != null) {
- aw.dumpStats();
- aw = null;
- }
-
- // Prepare sanitized bridge descriptor writer
- SanitizedBridgesWriter sbw = config.getWriteSanitizedBridges() ?
- new SanitizedBridgesWriter(
- new File(config.getSanitizedBridgesWriteDirectory()),
- statsDirectory, config.getReplaceIPAddressesWithHashes(),
- config.getLimitBridgeDescriptorMappings()) : null;
-
- // Prepare bridge descriptor parser
- BridgeDescriptorParser bdp = config.getWriteSanitizedBridges()
- ? new BridgeDescriptorParser(sbw) : null;
-
- // Import bridge descriptors
- if (bdp != null && config.getImportBridgeSnapshots()) {
- new BridgeSnapshotReader(bdp,
- new File(config.getBridgeSnapshotsDirectory()),
- statsDirectory);
+ if (config.getWriteDirectoryArchives()) {
+ new ArchiveWriter(config, statsDirectory);
}
- // Finish writing sanitized bridge descriptors to disk
- if (sbw != null) {
- sbw.finishWriting();
- sbw = null;
+ // Sanitize bridge descriptors
+ if (config.getImportBridgeSnapshots() &&
+ config.getWriteSanitizedBridges()) {
+ new SanitizedBridgesWriter(config, statsDirectory);
}
// Download exit list and store it to disk
@@ -137,15 +56,12 @@ public class Main {
// Process bridge pool assignments
if (config.getProcessBridgePoolAssignments()) {
- new BridgePoolAssignmentsProcessor(
- new File(config.getAssignmentsDirectory()),
- new File(config.getSanitizedAssignmentsDirectory()));
+ new BridgePoolAssignmentsProcessor(config);
}
// Process Torperf files
if (config.getProcessTorperfFiles()) {
- new TorperfDownloader(new File(config.getTorperfOutputDirectory()),
- config.getTorperfSources(), config.getTorperfFiles());
+ new TorperfDownloader(config);
}
// Copy recently published files to a local directory that can then
diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
index c632656..d53f726 100644
--- a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
+++ b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
@@ -11,6 +11,7 @@ import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
+import java.util.List;
import java.util.SortedSet;
import java.util.Stack;
import java.util.TimeZone;
@@ -23,6 +24,7 @@ import org.apache.commons.codec.binary.Hex;
import org.torproject.descriptor.DescriptorParser;
import org.torproject.descriptor.DescriptorSourceFactory;
import org.torproject.descriptor.impl.DescriptorParseException;
+import org.torproject.ernie.db.main.Configuration;
public class ArchiveWriter {
private Logger logger;
@@ -31,16 +33,57 @@ public class ArchiveWriter {
private int storedConsensuses = 0, storedVotes = 0, storedCerts = 0,
storedServerDescriptors = 0, storedExtraInfoDescriptors = 0;
- public ArchiveWriter(File outputDirectory) {
+ public ArchiveWriter(Configuration config, File statsDirectory) {
- if (outputDirectory == null) {
- throw new IllegalArgumentException();
- }
+ File outputDirectory =
+ new File(config.getDirectoryArchivesOutputDirectory());
this.logger = Logger.getLogger(ArchiveWriter.class.getName());
this.outputDirectory = outputDirectory;
this.descriptorParser =
DescriptorSourceFactory.createDescriptorParser();
+
+ // Prepare relay descriptor parser
+ RelayDescriptorParser rdp = new RelayDescriptorParser(this);
+
+ RelayDescriptorDownloader rdd = null;
+ if (config.getDownloadRelayDescriptors()) {
+ List<String> dirSources =
+ config.getDownloadFromDirectoryAuthorities();
+ rdd = new RelayDescriptorDownloader(rdp, dirSources,
+ config.getDownloadCurrentConsensus(),
+ config.getDownloadCurrentVotes(),
+ config.getDownloadMissingServerDescriptors(),
+ config.getDownloadMissingExtraInfoDescriptors(),
+ config.getDownloadAllServerDescriptors(),
+ config.getDownloadAllExtraInfoDescriptors(),
+ config.getCompressRelayDescriptorDownloads());
+ rdp.setRelayDescriptorDownloader(rdd);
+ }
+ if (config.getImportCachedRelayDescriptors()) {
+ new CachedRelayDescriptorReader(rdp,
+ config.getCachedRelayDescriptorDirectory(), statsDirectory);
+ this.intermediateStats("importing relay descriptors from local "
+ + "Tor data directories");
+ }
+ if (config.getImportDirectoryArchives()) {
+ new ArchiveReader(rdp,
+ new File(config.getDirectoryArchivesDirectory()),
+ statsDirectory,
+ config.getKeepDirectoryArchiveImportHistory());
+ this.intermediateStats("importing relay descriptors from local "
+ + "directory");
+ }
+ if (rdd != null) {
+ rdd.downloadDescriptors();
+ rdd.writeFile();
+ rdd = null;
+ this.intermediateStats("downloading relay descriptors from the "
+ + "directory authorities");
+ }
+
+ // Write output to disk that only depends on relay descriptors
+ this.dumpStats();
}
private boolean store(byte[] typeAnnotation, byte[] data,
diff --git a/src/org/torproject/ernie/db/torperf/TorperfDownloader.java b/src/org/torproject/ernie/db/torperf/TorperfDownloader.java
index 1ac593a..a931965 100644
--- a/src/org/torproject/ernie/db/torperf/TorperfDownloader.java
+++ b/src/org/torproject/ernie/db/torperf/TorperfDownloader.java
@@ -20,6 +20,8 @@ import java.util.TreeMap;
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.torproject.ernie.db.main.Configuration;
+
/* Download possibly truncated Torperf .data and .extradata files from
* configured sources, append them to the files we already have, and merge
* the two files into the .tpf format. */
@@ -31,12 +33,13 @@ public class TorperfDownloader {
private Logger logger = null;
private SimpleDateFormat dateFormat;
- public TorperfDownloader(File torperfOutputDirectory,
- SortedMap<String, String> torperfSources,
- List<String> torperfFilesLines) {
- if (torperfOutputDirectory == null) {
- throw new IllegalArgumentException();
- }
+ public TorperfDownloader(Configuration config) {
+
+ File torperfOutputDirectory =
+ new File(config.getTorperfOutputDirectory());
+ SortedMap<String, String> torperfSources = config.getTorperfSources();
+ List<String> torperfFilesLines = config.getTorperfFiles();
+
this.torperfOutputDirectory = torperfOutputDirectory;
this.torperfSources = torperfSources;
this.torperfFilesLines = torperfFilesLines;
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits