[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[or-cvs] [metrics-db/master] Allow limiting internal bridge descriptor mapping.
commit 62bd96d53037fa2712d75bbf9f23855a45f4689a
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Fri Feb 4 18:48:51 2011 +0100
Allow limiting internal bridge descriptor mapping.
---
config.template | 4 +
src/org/torproject/ernie/db/Configuration.java | 7 ++
src/org/torproject/ernie/db/Main.java | 3 +-
.../ernie/db/SanitizedBridgesWriter.java | 96 +++++++++++++++++++-
.../ernie/test/SanitizedBridgesWriterTest.java | 4 +-
5 files changed, 108 insertions(+), 6 deletions(-)
diff --git a/config.template b/config.template
index a1d35d4..1ef915c 100644
--- a/config.template
+++ b/config.template
@@ -98,6 +98,10 @@
## can learn about IP address changes.
#ReplaceIPAddressesWithHashes 0
#
+## Limit internal bridge descriptor mapping state to the following number
+## of days, or -1 for unlimited.
+#LimitBridgeDescriptorMappings -1
+#
## Relative path to directory to write sanitized bridges to
#SanitizedBridgesWriteDirectory sanitized-bridges/
#
diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java
index 48e5abe..37676ec 100644
--- a/src/org/torproject/ernie/db/Configuration.java
+++ b/src/org/torproject/ernie/db/Configuration.java
@@ -31,6 +31,7 @@ public class Configuration {
private String relayDescriptorRawFilesDirectory = "pg-import/";
private boolean writeSanitizedBridges = false;
private boolean replaceIPAddressesWithHashes = false;
+ private long limitBridgeDescriptorMappings = -1L;
private String sanitizedBridgesWriteDirectory = "sanitized-bridges/";
private boolean importSanitizedBridges = false;
private String sanitizedBridgesDirectory = "bridges/";
@@ -117,6 +118,9 @@ public class Configuration {
} else if (line.startsWith("ReplaceIPAddressesWithHashes")) {
this.replaceIPAddressesWithHashes = Integer.parseInt(
line.split(" ")[1]) != 0;
+ } else if (line.startsWith("LimitBridgeDescriptorMappings")) {
+ this.limitBridgeDescriptorMappings = Long.parseLong(
+ line.split(" ")[1]);
} else if (line.startsWith("SanitizedBridgesWriteDirectory")) {
this.sanitizedBridgesWriteDirectory = line.split(" ")[1];
} else if (line.startsWith("ImportSanitizedBridges")) {
@@ -297,6 +301,9 @@ public class Configuration {
public boolean getReplaceIPAddressesWithHashes() {
return this.replaceIPAddressesWithHashes;
}
+ public long getLimitBridgeDescriptorMappings() {
+ return this.limitBridgeDescriptorMappings;
+ }
public String getSanitizedBridgesWriteDirectory() {
return this.sanitizedBridgesWriteDirectory;
}
diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java
index a159642..20107d0 100644
--- a/src/org/torproject/ernie/db/Main.java
+++ b/src/org/torproject/ernie/db/Main.java
@@ -138,7 +138,8 @@ public class Main {
SanitizedBridgesWriter sbw = config.getWriteSanitizedBridges() ?
new SanitizedBridgesWriter(
new File(config.getSanitizedBridgesWriteDirectory()),
- statsDirectory, config.getReplaceIPAddressesWithHashes()) : null;
+ statsDirectory, config.getReplaceIPAddressesWithHashes(),
+ config.getLimitBridgeDescriptorMappings()) : null;
// Prepare bridge descriptor parser
BridgeDescriptorParser bdp = config.getWriteConsensusStats() ||
diff --git a/src/org/torproject/ernie/db/SanitizedBridgesWriter.java b/src/org/torproject/ernie/db/SanitizedBridgesWriter.java
index 243b0b6..6edcc62 100644
--- a/src/org/torproject/ernie/db/SanitizedBridgesWriter.java
+++ b/src/org/torproject/ernie/db/SanitizedBridgesWriter.java
@@ -169,12 +169,17 @@ public class SanitizedBridgesWriter {
private byte[] secretForHashingIPAddresses;
+ private String bridgeDescriptorMappingsCutOffTimestamp;
+
+ private boolean haveWarnedAboutLimitedMapping;
+
/**
* Initializes this class, including reading in the known descriptor
* mapping.
*/
public SanitizedBridgesWriter(File sanitizedBridgesDirectory,
- File statsDirectory, boolean replaceIPAddressesWithHashes) {
+ File statsDirectory, boolean replaceIPAddressesWithHashes,
+ long limitBridgeDescriptorMappings) {
if (sanitizedBridgesDirectory == null || statsDirectory == null) {
throw new IllegalArgumentException();
@@ -199,6 +204,20 @@ public class SanitizedBridgesWriter {
this.secretForHashingIPAddresses =
"secret for hashing IP addresses".getBytes();
+ /* If we're configured to keep descriptor mappings only for a limited
+ * time, define the cut-off day and time. */
+ if (limitBridgeDescriptorMappings >= 0L) {
+ SimpleDateFormat formatter = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
+ this.bridgeDescriptorMappingsCutOffTimestamp = formatter.format(
+ System.currentTimeMillis() - 24L * 60L * 60L * 1000L
+ * limitBridgeDescriptorMappings);
+ } else {
+ this.bridgeDescriptorMappingsCutOffTimestamp =
+ "1999-12-31 23:59:59";
+ }
+
/* Read known descriptor mappings from disk. */
this.bridgeDescriptorMappingsFile = new File(
"stats/bridge-descriptor-mappings");
@@ -207,9 +226,16 @@ public class SanitizedBridgesWriter {
BufferedReader br = new BufferedReader(new FileReader(
this.bridgeDescriptorMappingsFile));
String line = null;
+ int read = 0, skipped = 0;
while ((line = br.readLine()) != null) {
if (line.split(",").length == 4) {
String[] parts = line.split(",");
+ if (this.bridgeDescriptorMappingsCutOffTimestamp.
+ compareTo(parts[1]) > 0) {
+ skipped++;
+ continue;
+ }
+ read++;
DescriptorMapping dm = new DescriptorMapping(line);
dm.hashedBridgeIdentity = parts[0];
dm.published = parts[1];
@@ -225,6 +251,8 @@ public class SanitizedBridgesWriter {
}
}
br.close();
+ this.logger.fine("Finished reading " + read + " descriptor "
+ + "mappings from disk, skipped " + skipped + ".");
} catch (IOException e) {
this.logger.log(Level.WARNING, "Could not read in "
+ this.bridgeDescriptorMappingsFile.getAbsolutePath()
@@ -262,6 +290,15 @@ public class SanitizedBridgesWriter {
public void sanitizeAndStoreNetworkStatus(byte[] data,
String publicationTime) {
+ if (this.bridgeDescriptorMappingsCutOffTimestamp.
+ compareTo(publicationTime) > 0) {
+ this.logger.log(!this.haveWarnedAboutLimitedMapping ? Level.WARNING
+ : Level.FINE, "Sanitizing and storing network status with "
+ + "publication time outside our descriptor mapping interval. "
+ + "We might not be able to repair references.");
+ this.haveWarnedAboutLimitedMapping = true;
+ }
+
/* Parse the given network status line by line. */
StringBuilder scrubbed = new StringBuilder();
try {
@@ -425,6 +462,15 @@ public class SanitizedBridgesWriter {
* the sanitizing procedure. */
} else if (line.startsWith("published ")) {
published = line.substring("published ".length());
+ if (this.bridgeDescriptorMappingsCutOffTimestamp.
+ compareTo(published) > 0) {
+ this.logger.log(!this.haveWarnedAboutLimitedMapping
+ ? Level.WARNING : Level.FINE, "Sanitizing and storing "
+ + "server descriptor with publication time outside our "
+ + "descriptor mapping interval. We might not be able to "
+ + "repair references.");
+ this.haveWarnedAboutLimitedMapping = true;
+ }
this.descriptorPublicationTimes.add(published);
scrubbed.append(line + "\n");
@@ -618,6 +664,15 @@ public class SanitizedBridgesWriter {
} else if (line.startsWith("published ")) {
scrubbed.append(line + "\n");
published = line.substring("published ".length());
+ if (this.bridgeDescriptorMappingsCutOffTimestamp.
+ compareTo(published) > 0) {
+ this.logger.log(!this.haveWarnedAboutLimitedMapping
+ ? Level.WARNING : Level.FINE, "Sanitizing and storing "
+ + "extra-info descriptor with publication time outside "
+ + "our descriptor mapping interval. We might not be able "
+ + "to repair references.");
+ this.haveWarnedAboutLimitedMapping = true;
+ }
/* Write the following lines unmodified to the sanitized
* descriptor. */
@@ -697,6 +752,14 @@ public class SanitizedBridgesWriter {
}
public void storeSanitizedNetworkStatus(byte[] data, String published) {
+ if (this.bridgeDescriptorMappingsCutOffTimestamp.
+ compareTo(published) > 0) {
+ this.logger.log(!this.haveWarnedAboutLimitedMapping ? Level.WARNING
+ : Level.FINE, "Storing sanitized network status with "
+ + "publication time outside our descriptor mapping interval. "
+ + "We might not be able to repair references.");
+ this.haveWarnedAboutLimitedMapping = true;
+ }
String scrubbed = null;
try {
String ascii = new String(data, "US-ASCII");
@@ -806,6 +869,15 @@ public class SanitizedBridgesWriter {
+ line2.split(" ")[5] + "\n");
} else if (line2.startsWith("published ")) {
published = line2.substring("published ".length());
+ if (this.bridgeDescriptorMappingsCutOffTimestamp.
+ compareTo(published) > 0) {
+ this.logger.log(!this.haveWarnedAboutLimitedMapping
+ ? Level.WARNING : Level.FINE, "Storing sanitized "
+ + "server descriptor with publication time outside our "
+ + "descriptor mapping interval. We might not be able to "
+ + "repair references.");
+ this.haveWarnedAboutLimitedMapping = true;
+ }
sb.append(line2 + "\n");
this.descriptorPublicationTimes.add(published);
} else if (line2.startsWith("opt fingerprint ")) {
@@ -874,6 +946,15 @@ public class SanitizedBridgesWriter {
} else if (line2.startsWith("published ")) {
sb.append(line2 + "\n");
published = line2.substring("published ".length());
+ if (this.bridgeDescriptorMappingsCutOffTimestamp.
+ compareTo(published) > 0) {
+ this.logger.log(!this.haveWarnedAboutLimitedMapping
+ ? Level.WARNING : Level.FINE, "Storing sanitized "
+ + "extra-info descriptor with publication time outside "
+ + "our descriptor mapping interval. We might not be able "
+ + "to repair references.");
+ this.haveWarnedAboutLimitedMapping = true;
+ }
this.descriptorPublicationTimes.add(published);
} else {
sb.append(line2 + "\n");
@@ -1038,12 +1119,21 @@ public class SanitizedBridgesWriter {
this.logger.fine("Writing descriptor mappings to disk.");
BufferedWriter bw = new BufferedWriter(new FileWriter(
this.bridgeDescriptorMappingsFile));
+ int wrote = 0, skipped = 0;
for (DescriptorMapping mapping :
this.bridgeDescriptorMappings.values()) {
- bw.write(mapping.toString() + "\n");
+ String mappingString = mapping.toString();
+ if (this.bridgeDescriptorMappingsCutOffTimestamp.
+ compareTo(mappingString.split(",")[1]) > 0) {
+ skipped++;
+ } else {
+ wrote++;
+ bw.write(mapping.toString() + "\n");
+ }
}
bw.close();
- this.logger.fine("Finished writing descriptor mappings to disk.");
+ this.logger.fine("Finished writing " + wrote + " descriptor "
+ + "mappings to disk, skipped " + skipped + ".");
} catch (IOException e) {
this.logger.log(Level.WARNING, "Could not write descriptor "
+ "mappings to disk.", e);
diff --git a/src/org/torproject/ernie/test/SanitizedBridgesWriterTest.java b/src/org/torproject/ernie/test/SanitizedBridgesWriterTest.java
index d256122..f5b3b64 100644
--- a/src/org/torproject/ernie/test/SanitizedBridgesWriterTest.java
+++ b/src/org/torproject/ernie/test/SanitizedBridgesWriterTest.java
@@ -27,13 +27,13 @@ public class SanitizedBridgesWriterTest {
@Test(expected = IllegalArgumentException.class)
public void testSanitizedBridgesDirectoryNull() {
- new SanitizedBridgesWriter(null, this.tempStatsDirectory, false);
+ new SanitizedBridgesWriter(null, this.tempStatsDirectory, false, -1L);
}
@Test(expected = IllegalArgumentException.class)
public void testStatsDirectoryNull() {
new SanitizedBridgesWriter(this.tempSanitizedBridgesDirectory, null,
- false);
+ false, -1L);
}
}