[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[or-cvs] [metrics-db/master] Add import history to CachedRelayDescriptorReader.
commit e6d93810a0d31f382f4a4433fe176bc463bc534f
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Sun Jan 16 11:20:17 2011 +0100
Add import history to CachedRelayDescriptorReader.
---
.../ernie/db/CachedRelayDescriptorReader.java | 89 +++++++++++++++++---
1 files changed, 77 insertions(+), 12 deletions(-)
diff --git a/src/org/torproject/ernie/db/CachedRelayDescriptorReader.java b/src/org/torproject/ernie/db/CachedRelayDescriptorReader.java
index a8ccc14..098038b 100644
--- a/src/org/torproject/ernie/db/CachedRelayDescriptorReader.java
+++ b/src/org/torproject/ernie/db/CachedRelayDescriptorReader.java
@@ -7,6 +7,9 @@ import java.text.*;
import java.util.*;
import java.util.logging.*;
+import org.apache.commons.codec.binary.*;
+import org.apache.commons.codec.digest.*;
+
/**
* Parses all descriptors in local directory cacheddesc/ and sorts them
* into directory structure in directory-archive/.
@@ -18,6 +21,28 @@ public class CachedRelayDescriptorReader {
+ "relay descriptors from local Tor data directories:");
Logger logger = Logger.getLogger(
CachedRelayDescriptorReader.class.getName());
+
+ /* Read import history containing SHA-1 digests of previously parsed
+ * statuses and descriptors, so that we can skip them in this run. */
+ Set<String> lastImportHistory = new HashSet<String>(),
+ currentImportHistory = new HashSet<String>();
+ File importHistoryFile = new File("stats/cacheddesc-import-history");
+ if (importHistoryFile.exists()) {
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ importHistoryFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ lastImportHistory.add(line);
+ }
+ br.close();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Could not read import history from "
+ + importHistoryFile.getAbsolutePath() + ".", e);
+ }
+ }
+
+ /* Read cached descriptors directories. */
for (String inputDirectory : inputDirectories) {
File cachedDescDir = new File(inputDirectory);
if (!cachedDescDir.exists()) {
@@ -67,13 +92,21 @@ public class CachedRelayDescriptorReader {
}
br.close();
- /* Parse the cached consensus (regardless of whether it's
- * stale or not. */
+ /* Parse the cached consensus if we haven't parsed it before
+ * (but regardless of whether it's stale or not). */
if (rdp != null) {
- rdp.parse(allData);
+ String digest = Hex.encodeHexString(DigestUtils.sha(
+ allData));
+ if (!lastImportHistory.contains(digest) &&
+ !currentImportHistory.contains(digest)) {
+ rdp.parse(allData);
+ } else {
+ dumpStats.append(" (skipped)");
+ }
+ currentImportHistory.add(digest);
}
} else if (f.getName().equals("v3-status-votes")) {
- int parsedNum = 0;
+ int parsedNum = 0, skippedNum = 0;
String ascii = new String(allData, "US-ASCII");
String startToken = "network-status-version ";
int end = ascii.length();
@@ -88,14 +121,22 @@ public class CachedRelayDescriptorReader {
System.arraycopy(allData, start, rawNetworkStatusBytes, 0,
next - start);
if (rdp != null) {
- rdp.parse(rawNetworkStatusBytes);
+ String digest = Hex.encodeHexString(DigestUtils.sha(
+ rawNetworkStatusBytes));
+ if (!lastImportHistory.contains(digest) &&
+ !currentImportHistory.contains(digest)) {
+ rdp.parse(rawNetworkStatusBytes);
+ parsedNum++;
+ } else {
+ skippedNum++;
+ }
+ currentImportHistory.add(digest);
}
- parsedNum++;
}
start = next;
}
- dumpStats.append("\n" + f.getName() + ": " + parsedNum
- + " votes");
+ dumpStats.append("\n" + f.getName() + ": parsed " + parsedNum
+ + ", skipped " + skippedNum + " votes");
} else if (f.getName().startsWith("cached-descriptors") ||
f.getName().startsWith("cached-extrainfo")) {
String ascii = new String(allData, "US-ASCII");
@@ -105,7 +146,7 @@ public class CachedRelayDescriptorReader {
"router " : "extra-info ";
String sigToken = "\nrouter-signature\n";
String endToken = "\n-----END SIGNATURE-----\n";
- int parsedNum = 0;
+ int parsedNum = 0, skippedNum = 0;
while (end < ascii.length()) {
start = ascii.indexOf(startToken, end);
if (start < 0) {
@@ -124,11 +165,20 @@ public class CachedRelayDescriptorReader {
byte[] descBytes = new byte[end - start];
System.arraycopy(allData, start, descBytes, 0, end - start);
if (rdp != null) {
- rdp.parse(descBytes);
- parsedNum++;
+ String digest = Hex.encodeHexString(DigestUtils.sha(
+ descBytes));
+ if (!lastImportHistory.contains(digest) &&
+ !currentImportHistory.contains(digest)) {
+ rdp.parse(descBytes);
+ parsedNum++;
+ } else {
+ skippedNum++;
+ }
+ currentImportHistory.add(digest);
}
}
- dumpStats.append("\n" + f.getName() + ": " + parsedNum + " "
+ dumpStats.append("\n" + f.getName() + ": parsed " + parsedNum
+ + ", skipped " + skippedNum + " "
+ (f.getName().startsWith("cached-descriptors") ?
"server" : "extra-info") + " descriptors");
}
@@ -143,6 +193,21 @@ public class CachedRelayDescriptorReader {
logger.fine("Finished reading "
+ cachedDescDir.getAbsolutePath() + " directory.");
}
+
+ /* Write import history containing SHA-1 digests to disk. */
+ try {
+ importHistoryFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ importHistoryFile));
+ for (String digest : currentImportHistory) {
+ bw.write(digest + "\n");
+ }
+ bw.close();
+ } catch (IOException e) {
+ logger.log(Level.WARNING, "Could not write import history to "
+ + importHistoryFile.getAbsolutePath() + ".", e);
+ }
+
logger.info(dumpStats.toString());
}
}