[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [collector/master] Refactor CachedRelayDescriptorReader.
commit 698d15d94f52fe1192fd2afb82439abace27066c
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Mon Aug 20 21:17:03 2018 +0200
Refactor CachedRelayDescriptorReader.
---
.../collector/relaydescs/ArchiveWriter.java | 3 +-
.../relaydescs/CachedRelayDescriptorReader.java | 96 ++++++++++++++--------
2 files changed, 62 insertions(+), 37 deletions(-)
diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
index 8679439..3429a0a 100644
--- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
+++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
@@ -161,7 +161,8 @@ public class ArchiveWriter extends CollecTorMain {
}
if (sources.contains(SourceType.Cache)) {
new CachedRelayDescriptorReader(rdp,
- config.getStringArray(Key.RelayCacheOrigins), statsDirectory);
+ config.getStringArray(Key.RelayCacheOrigins), statsDirectory)
+ .readDescriptors();
this.intermediateStats("importing relay descriptors from local "
+ "Tor data directories");
}
diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java
index f64d019..4f0d012 100644
--- a/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java
+++ b/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java
@@ -37,32 +37,52 @@ public class CachedRelayDescriptorReader {
private static final Logger logger = LoggerFactory.getLogger(
CachedRelayDescriptorReader.class);
- /** Reads cached-descriptor files from one or more directories and
- * passes them to the given descriptor parser. */
- public CachedRelayDescriptorReader(RelayDescriptorParser rdp,
- String[] inputDirectories, File statsDirectory) {
+ private RelayDescriptorParser rdp;
+
+ private String[] inputDirectories;
+
+ private File importHistoryFile;
+ private StringBuilder dumpStats;
+
+ private Set<String> lastImportHistory = new HashSet<>();
+
+ private Set<String> currentImportHistory = new HashSet<>();
+
+ /** Initializes this reader but without starting to read yet. */
+ CachedRelayDescriptorReader(RelayDescriptorParser rdp,
+ String[] inputDirectories, File statsDirectory) {
if (rdp == null || inputDirectories == null
|| inputDirectories.length == 0 || statsDirectory == null) {
throw new IllegalArgumentException();
}
+ this.rdp = rdp;
+ this.inputDirectories = inputDirectories;
+ this.importHistoryFile = new File(statsDirectory,
+ "cacheddesc-import-history");
- StringBuilder dumpStats = new StringBuilder("Finished importing "
+ this.dumpStats = new StringBuilder("Finished importing "
+ "relay descriptors from local Tor data directories:");
+ }
- /* Read import history containing SHA-1 digests of previously parsed
- * statuses and descriptors, so that we can skip them in this run. */
- Set<String> lastImportHistory = new HashSet<>();
- Set<String> currentImportHistory = new HashSet<>();
- File importHistoryFile = new File(statsDirectory,
- "cacheddesc-import-history");
+ /** Reads cached-descriptor files from one or more directories and
+ * passes them to the given descriptor parser. */
+ public void readDescriptors() {
+ this.readHistoryFile();
+ this.readDescriptorFiles();
+ this.writeHistoryFile();
+ }
+
+ /** Read import history containing SHA-1 digests of previously parsed
+ * statuses and descriptors, so that we can skip them in this run. */
+ private void readHistoryFile() {
if (importHistoryFile.exists()) {
try {
BufferedReader br = new BufferedReader(new FileReader(
importHistoryFile));
String line;
while ((line = br.readLine()) != null) {
- lastImportHistory.add(line);
+ this.lastImportHistory.add(line);
}
br.close();
} catch (IOException e) {
@@ -70,9 +90,11 @@ public class CachedRelayDescriptorReader {
+ importHistoryFile.getAbsolutePath() + ".", e);
}
}
+ }
- /* Read cached descriptors directories. */
- for (String inputDirectory : inputDirectories) {
+ /** Read cached descriptors directories. */
+ private void readDescriptorFiles() {
+ for (String inputDirectory : this.inputDirectories) {
File cachedDescDir = new File(inputDirectory);
if (!cachedDescDir.exists()) {
logger.warn("Directory " + cachedDescDir.getAbsolutePath()
@@ -113,7 +135,7 @@ public class CachedRelayDescriptorReader {
String line;
while ((line = br.readLine()) != null) {
if (line.startsWith("valid-after ")) {
- dumpStats.append("\n").append(f.getName()).append(": ")
+ this.dumpStats.append("\n").append(f.getName()).append(": ")
.append(line.substring("valid-after ".length()));
SimpleDateFormat dateTimeFormat =
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
@@ -125,7 +147,7 @@ public class CachedRelayDescriptorReader {
+ cachedDescDir.getAbsolutePath() + " are stale. "
+ "The valid-after line in cached-consensus is '"
+ line + "'.");
- dumpStats.append(" (stale!)");
+ this.dumpStats.append(" (stale!)");
}
break;
}
@@ -136,13 +158,13 @@ public class CachedRelayDescriptorReader {
* (but regardless of whether it's stale or not). */
String digest = Hex.encodeHexString(DigestUtils.sha1(
allData));
- if (!lastImportHistory.contains(digest)
- && !currentImportHistory.contains(digest)) {
- rdp.parse(allData);
+ if (!this.lastImportHistory.contains(digest)
+ && !this.currentImportHistory.contains(digest)) {
+ this.rdp.parse(allData);
} else {
- dumpStats.append(" (skipped)");
+ this.dumpStats.append(" (skipped)");
}
- currentImportHistory.add(digest);
+ this.currentImportHistory.add(digest);
} else if (f.getName().equals("v3-status-votes")) {
int parsedNum = 0;
int skippedNum = 0;
@@ -161,18 +183,18 @@ public class CachedRelayDescriptorReader {
next - start);
String digest = Hex.encodeHexString(DigestUtils.sha1(
rawNetworkStatusBytes));
- if (!lastImportHistory.contains(digest)
- && !currentImportHistory.contains(digest)) {
- rdp.parse(rawNetworkStatusBytes);
+ if (!this.lastImportHistory.contains(digest)
+ && !this.currentImportHistory.contains(digest)) {
+ this.rdp.parse(rawNetworkStatusBytes);
parsedNum++;
} else {
skippedNum++;
}
- currentImportHistory.add(digest);
+ this.currentImportHistory.add(digest);
}
start = next;
}
- dumpStats.append("\n").append(f.getName()).append(": parsed ")
+ this.dumpStats.append("\n").append(f.getName()).append(": parsed ")
.append(parsedNum).append(", skipped ").append(skippedNum)
.append(" votes");
} else if (f.getName().startsWith("cached-descriptors")
@@ -183,7 +205,7 @@ public class CachedRelayDescriptorReader {
int end = -1;
String startToken =
f.getName().startsWith("cached-descriptors")
- ? "router " : "extra-info ";
+ ? "router " : "extra-info ";
String sigToken = "\nrouter-signature\n";
String endToken = "\n-----END SIGNATURE-----\n";
int parsedNum = 0;
@@ -207,16 +229,16 @@ public class CachedRelayDescriptorReader {
System.arraycopy(allData, start, descBytes, 0, end - start);
String digest = Hex.encodeHexString(DigestUtils.sha1(
descBytes));
- if (!lastImportHistory.contains(digest)
- && !currentImportHistory.contains(digest)) {
- rdp.parse(descBytes);
+ if (!this.lastImportHistory.contains(digest)
+ && !this.currentImportHistory.contains(digest)) {
+ this.rdp.parse(descBytes);
parsedNum++;
} else {
skippedNum++;
}
- currentImportHistory.add(digest);
+ this.currentImportHistory.add(digest);
}
- dumpStats.append("\n").append(f.getName()).append(": parsed ")
+ this.dumpStats.append("\n").append(f.getName()).append(": parsed ")
.append(parsedNum).append(", skipped ").append(skippedNum)
.append(" ").append(f.getName().startsWith("cached-descriptors")
? "server" : "extra-info").append(" descriptors");
@@ -229,19 +251,21 @@ public class CachedRelayDescriptorReader {
logger.debug("Finished reading "
+ cachedDescDir.getAbsolutePath() + " directory.");
}
+ }
- /* Write import history containing SHA-1 digests to disk. */
+ /** Write import history containing SHA-1 digests to disk. */
+ private void writeHistoryFile() {
try {
- importHistoryFile.getParentFile().mkdirs();
+ this.importHistoryFile.getParentFile().mkdirs();
BufferedWriter bw = new BufferedWriter(new FileWriter(
- importHistoryFile));
+ this.importHistoryFile));
for (String digest : currentImportHistory) {
bw.write(digest + "\n");
}
bw.close();
} catch (IOException e) {
logger.warn("Could not write import history to "
- + importHistoryFile.getAbsolutePath() + ".", e);
+ + this.importHistoryFile.getAbsolutePath() + ".", e);
}
logger.info(dumpStats.toString());
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits