[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [onionoo/release] Fetch descriptors from both CollecTor instances.
commit 520c1577c119f46f5de8db83ee9d4eb498a30e7a
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Tue Apr 3 15:19:36 2018 +0200
Fetch descriptors from both CollecTor instances.
Fixes #25700.
---
CHANGELOG.md | 1 +
.../onionoo/updater/DescriptorSource.java | 85 +++++++++++++++-------
2 files changed, 60 insertions(+), 26 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 36d6471..ac15a8f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,7 @@
* Medium changes
- Add version_status field to details documents.
+ - Fetch descriptors from both CollecTor instances.
* Minor changes
- Don't attempt to un-escape character sequences in contact lines
diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java
index 6bddd02..20e249d 100644
--- a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java
+++ b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java
@@ -25,32 +25,54 @@ public class DescriptorSource {
private final File inDir = new File("in");
- private final File inRecentDir = new File(inDir, "recent");
+ private final String[] collecTorHosts = new String[] {
+ "collector.torproject.org", "collector2.torproject.org" };
+
+ private File[] inCollecTorHostDirs;
+
+ private File[] inCollecTorHostRecentDirs;
+
+ private List<DescriptorQueue> recentDescriptorQueues;
private final File inArchiveDir = new File(inDir, "archive");
private final File statusDir = new File("status");
- private List<DescriptorQueue> descriptorQueues;
-
private DescriptorQueue archiveDescriptorQueue;
/** Instantiates a new descriptor source. */
public DescriptorSource() {
- this.descriptorQueues = new ArrayList<>();
+ this.inCollecTorHostDirs = new File[this.collecTorHosts.length];
+ this.inCollecTorHostRecentDirs = new File[this.collecTorHosts.length];
+ for (int collecTorHostIndex = 0;
+ collecTorHostIndex < this.collecTorHosts.length;
+ collecTorHostIndex++) {
+ this.inCollecTorHostDirs[collecTorHostIndex]
+ = new File(this.statusDir, this.collecTorHosts[collecTorHostIndex]);
+ this.inCollecTorHostRecentDirs[collecTorHostIndex]
+ = new File(this.inCollecTorHostDirs[collecTorHostIndex], "recent");
+ }
+ this.recentDescriptorQueues = new ArrayList<>();
this.descriptorListeners = new HashMap<>();
}
- private DescriptorQueue getDescriptorQueue(
+ private List<DescriptorQueue> getDescriptorQueues(
DescriptorType descriptorType,
DescriptorHistory descriptorHistory) {
- DescriptorQueue descriptorQueue = new DescriptorQueue(
- this.inRecentDir, descriptorType, this.statusDir);
- if (descriptorHistory != null) {
- descriptorQueue.readHistoryFile(descriptorHistory);
+ List<DescriptorQueue> descriptorQueues = new ArrayList<>();
+ for (int collecTorHostIndex = 0;
+ collecTorHostIndex < this.collecTorHosts.length;
+ collecTorHostIndex++) {
+ DescriptorQueue descriptorQueue = new DescriptorQueue(
+ this.inCollecTorHostRecentDirs[collecTorHostIndex], descriptorType,
+ this.inCollecTorHostDirs[collecTorHostIndex]);
+ if (descriptorHistory != null) {
+ descriptorQueue.readHistoryFile(descriptorHistory);
+ }
+ descriptorQueues.add(descriptorQueue);
}
- this.descriptorQueues.add(descriptorQueue);
- return descriptorQueue;
+ this.recentDescriptorQueues.addAll(descriptorQueues);
+ return descriptorQueues;
}
private Map<DescriptorType, Set<DescriptorListener>>
@@ -68,14 +90,24 @@ public class DescriptorSource {
/** Downloads descriptors from CollecTor. */
public void downloadDescriptors() {
- List<String> remoteDirectories = new ArrayList<>();
+ List<String> remoteDirectoriesList = new ArrayList<>();
for (DescriptorType descriptorType : DescriptorType.values()) {
- remoteDirectories.add("/recent/" + descriptorType.getDir());
+ remoteDirectoriesList.add("/recent/" + descriptorType.getDir());
+ }
+ for (int collecTorHostIndex = 0;
+ collecTorHostIndex < this.collecTorHosts.length;
+ collecTorHostIndex++) {
+ String collecTorBaseUrl = "https://";
+ + this.collecTorHosts[collecTorHostIndex];
+ String[] remoteDirectories = remoteDirectoriesList.toArray(new String[0]);
+ long minLastModified = 0L;
+ File localDirectory = this.inCollecTorHostDirs[collecTorHostIndex];
+ boolean deleteExtraneousLocalFiles = true;
+ DescriptorCollector dc = org.torproject.descriptor.DescriptorSourceFactory
+ .createDescriptorCollector();
+ dc.collectDescriptors(collecTorBaseUrl, remoteDirectories,
+ minLastModified, localDirectory, deleteExtraneousLocalFiles);
}
- DescriptorCollector dc = org.torproject.descriptor.DescriptorSourceFactory
- .createDescriptorCollector();
- dc.collectDescriptors("https://collector.torproject.org";,
- remoteDirectories.toArray(new String[0]), 0L, inDir, true);
}
/** Reads archived and recent descriptors from disk and feeds them into
@@ -113,12 +145,13 @@ public class DescriptorSource {
}
Set<DescriptorListener> descriptorListeners =
this.descriptorListeners.get(descriptorType);
- DescriptorQueue descriptorQueue = this.getDescriptorQueue(
- descriptorType, descriptorHistory);
- Descriptor descriptor;
- while ((descriptor = descriptorQueue.nextDescriptor()) != null) {
- for (DescriptorListener descriptorListener : descriptorListeners) {
- descriptorListener.processDescriptor(descriptor, relay);
+ for (DescriptorQueue descriptorQueue
+ : this.getDescriptorQueues(descriptorType, descriptorHistory)) {
+ Descriptor descriptor;
+ while ((descriptor = descriptorQueue.nextDescriptor()) != null) {
+ for (DescriptorListener descriptorListener : descriptorListeners) {
+ descriptorListener.processDescriptor(descriptor, relay);
+ }
}
}
log.info("Read recent/{}.", descriptorType.getDir());
@@ -185,7 +218,7 @@ public class DescriptorSource {
/** Writes parse histories for recent descriptors to disk. */
public void writeHistoryFiles() {
log.debug("Writing parse histories for recent descriptors...");
- for (DescriptorQueue descriptorQueue : this.descriptorQueues) {
+ for (DescriptorQueue descriptorQueue : this.recentDescriptorQueues) {
descriptorQueue.writeHistoryFile();
}
}
@@ -194,13 +227,13 @@ public class DescriptorSource {
* descriptors during the current execution. */
public String getStatsString() {
StringBuilder sb = new StringBuilder();
- sb.append(" ").append(this.descriptorQueues.size())
+ sb.append(" ").append(this.recentDescriptorQueues.size())
.append(" descriptor ").append("queues created for recent descriptors\n");
int historySizeBefore = 0;
int historySizeAfter = 0;
long descriptors = 0L;
long bytes = 0L;
- for (DescriptorQueue descriptorQueue : this.descriptorQueues) {
+ for (DescriptorQueue descriptorQueue : this.recentDescriptorQueues) {
historySizeBefore += descriptorQueue.getHistorySizeBefore();
historySizeAfter += descriptorQueue.getHistorySizeAfter();
descriptors += descriptorQueue.getReturnedDescriptors();
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits