[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [onionoo/master] Sort classes into six sub packages.
commit 43b563b5a0fe46cfaa899b67ca1fc6804166b468
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Wed Jul 23 17:34:36 2014 +0200
Sort classes into six sub packages.
---
build.xml | 2 +-
etc/web.xml.template | 4 +-
src/org/torproject/onionoo/ApplicationFactory.java | 51 --
src/org/torproject/onionoo/BandwidthDocument.java | 27 -
.../onionoo/BandwidthDocumentWriter.java | 190 -----
src/org/torproject/onionoo/BandwidthStatus.java | 78 --
.../torproject/onionoo/BandwidthStatusUpdater.java | 145 ----
src/org/torproject/onionoo/ClientsDocument.java | 22 -
.../torproject/onionoo/ClientsDocumentWriter.java | 284 --------
.../torproject/onionoo/ClientsGraphHistory.java | 83 ---
src/org/torproject/onionoo/ClientsStatus.java | 211 ------
.../torproject/onionoo/ClientsStatusUpdater.java | 224 ------
src/org/torproject/onionoo/DateTimeHelper.java | 92 ---
src/org/torproject/onionoo/DescriptorSource.java | 665 -----------------
src/org/torproject/onionoo/DetailsDocument.java | 365 ----------
.../torproject/onionoo/DetailsDocumentWriter.java | 222 ------
src/org/torproject/onionoo/DetailsStatus.java | 141 ----
src/org/torproject/onionoo/Document.java | 24 -
src/org/torproject/onionoo/DocumentStore.java | 744 -------------------
src/org/torproject/onionoo/DocumentWriter.java | 11 -
src/org/torproject/onionoo/GraphHistory.java | 56 --
src/org/torproject/onionoo/LockFile.java | 43 --
src/org/torproject/onionoo/Logger.java | 81 ---
src/org/torproject/onionoo/LookupService.java | 408 -----------
src/org/torproject/onionoo/Main.java | 119 ----
.../onionoo/NodeDetailsStatusUpdater.java | 620 ----------------
src/org/torproject/onionoo/NodeIndexer.java | 425 -----------
src/org/torproject/onionoo/NodeStatus.java | 581 ---------------
src/org/torproject/onionoo/RequestHandler.java | 548 --------------
src/org/torproject/onionoo/ResourceServlet.java | 448 ------------
src/org/torproject/onionoo/ResponseBuilder.java | 311 --------
.../onionoo/ReverseDomainNameResolver.java | 174 -----
src/org/torproject/onionoo/StatusUpdater.java | 11 -
src/org/torproject/onionoo/SummaryDocument.java | 201 ------
.../torproject/onionoo/SummaryDocumentWriter.java | 87 ---
src/org/torproject/onionoo/Time.java | 14 -
src/org/torproject/onionoo/UpdateStatus.java | 7 -
src/org/torproject/onionoo/UptimeDocument.java | 23 -
.../torproject/onionoo/UptimeDocumentWriter.java | 291 --------
src/org/torproject/onionoo/UptimeStatus.java | 226 ------
.../torproject/onionoo/UptimeStatusUpdater.java | 126 ----
src/org/torproject/onionoo/WeightsDocument.java | 64 --
.../torproject/onionoo/WeightsDocumentWriter.java | 222 ------
src/org/torproject/onionoo/WeightsStatus.java | 97 ---
.../torproject/onionoo/WeightsStatusUpdater.java | 328 ---------
src/org/torproject/onionoo/cron/Main.java | 140 ++++
.../torproject/onionoo/docs/BandwidthDocument.java | 27 +
.../torproject/onionoo/docs/BandwidthStatus.java | 80 +++
.../torproject/onionoo/docs/ClientsDocument.java | 22 +
.../onionoo/docs/ClientsGraphHistory.java | 83 +++
.../torproject/onionoo/docs/ClientsHistory.java | 174 +++++
src/org/torproject/onionoo/docs/ClientsStatus.java | 43 ++
.../torproject/onionoo/docs/DetailsDocument.java | 365 ++++++++++
src/org/torproject/onionoo/docs/DetailsStatus.java | 141 ++++
src/org/torproject/onionoo/docs/Document.java | 24 +
src/org/torproject/onionoo/docs/DocumentStore.java | 748 ++++++++++++++++++++
src/org/torproject/onionoo/docs/GraphHistory.java | 56 ++
src/org/torproject/onionoo/docs/NodeStatus.java | 582 +++++++++++++++
.../torproject/onionoo/docs/SummaryDocument.java | 202 ++++++
src/org/torproject/onionoo/docs/UpdateStatus.java | 7 +
.../torproject/onionoo/docs/UptimeDocument.java | 23 +
src/org/torproject/onionoo/docs/UptimeHistory.java | 90 +++
src/org/torproject/onionoo/docs/UptimeStatus.java | 142 ++++
.../torproject/onionoo/docs/WeightsDocument.java | 64 ++
src/org/torproject/onionoo/docs/WeightsStatus.java | 99 +++
src/org/torproject/onionoo/server/NodeIndexer.java | 432 +++++++++++
.../torproject/onionoo/server/RequestHandler.java | 552 +++++++++++++++
.../torproject/onionoo/server/ResourceServlet.java | 451 ++++++++++++
.../torproject/onionoo/server/ResponseBuilder.java | 320 +++++++++
.../onionoo/updater/BandwidthStatusUpdater.java | 149 ++++
.../onionoo/updater/ClientsStatusUpdater.java | 230 ++++++
.../onionoo/updater/DescriptorListener.java | 7 +
.../onionoo/updater/DescriptorSource.java | 646 +++++++++++++++++
.../torproject/onionoo/updater/DescriptorType.java | 15 +
.../onionoo/updater/FingerprintListener.java | 10 +
.../torproject/onionoo/updater/LookupResult.java | 70 ++
.../torproject/onionoo/updater/LookupService.java | 343 +++++++++
.../onionoo/updater/NodeDetailsStatusUpdater.java | 626 ++++++++++++++++
.../onionoo/updater/ReverseDomainNameResolver.java | 179 +++++
.../torproject/onionoo/updater/StatusUpdater.java | 11 +
.../onionoo/updater/UptimeStatusUpdater.java | 130 ++++
.../onionoo/updater/WeightsStatusUpdater.java | 332 +++++++++
.../onionoo/util/ApplicationFactory.java | 55 ++
.../torproject/onionoo/util/DateTimeHelper.java | 92 +++
src/org/torproject/onionoo/util/LockFile.java | 43 ++
src/org/torproject/onionoo/util/Logger.java | 81 +++
src/org/torproject/onionoo/util/Time.java | 14 +
.../onionoo/writer/BandwidthDocumentWriter.java | 201 ++++++
.../onionoo/writer/ClientsDocumentWriter.java | 296 ++++++++
.../onionoo/writer/DetailsDocumentWriter.java | 233 ++++++
.../torproject/onionoo/writer/DocumentWriter.java | 11 +
.../onionoo/writer/SummaryDocumentWriter.java | 94 +++
.../onionoo/writer/UptimeDocumentWriter.java | 303 ++++++++
.../onionoo/writer/WeightsDocumentWriter.java | 233 ++++++
.../torproject/onionoo/DummyDescriptorSource.java | 4 +
.../org/torproject/onionoo/DummyDocumentStore.java | 3 +
test/org/torproject/onionoo/DummyTime.java | 2 +
test/org/torproject/onionoo/LookupServiceTest.java | 2 +
.../torproject/onionoo/ResourceServletTest.java | 44 +-
.../onionoo/UptimeDocumentWriterTest.java | 7 +
test/org/torproject/onionoo/UptimeStatusTest.java | 4 +
.../onionoo/UptimeStatusUpdaterTest.java | 6 +
102 files changed, 9327 insertions(+), 9112 deletions(-)
diff --git a/build.xml b/build.xml
index dccf374..a69bac2 100644
--- a/build.xml
+++ b/build.xml
@@ -93,7 +93,7 @@
<target name="run" depends="compile">
<java fork="true"
maxmemory="4g"
- classname="org.torproject.onionoo.Main"
+ classname="org.torproject.onionoo.cron.Main"
error="errors">
<classpath refid="classpath"/>
</java>
diff --git a/etc/web.xml.template b/etc/web.xml.template
index 988d47a..2c0b280 100644
--- a/etc/web.xml.template
+++ b/etc/web.xml.template
@@ -9,7 +9,7 @@
<servlet>
<servlet-name>Resource</servlet-name>
<servlet-class>
- org.torproject.onionoo.ResourceServlet
+ org.torproject.onionoo.server.ResourceServlet
</servlet-class>
<init-param>
<param-name>maintenance</param-name>
@@ -48,7 +48,7 @@
<listener>
<listener-class>
- org.torproject.onionoo.NodeIndexer
+ org.torproject.onionoo.server.NodeIndexer
</listener-class>
</listener>
</web-app>
diff --git a/src/org/torproject/onionoo/ApplicationFactory.java b/src/org/torproject/onionoo/ApplicationFactory.java
deleted file mode 100644
index 44f2c17..0000000
--- a/src/org/torproject/onionoo/ApplicationFactory.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/* Copyright 2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-public class ApplicationFactory {
-
- private static Time timeInstance;
- public static void setTime(Time time) {
- timeInstance = time;
- }
- public static Time getTime() {
- if (timeInstance == null) {
- timeInstance = new Time();
- }
- return timeInstance;
- }
-
- private static DescriptorSource descriptorSourceInstance;
- public static void setDescriptorSource(
- DescriptorSource descriptorSource) {
- descriptorSourceInstance = descriptorSource;
- }
- public static DescriptorSource getDescriptorSource() {
- if (descriptorSourceInstance == null) {
- descriptorSourceInstance = new DescriptorSource();
- }
- return descriptorSourceInstance;
- }
-
- private static DocumentStore documentStoreInstance;
- public static void setDocumentStore(DocumentStore documentStore) {
- documentStoreInstance = documentStore;
- }
- public static DocumentStore getDocumentStore() {
- if (documentStoreInstance == null) {
- documentStoreInstance = new DocumentStore();
- }
- return documentStoreInstance;
- }
-
- private static NodeIndexer nodeIndexerInstance;
- public static void setNodeIndexer(NodeIndexer nodeIndexer) {
- nodeIndexerInstance = nodeIndexer;
- }
- public static NodeIndexer getNodeIndexer() {
- if (nodeIndexerInstance == null) {
- nodeIndexerInstance = new NodeIndexer();
- }
- return nodeIndexerInstance;
- }
-}
diff --git a/src/org/torproject/onionoo/BandwidthDocument.java b/src/org/torproject/onionoo/BandwidthDocument.java
deleted file mode 100644
index 9c7cd4d..0000000
--- a/src/org/torproject/onionoo/BandwidthDocument.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/* Copyright 2013 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.Map;
-
-public class BandwidthDocument extends Document {
-
- @SuppressWarnings("unused")
- private String fingerprint;
- public void setFingerprint(String fingerprint) {
- this.fingerprint = fingerprint;
- }
-
- @SuppressWarnings("unused")
- private Map<String, GraphHistory> write_history;
- public void setWriteHistory(Map<String, GraphHistory> writeHistory) {
- this.write_history = writeHistory;
- }
-
- @SuppressWarnings("unused")
- private Map<String, GraphHistory> read_history;
- public void setReadHistory(Map<String, GraphHistory> readHistory) {
- this.read_history = readHistory;
- }
-}
-
diff --git a/src/org/torproject/onionoo/BandwidthDocumentWriter.java b/src/org/torproject/onionoo/BandwidthDocumentWriter.java
deleted file mode 100644
index 164ab30..0000000
--- a/src/org/torproject/onionoo/BandwidthDocumentWriter.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/* Copyright 2011--2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.SortedSet;
-
-public class BandwidthDocumentWriter implements FingerprintListener,
- DocumentWriter{
-
- private DescriptorSource descriptorSource;
-
- private DocumentStore documentStore;
-
- private long now;
-
- public BandwidthDocumentWriter() {
- this.descriptorSource = ApplicationFactory.getDescriptorSource();
- this.documentStore = ApplicationFactory.getDocumentStore();
- this.now = ApplicationFactory.getTime().currentTimeMillis();
- this.registerFingerprintListeners();
- }
-
- private void registerFingerprintListeners() {
- this.descriptorSource.registerFingerprintListener(this,
- DescriptorType.RELAY_EXTRA_INFOS);
- this.descriptorSource.registerFingerprintListener(this,
- DescriptorType.BRIDGE_EXTRA_INFOS);
- }
-
- private Set<String> updateBandwidthDocuments = new HashSet<String>();
-
- public void processFingerprints(SortedSet<String> fingerprints,
- boolean relay) {
- this.updateBandwidthDocuments.addAll(fingerprints);
- }
-
- public void writeDocuments() {
- for (String fingerprint : this.updateBandwidthDocuments) {
- BandwidthStatus bandwidthStatus = this.documentStore.retrieve(
- BandwidthStatus.class, true, fingerprint);
- if (bandwidthStatus == null) {
- continue;
- }
- BandwidthDocument bandwidthDocument = this.compileBandwidthDocument(
- fingerprint, bandwidthStatus);
- this.documentStore.store(bandwidthDocument, fingerprint);
- }
- Logger.printStatusTime("Wrote bandwidth document files");
- }
-
-
- private BandwidthDocument compileBandwidthDocument(String fingerprint,
- BandwidthStatus bandwidthStatus) {
- BandwidthDocument bandwidthDocument = new BandwidthDocument();
- bandwidthDocument.setFingerprint(fingerprint);
- bandwidthDocument.setWriteHistory(this.compileGraphType(
- bandwidthStatus.getWriteHistory()));
- bandwidthDocument.setReadHistory(this.compileGraphType(
- bandwidthStatus.getReadHistory()));
- return bandwidthDocument;
- }
-
- private String[] graphNames = new String[] {
- "3_days",
- "1_week",
- "1_month",
- "3_months",
- "1_year",
- "5_years" };
-
- private long[] graphIntervals = new long[] {
- DateTimeHelper.THREE_DAYS,
- DateTimeHelper.ONE_WEEK,
- DateTimeHelper.ROUGHLY_ONE_MONTH,
- DateTimeHelper.ROUGHLY_THREE_MONTHS,
- DateTimeHelper.ROUGHLY_ONE_YEAR,
- DateTimeHelper.ROUGHLY_FIVE_YEARS };
-
- private long[] dataPointIntervals = new long[] {
- DateTimeHelper.FIFTEEN_MINUTES,
- DateTimeHelper.ONE_HOUR,
- DateTimeHelper.FOUR_HOURS,
- DateTimeHelper.TWELVE_HOURS,
- DateTimeHelper.TWO_DAYS,
- DateTimeHelper.TEN_DAYS };
-
- private Map<String, GraphHistory> compileGraphType(
- SortedMap<Long, long[]> history) {
- Map<String, GraphHistory> graphs =
- new LinkedHashMap<String, GraphHistory>();
- for (int i = 0; i < this.graphIntervals.length; i++) {
- String graphName = this.graphNames[i];
- long graphInterval = this.graphIntervals[i];
- long dataPointInterval = this.dataPointIntervals[i];
- List<Long> dataPoints = new ArrayList<Long>();
- long intervalStartMillis = ((this.now - graphInterval)
- / dataPointInterval) * dataPointInterval;
- long totalMillis = 0L, totalBandwidth = 0L;
- for (long[] v : history.values()) {
- long startMillis = v[0], endMillis = v[1], bandwidth = v[2];
- if (endMillis < intervalStartMillis) {
- continue;
- }
- while ((intervalStartMillis / dataPointInterval) !=
- (endMillis / dataPointInterval)) {
- dataPoints.add(totalMillis * 5L < dataPointInterval
- ? -1L : (totalBandwidth * DateTimeHelper.ONE_SECOND)
- / totalMillis);
- totalBandwidth = 0L;
- totalMillis = 0L;
- intervalStartMillis += dataPointInterval;
- }
- totalBandwidth += bandwidth;
- totalMillis += (endMillis - startMillis);
- }
- dataPoints.add(totalMillis * 5L < dataPointInterval
- ? -1L : (totalBandwidth * DateTimeHelper.ONE_SECOND)
- / totalMillis);
- long maxValue = 1L;
- int firstNonNullIndex = -1, lastNonNullIndex = -1;
- for (int j = 0; j < dataPoints.size(); j++) {
- long dataPoint = dataPoints.get(j);
- if (dataPoint >= 0L) {
- if (firstNonNullIndex < 0) {
- firstNonNullIndex = j;
- }
- lastNonNullIndex = j;
- if (dataPoint > maxValue) {
- maxValue = dataPoint;
- }
- }
- }
- if (firstNonNullIndex < 0) {
- continue;
- }
- long firstDataPointMillis = (((this.now - graphInterval)
- / dataPointInterval) + firstNonNullIndex) * dataPointInterval
- + dataPointInterval / 2L;
- if (i > 0 &&
- firstDataPointMillis >= this.now - graphIntervals[i - 1]) {
- /* Skip bandwidth history object, because it doesn't contain
- * anything new that wasn't already contained in the last
- * bandwidth history object(s). */
- continue;
- }
- long lastDataPointMillis = firstDataPointMillis
- + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval;
- double factor = ((double) maxValue) / 999.0;
- int count = lastNonNullIndex - firstNonNullIndex + 1;
- GraphHistory graphHistory = new GraphHistory();
- graphHistory.setFirst(DateTimeHelper.format(firstDataPointMillis));
- graphHistory.setLast(DateTimeHelper.format(lastDataPointMillis));
- graphHistory.setInterval((int) (dataPointInterval
- / DateTimeHelper.ONE_SECOND));
- graphHistory.setFactor(factor);
- graphHistory.setCount(count);
- int previousNonNullIndex = -2;
- boolean foundTwoAdjacentDataPoints = false;
- List<Integer> values = new ArrayList<Integer>();
- for (int j = firstNonNullIndex; j <= lastNonNullIndex; j++) {
- long dataPoint = dataPoints.get(j);
- if (dataPoint >= 0L) {
- if (j - previousNonNullIndex == 1) {
- foundTwoAdjacentDataPoints = true;
- }
- previousNonNullIndex = j;
- }
- values.add(dataPoint < 0L ? null :
- (int) ((dataPoint * 999L) / maxValue));
- }
- graphHistory.setValues(values);
- if (foundTwoAdjacentDataPoints) {
- graphs.put(graphName, graphHistory);
- }
- }
- return graphs;
- }
-
- public String getStatsString() {
- /* TODO Add statistics string. */
- return null;
- }
-}
diff --git a/src/org/torproject/onionoo/BandwidthStatus.java b/src/org/torproject/onionoo/BandwidthStatus.java
deleted file mode 100644
index 3252d67..0000000
--- a/src/org/torproject/onionoo/BandwidthStatus.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/* Copyright 2013--2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.Scanner;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-public class BandwidthStatus extends Document {
-
- private SortedMap<Long, long[]> writeHistory =
- new TreeMap<Long, long[]>();
- public void setWriteHistory(SortedMap<Long, long[]> writeHistory) {
- this.writeHistory = writeHistory;
- }
- public SortedMap<Long, long[]> getWriteHistory() {
- return this.writeHistory;
- }
-
- private SortedMap<Long, long[]> readHistory =
- new TreeMap<Long, long[]>();
- public void setReadHistory(SortedMap<Long, long[]> readHistory) {
- this.readHistory = readHistory;
- }
- public SortedMap<Long, long[]> getReadHistory() {
- return this.readHistory;
- }
-
- public void fromDocumentString(String documentString) {
- Scanner s = new Scanner(documentString);
- while (s.hasNextLine()) {
- String line = s.nextLine();
- String[] parts = line.split(" ");
- if (parts.length != 6) {
- System.err.println("Illegal line '" + line + "' in bandwidth "
- + "history. Skipping this line.");
- continue;
- }
- SortedMap<Long, long[]> history = parts[0].equals("r")
- ? readHistory : writeHistory;
- long startMillis = DateTimeHelper.parse(parts[1] + " " + parts[2]);
- long endMillis = DateTimeHelper.parse(parts[3] + " " + parts[4]);
- if (startMillis < 0L || endMillis < 0L) {
- System.err.println("Could not parse timestamp while reading "
- + "bandwidth history. Skipping.");
- break;
- }
- long bandwidth = Long.parseLong(parts[5]);
- long previousEndMillis = history.headMap(startMillis).isEmpty()
- ? startMillis
- : history.get(history.headMap(startMillis).lastKey())[1];
- long nextStartMillis = history.tailMap(startMillis).isEmpty()
- ? endMillis : history.tailMap(startMillis).firstKey();
- if (previousEndMillis <= startMillis &&
- nextStartMillis >= endMillis) {
- history.put(startMillis, new long[] { startMillis, endMillis,
- bandwidth });
- }
- }
- s.close();
- }
-
- public String toDocumentString() {
- StringBuilder sb = new StringBuilder();
- for (long[] v : writeHistory.values()) {
- sb.append("w " + DateTimeHelper.format(v[0]) + " "
- + DateTimeHelper.format(v[1]) + " " + String.valueOf(v[2])
- + "\n");
- }
- for (long[] v : readHistory.values()) {
- sb.append("r " + DateTimeHelper.format(v[0]) + " "
- + DateTimeHelper.format(v[1]) + " " + String.valueOf(v[2])
- + "\n");
- }
- return sb.toString();
- }
-}
-
diff --git a/src/org/torproject/onionoo/BandwidthStatusUpdater.java b/src/org/torproject/onionoo/BandwidthStatusUpdater.java
deleted file mode 100644
index 6320f6e..0000000
--- a/src/org/torproject/onionoo/BandwidthStatusUpdater.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/* Copyright 2011--2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.ExtraInfoDescriptor;
-
-public class BandwidthStatusUpdater implements DescriptorListener,
- StatusUpdater {
-
- private DescriptorSource descriptorSource;
-
- private DocumentStore documentStore;
-
- private long now;
-
- public BandwidthStatusUpdater() {
- this.descriptorSource = ApplicationFactory.getDescriptorSource();
- this.documentStore = ApplicationFactory.getDocumentStore();
- this.now = ApplicationFactory.getTime().currentTimeMillis();
- this.registerDescriptorListeners();
- }
-
- private void registerDescriptorListeners() {
- this.descriptorSource.registerDescriptorListener(this,
- DescriptorType.RELAY_EXTRA_INFOS);
- this.descriptorSource.registerDescriptorListener(this,
- DescriptorType.BRIDGE_EXTRA_INFOS);
- }
-
- public void processDescriptor(Descriptor descriptor, boolean relay) {
- if (descriptor instanceof ExtraInfoDescriptor) {
- this.parseDescriptor((ExtraInfoDescriptor) descriptor);
- }
- }
-
- public void updateStatuses() {
- /* Status files are already updated while processing descriptors. */
- }
-
- private void parseDescriptor(ExtraInfoDescriptor descriptor) {
- String fingerprint = descriptor.getFingerprint();
- BandwidthStatus bandwidthStatus = this.documentStore.retrieve(
- BandwidthStatus.class, true, fingerprint);
- if (bandwidthStatus == null) {
- bandwidthStatus = new BandwidthStatus();
- }
- if (descriptor.getWriteHistory() != null) {
- parseHistoryLine(descriptor.getWriteHistory().getLine(),
- bandwidthStatus.getWriteHistory());
- }
- if (descriptor.getReadHistory() != null) {
- parseHistoryLine(descriptor.getReadHistory().getLine(),
- bandwidthStatus.getReadHistory());
- }
- this.compressHistory(bandwidthStatus.getWriteHistory());
- this.compressHistory(bandwidthStatus.getReadHistory());
- this.documentStore.store(bandwidthStatus, fingerprint);
- }
-
- private void parseHistoryLine(String line,
- SortedMap<Long, long[]> history) {
- String[] parts = line.split(" ");
- if (parts.length < 6) {
- return;
- }
- long endMillis = DateTimeHelper.parse(parts[1] + " " + parts[2]);
- if (endMillis < 0L) {
- System.err.println("Could not parse timestamp in line '" + line
- + "'. Skipping.");
- return;
- }
- long intervalMillis = Long.parseLong(parts[3].substring(1))
- * DateTimeHelper.ONE_SECOND;
- String[] values = parts[5].split(",");
- for (int i = values.length - 1; i >= 0; i--) {
- long bandwidthValue = Long.parseLong(values[i]);
- long startMillis = endMillis - intervalMillis;
- /* TODO Should we first check whether an interval is already
- * contained in history? */
- history.put(startMillis, new long[] { startMillis, endMillis,
- bandwidthValue });
- endMillis -= intervalMillis;
- }
- }
-
- private void compressHistory(SortedMap<Long, long[]> history) {
- SortedMap<Long, long[]> uncompressedHistory =
- new TreeMap<Long, long[]>(history);
- history.clear();
- long lastStartMillis = 0L, lastEndMillis = 0L, lastBandwidth = 0L;
- String lastMonthString = "1970-01";
- for (long[] v : uncompressedHistory.values()) {
- long startMillis = v[0], endMillis = v[1], bandwidth = v[2];
- long intervalLengthMillis;
- if (this.now - endMillis <= DateTimeHelper.THREE_DAYS) {
- intervalLengthMillis = DateTimeHelper.FIFTEEN_MINUTES;
- } else if (this.now - endMillis <= DateTimeHelper.ONE_WEEK) {
- intervalLengthMillis = DateTimeHelper.ONE_HOUR;
- } else if (this.now - endMillis <=
- DateTimeHelper.ROUGHLY_ONE_MONTH) {
- intervalLengthMillis = DateTimeHelper.FOUR_HOURS;
- } else if (this.now - endMillis <=
- DateTimeHelper.ROUGHLY_THREE_MONTHS) {
- intervalLengthMillis = DateTimeHelper.TWELVE_HOURS;
- } else if (this.now - endMillis <=
- DateTimeHelper.ROUGHLY_ONE_YEAR) {
- intervalLengthMillis = DateTimeHelper.TWO_DAYS;
- } else {
- intervalLengthMillis = DateTimeHelper.TEN_DAYS;
- }
- String monthString = DateTimeHelper.format(startMillis,
- DateTimeHelper.ISO_YEARMONTH_FORMAT);
- if (lastEndMillis == startMillis &&
- ((lastEndMillis - 1L) / intervalLengthMillis) ==
- ((endMillis - 1L) / intervalLengthMillis) &&
- lastMonthString.equals(monthString)) {
- lastEndMillis = endMillis;
- lastBandwidth += bandwidth;
- } else {
- if (lastStartMillis > 0L) {
- history.put(lastStartMillis, new long[] { lastStartMillis,
- lastEndMillis, lastBandwidth });
- }
- lastStartMillis = startMillis;
- lastEndMillis = endMillis;
- lastBandwidth = bandwidth;
- }
- lastMonthString = monthString;
- }
- if (lastStartMillis > 0L) {
- history.put(lastStartMillis, new long[] { lastStartMillis,
- lastEndMillis, lastBandwidth });
- }
- }
-
- public String getStatsString() {
- /* TODO Add statistics string. */
- return null;
- }
-}
-
diff --git a/src/org/torproject/onionoo/ClientsDocument.java b/src/org/torproject/onionoo/ClientsDocument.java
deleted file mode 100644
index e8e40ee..0000000
--- a/src/org/torproject/onionoo/ClientsDocument.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/* Copyright 2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.Map;
-
-public class ClientsDocument extends Document {
-
- @SuppressWarnings("unused")
- private String fingerprint;
- public void setFingerprint(String fingerprint) {
- this.fingerprint = fingerprint;
- }
-
- @SuppressWarnings("unused")
- private Map<String, ClientsGraphHistory> average_clients;
- public void setAverageClients(
- Map<String, ClientsGraphHistory> averageClients) {
- this.average_clients = averageClients;
- }
-}
-
diff --git a/src/org/torproject/onionoo/ClientsDocumentWriter.java b/src/org/torproject/onionoo/ClientsDocumentWriter.java
deleted file mode 100644
index 1fced6d..0000000
--- a/src/org/torproject/onionoo/ClientsDocumentWriter.java
+++ /dev/null
@@ -1,284 +0,0 @@
-/* Copyright 2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-/*
- * Clients status file produced as intermediate output:
- *
- * 2014-02-15 16:42:11 2014-02-16 00:00:00
- * 259.042 in=86.347,se=86.347 v4=259.042
- * 2014-02-16 00:00:00 2014-02-16 16:42:11
- * 592.958 in=197.653,se=197.653 v4=592.958
- *
- * Clients document file produced as output:
- *
- * "1_month":{
- * "first":"2014-02-03 12:00:00",
- * "last":"2014-02-28 12:00:00",
- * "interval":86400,
- * "factor":0.139049349,
- * "count":26,
- * "values":[371,354,349,374,432,null,485,458,493,536,null,null,524,576,
- * 607,622,null,635,null,566,774,999,945,690,656,681],
- * "countries":{"cn":0.0192,"in":0.1768,"ir":0.2487,"ru":0.0104,
- * "se":0.1698,"sy":0.0325,"us":0.0406},
- * "transports":{"obfs2":0.4581},
- * "versions":{"v4":1.0000}}
- */
-public class ClientsDocumentWriter implements FingerprintListener,
- DocumentWriter {
-
- private DescriptorSource descriptorSource;
-
- private DocumentStore documentStore;
-
- private long now;
-
- public ClientsDocumentWriter() {
- this.descriptorSource = ApplicationFactory.getDescriptorSource();
- this.documentStore = ApplicationFactory.getDocumentStore();
- this.now = ApplicationFactory.getTime().currentTimeMillis();
- this.registerFingerprintListeners();
- }
-
- private void registerFingerprintListeners() {
- this.descriptorSource.registerFingerprintListener(this,
- DescriptorType.BRIDGE_EXTRA_INFOS);
- }
-
- private SortedSet<String> updateDocuments = new TreeSet<String>();
-
- public void processFingerprints(SortedSet<String> fingerprints,
- boolean relay) {
- if (!relay) {
- this.updateDocuments.addAll(fingerprints);
- }
- }
-
- private int writtenDocuments = 0;
-
- public void writeDocuments() {
- for (String hashedFingerprint : this.updateDocuments) {
- ClientsStatus clientsStatus = this.documentStore.retrieve(
- ClientsStatus.class, true, hashedFingerprint);
- if (clientsStatus == null) {
- continue;
- }
- SortedSet<ClientsHistory> history = clientsStatus.getHistory();
- ClientsDocument clientsDocument = this.compileClientsDocument(
- hashedFingerprint, history);
- this.documentStore.store(clientsDocument, hashedFingerprint);
- this.writtenDocuments++;
- }
- Logger.printStatusTime("Wrote clients document files");
- }
-
- private String[] graphNames = new String[] {
- "1_week",
- "1_month",
- "3_months",
- "1_year",
- "5_years" };
-
- private long[] graphIntervals = new long[] {
- DateTimeHelper.ONE_WEEK,
- DateTimeHelper.ROUGHLY_ONE_MONTH,
- DateTimeHelper.ROUGHLY_THREE_MONTHS,
- DateTimeHelper.ROUGHLY_ONE_YEAR,
- DateTimeHelper.ROUGHLY_FIVE_YEARS };
-
- private long[] dataPointIntervals = new long[] {
- DateTimeHelper.ONE_DAY,
- DateTimeHelper.ONE_DAY,
- DateTimeHelper.ONE_DAY,
- DateTimeHelper.TWO_DAYS,
- DateTimeHelper.TEN_DAYS };
-
- private ClientsDocument compileClientsDocument(String hashedFingerprint,
- SortedSet<ClientsHistory> history) {
- ClientsDocument clientsDocument = new ClientsDocument();
- clientsDocument.setFingerprint(hashedFingerprint);
- Map<String, ClientsGraphHistory> averageClients =
- new LinkedHashMap<String, ClientsGraphHistory>();
- for (int graphIntervalIndex = 0; graphIntervalIndex <
- this.graphIntervals.length; graphIntervalIndex++) {
- String graphName = this.graphNames[graphIntervalIndex];
- ClientsGraphHistory graphHistory = this.compileClientsHistory(
- graphIntervalIndex, history);
- if (graphHistory != null) {
- averageClients.put(graphName, graphHistory);
- }
- }
- clientsDocument.setAverageClients(averageClients);
- return clientsDocument;
- }
-
- private ClientsGraphHistory compileClientsHistory(
- int graphIntervalIndex, SortedSet<ClientsHistory> history) {
- long graphInterval = this.graphIntervals[graphIntervalIndex];
- long dataPointInterval =
- this.dataPointIntervals[graphIntervalIndex];
- List<Double> dataPoints = new ArrayList<Double>();
- long intervalStartMillis = ((this.now - graphInterval)
- / dataPointInterval) * dataPointInterval;
- long millis = 0L;
- double responses = 0.0, totalResponses = 0.0;
- SortedMap<String, Double>
- totalResponsesByCountry = new TreeMap<String, Double>(),
- totalResponsesByTransport = new TreeMap<String, Double>(),
- totalResponsesByVersion = new TreeMap<String, Double>();
- for (ClientsHistory hist : history) {
- if (hist.getEndMillis() < intervalStartMillis) {
- continue;
- }
- while ((intervalStartMillis / dataPointInterval) !=
- (hist.getEndMillis() / dataPointInterval)) {
- dataPoints.add(millis * 2L < dataPointInterval
- ? -1.0 : responses * ((double) DateTimeHelper.ONE_DAY)
- / (((double) millis) * 10.0));
- responses = 0.0;
- millis = 0L;
- intervalStartMillis += dataPointInterval;
- }
- responses += hist.getTotalResponses();
- totalResponses += hist.getTotalResponses();
- for (Map.Entry<String, Double> e :
- hist.getResponsesByCountry().entrySet()) {
- if (!totalResponsesByCountry.containsKey(e.getKey())) {
- totalResponsesByCountry.put(e.getKey(), 0.0);
- }
- totalResponsesByCountry.put(e.getKey(), e.getValue()
- + totalResponsesByCountry.get(e.getKey()));
- }
- for (Map.Entry<String, Double> e :
- hist.getResponsesByTransport().entrySet()) {
- if (!totalResponsesByTransport.containsKey(e.getKey())) {
- totalResponsesByTransport.put(e.getKey(), 0.0);
- }
- totalResponsesByTransport.put(e.getKey(), e.getValue()
- + totalResponsesByTransport.get(e.getKey()));
- }
- for (Map.Entry<String, Double> e :
- hist.getResponsesByVersion().entrySet()) {
- if (!totalResponsesByVersion.containsKey(e.getKey())) {
- totalResponsesByVersion.put(e.getKey(), 0.0);
- }
- totalResponsesByVersion.put(e.getKey(), e.getValue()
- + totalResponsesByVersion.get(e.getKey()));
- }
- millis += (hist.getEndMillis() - hist.getStartMillis());
- }
- dataPoints.add(millis * 2L < dataPointInterval
- ? -1.0 : responses * ((double) DateTimeHelper.ONE_DAY)
- / (((double) millis) * 10.0));
- double maxValue = 0.0;
- int firstNonNullIndex = -1, lastNonNullIndex = -1;
- for (int dataPointIndex = 0; dataPointIndex < dataPoints.size();
- dataPointIndex++) {
- double dataPoint = dataPoints.get(dataPointIndex);
- if (dataPoint >= 0.0) {
- if (firstNonNullIndex < 0) {
- firstNonNullIndex = dataPointIndex;
- }
- lastNonNullIndex = dataPointIndex;
- if (dataPoint > maxValue) {
- maxValue = dataPoint;
- }
- }
- }
- if (firstNonNullIndex < 0) {
- return null;
- }
- long firstDataPointMillis = (((this.now - graphInterval)
- / dataPointInterval) + firstNonNullIndex) * dataPointInterval
- + dataPointInterval / 2L;
- if (graphIntervalIndex > 0 && firstDataPointMillis >=
- this.now - graphIntervals[graphIntervalIndex - 1]) {
- /* Skip clients history object, because it doesn't contain
- * anything new that wasn't already contained in the last
- * clients history object(s). */
- return null;
- }
- long lastDataPointMillis = firstDataPointMillis
- + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval;
- double factor = ((double) maxValue) / 999.0;
- int count = lastNonNullIndex - firstNonNullIndex + 1;
- ClientsGraphHistory graphHistory = new ClientsGraphHistory();
- graphHistory.setFirst(DateTimeHelper.format(firstDataPointMillis));
- graphHistory.setLast(DateTimeHelper.format(lastDataPointMillis));
- graphHistory.setInterval((int) (dataPointInterval
- / DateTimeHelper.ONE_SECOND));
- graphHistory.setFactor(factor);
- graphHistory.setCount(count);
- int previousNonNullIndex = -2;
- boolean foundTwoAdjacentDataPoints = false;
- List<Integer> values = new ArrayList<Integer>();
- for (int dataPointIndex = firstNonNullIndex; dataPointIndex <=
- lastNonNullIndex; dataPointIndex++) {
- double dataPoint = dataPoints.get(dataPointIndex);
- if (dataPoint >= 0.0) {
- if (dataPointIndex - previousNonNullIndex == 1) {
- foundTwoAdjacentDataPoints = true;
- }
- previousNonNullIndex = dataPointIndex;
- }
- values.add(dataPoint < 0.0 ? null :
- (int) ((dataPoint * 999.0) / maxValue));
- }
- graphHistory.setValues(values);
- if (!totalResponsesByCountry.isEmpty()) {
- SortedMap<String, Float> countries = new TreeMap<String, Float>();
- for (Map.Entry<String, Double> e :
- totalResponsesByCountry.entrySet()) {
- if (e.getValue() > totalResponses / 100.0) {
- countries.put(e.getKey(),
- (float) (e.getValue() / totalResponses));
- }
- }
- graphHistory.setCountries(countries);
- }
- if (!totalResponsesByTransport.isEmpty()) {
- SortedMap<String, Float> transports = new TreeMap<String, Float>();
- for (Map.Entry<String, Double> e :
- totalResponsesByTransport.entrySet()) {
- if (e.getValue() > totalResponses / 100.0) {
- transports.put(e.getKey(),
- (float) (e.getValue() / totalResponses));
- }
- }
- graphHistory.setTransports(transports);
- }
- if (!totalResponsesByVersion.isEmpty()) {
- SortedMap<String, Float> versions = new TreeMap<String, Float>();
- for (Map.Entry<String, Double> e :
- totalResponsesByVersion.entrySet()) {
- if (e.getValue() > totalResponses / 100.0) {
- versions.put(e.getKey(),
- (float) (e.getValue() / totalResponses));
- }
- }
- graphHistory.setVersions(versions);
- }
- if (foundTwoAdjacentDataPoints) {
- return graphHistory;
- } else {
- return null;
- }
- }
-
- public String getStatsString() {
- StringBuilder sb = new StringBuilder();
- sb.append(" " + Logger.formatDecimalNumber(this.writtenDocuments)
- + " clients document files updated\n");
- return sb.toString();
- }
-}
diff --git a/src/org/torproject/onionoo/ClientsGraphHistory.java b/src/org/torproject/onionoo/ClientsGraphHistory.java
deleted file mode 100644
index b7b312b..0000000
--- a/src/org/torproject/onionoo/ClientsGraphHistory.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/* Copyright 2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.SortedMap;
-
-public class ClientsGraphHistory {
-
- private String first;
- public void setFirst(String first) {
- this.first = first;
- }
- public String getFirst() {
- return this.first;
- }
-
- private String last;
- public void setLast(String last) {
- this.last = last;
- }
- public String getLast() {
- return this.last;
- }
-
- private Integer interval;
- public void setInterval(Integer interval) {
- this.interval = interval;
- }
- public Integer getInterval() {
- return this.interval;
- }
-
- private Double factor;
- public void setFactor(Double factor) {
- this.factor = factor;
- }
- public Double getFactor() {
- return this.factor;
- }
-
- private Integer count;
- public void setCount(Integer count) {
- this.count = count;
- }
- public Integer getCount() {
- return this.count;
- }
-
- private List<Integer> values = new ArrayList<Integer>();
- public void setValues(List<Integer> values) {
- this.values = values;
- }
- public List<Integer> getValues() {
- return this.values;
- }
-
- private SortedMap<String, Float> countries;
- public void setCountries(SortedMap<String, Float> countries) {
- this.countries = countries;
- }
- public SortedMap<String, Float> getCountries() {
- return this.countries;
- }
-
- private SortedMap<String, Float> transports;
- public void setTransports(SortedMap<String, Float> transports) {
- this.transports = transports;
- }
- public SortedMap<String, Float> getTransports() {
- return this.transports;
- }
-
- private SortedMap<String, Float> versions;
- public void setVersions(SortedMap<String, Float> versions) {
- this.versions = versions;
- }
- public SortedMap<String, Float> getVersions() {
- return this.versions;
- }
-}
-
diff --git a/src/org/torproject/onionoo/ClientsStatus.java b/src/org/torproject/onionoo/ClientsStatus.java
deleted file mode 100644
index 1ea16a7..0000000
--- a/src/org/torproject/onionoo/ClientsStatus.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/* Copyright 2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.Map;
-import java.util.Scanner;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-class ClientsHistory implements Comparable<ClientsHistory> {
-
- private long startMillis;
- public long getStartMillis() {
- return this.startMillis;
- }
-
- private long endMillis;
- public long getEndMillis() {
- return this.endMillis;
- }
-
- private double totalResponses;
- public double getTotalResponses() {
- return this.totalResponses;
- }
-
- private SortedMap<String, Double> responsesByCountry;
- public SortedMap<String, Double> getResponsesByCountry() {
- return this.responsesByCountry;
- }
-
- private SortedMap<String, Double> responsesByTransport;
- public SortedMap<String, Double> getResponsesByTransport() {
- return this.responsesByTransport;
- }
-
- private SortedMap<String, Double> responsesByVersion;
- public SortedMap<String, Double> getResponsesByVersion() {
- return this.responsesByVersion;
- }
-
- ClientsHistory(long startMillis, long endMillis,
- double totalResponses,
- SortedMap<String, Double> responsesByCountry,
- SortedMap<String, Double> responsesByTransport,
- SortedMap<String, Double> responsesByVersion) {
- this.startMillis = startMillis;
- this.endMillis = endMillis;
- this.totalResponses = totalResponses;
- this.responsesByCountry = responsesByCountry;
- this.responsesByTransport = responsesByTransport;
- this.responsesByVersion = responsesByVersion;
- }
-
- public static ClientsHistory fromString(
- String responseHistoryString) {
- String[] parts = responseHistoryString.split(" ", 8);
- if (parts.length != 8) {
- return null;
- }
- long startMillis = DateTimeHelper.parse(parts[0] + " " + parts[1]);
- long endMillis = DateTimeHelper.parse(parts[2] + " " + parts[3]);
- if (startMillis < 0L || endMillis < 0L) {
- return null;
- }
- if (startMillis >= endMillis) {
- return null;
- }
- double totalResponses = 0.0;
- try {
- totalResponses = Double.parseDouble(parts[4]);
- } catch (NumberFormatException e) {
- return null;
- }
- SortedMap<String, Double> responsesByCountry =
- parseResponses(parts[5]);
- SortedMap<String, Double> responsesByTransport =
- parseResponses(parts[6]);
- SortedMap<String, Double> responsesByVersion =
- parseResponses(parts[7]);
- if (responsesByCountry == null || responsesByTransport == null ||
- responsesByVersion == null) {
- return null;
- }
- return new ClientsHistory(startMillis, endMillis, totalResponses,
- responsesByCountry, responsesByTransport, responsesByVersion);
- }
-
- private static SortedMap<String, Double> parseResponses(
- String responsesString) {
- SortedMap<String, Double> responses = new TreeMap<String, Double>();
- if (responsesString.length() > 0) {
- for (String pair : responsesString.split(",")) {
- String[] keyValue = pair.split("=");
- if (keyValue.length != 2) {
- return null;
- }
- double value = 0.0;
- try {
- value = Double.parseDouble(keyValue[1]);
- } catch (NumberFormatException e) {
- return null;
- }
- responses.put(keyValue[0], value);
- }
- }
- return responses;
- }
-
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append(DateTimeHelper.format(startMillis));
- sb.append(" " + DateTimeHelper.format(endMillis));
- sb.append(" " + String.format("%.3f", this.totalResponses));
- this.appendResponses(sb, this.responsesByCountry);
- this.appendResponses(sb, this.responsesByTransport);
- this.appendResponses(sb, this.responsesByVersion);
- return sb.toString();
- }
-
- private void appendResponses(StringBuilder sb,
- SortedMap<String, Double> responses) {
- sb.append(" ");
- int written = 0;
- for (Map.Entry<String, Double> e : responses.entrySet()) {
- sb.append((written++ > 0 ? "," : "") + e.getKey() + "="
- + String.format("%.3f", e.getValue()));
- }
- }
-
- public void addResponses(ClientsHistory other) {
- this.totalResponses += other.totalResponses;
- this.addResponsesByCategory(this.responsesByCountry,
- other.responsesByCountry);
- this.addResponsesByCategory(this.responsesByTransport,
- other.responsesByTransport);
- this.addResponsesByCategory(this.responsesByVersion,
- other.responsesByVersion);
- if (this.startMillis > other.startMillis) {
- this.startMillis = other.startMillis;
- }
- if (this.endMillis < other.endMillis) {
- this.endMillis = other.endMillis;
- }
- }
-
- private void addResponsesByCategory(
- SortedMap<String, Double> thisResponses,
- SortedMap<String, Double> otherResponses) {
- for (Map.Entry<String, Double> e : otherResponses.entrySet()) {
- if (thisResponses.containsKey(e.getKey())) {
- thisResponses.put(e.getKey(), thisResponses.get(e.getKey())
- + e.getValue());
- } else {
- thisResponses.put(e.getKey(), e.getValue());
- }
- }
- }
-
- public int compareTo(ClientsHistory other) {
- return this.startMillis < other.startMillis ? -1 :
- this.startMillis > other.startMillis ? 1 : 0;
- }
-
- public boolean equals(Object other) {
- return other instanceof ClientsHistory &&
- this.startMillis == ((ClientsHistory) other).startMillis;
- }
-
- public int hashCode() {
- return (int) this.startMillis;
- }
-}
-
-public class ClientsStatus extends Document {
-
- private SortedSet<ClientsHistory> history =
- new TreeSet<ClientsHistory>();
- public void setHistory(SortedSet<ClientsHistory> history) {
- this.history = history;
- }
- public SortedSet<ClientsHistory> getHistory() {
- return this.history;
- }
-
- public void fromDocumentString(String documentString) {
- Scanner s = new Scanner(documentString);
- while (s.hasNextLine()) {
- String line = s.nextLine();
- ClientsHistory parsedLine = ClientsHistory.fromString(line);
- if (parsedLine != null) {
- this.history.add(parsedLine);
- } else {
- System.err.println("Could not parse clients history line '"
- + line + "'. Skipping.");
- }
- }
- s.close();
- }
-
- public String toDocumentString() {
- StringBuilder sb = new StringBuilder();
- for (ClientsHistory interval : this.history) {
- sb.append(interval.toString() + "\n");
- }
- return sb.toString();
- }
-}
-
diff --git a/src/org/torproject/onionoo/ClientsStatusUpdater.java b/src/org/torproject/onionoo/ClientsStatusUpdater.java
deleted file mode 100644
index 4fe2b2d..0000000
--- a/src/org/torproject/onionoo/ClientsStatusUpdater.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/* Copyright 2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.ExtraInfoDescriptor;
-
-/*
- * Example extra-info descriptor used as input:
- *
- * extra-info ndnop2 DE6397A047ABE5F78B4C87AF725047831B221AAB
- * dirreq-stats-end 2014-02-16 16:42:11 (86400 s)
- * dirreq-v3-resp ok=856,not-enough-sigs=0,unavailable=0,not-found=0,
- * not-modified=40,busy=0
- * bridge-stats-end 2014-02-16 16:42:17 (86400 s)
- * bridge-ips ??=8,in=8,se=8
- * bridge-ip-versions v4=8,v6=0
- *
- * Clients status file produced as intermediate output:
- *
- * 2014-02-15 16:42:11 2014-02-16 00:00:00
- * 259.042 in=86.347,se=86.347 v4=259.042
- * 2014-02-16 00:00:00 2014-02-16 16:42:11
- * 592.958 in=197.653,se=197.653 v4=592.958
- */
-public class ClientsStatusUpdater implements DescriptorListener,
- StatusUpdater {
-
- private DescriptorSource descriptorSource;
-
- private DocumentStore documentStore;
-
- private long now;
-
- public ClientsStatusUpdater() {
- this.descriptorSource = ApplicationFactory.getDescriptorSource();
- this.documentStore = ApplicationFactory.getDocumentStore();
- this.now = ApplicationFactory.getTime().currentTimeMillis();
- this.registerDescriptorListeners();
- }
-
- private void registerDescriptorListeners() {
- this.descriptorSource.registerDescriptorListener(this,
- DescriptorType.BRIDGE_EXTRA_INFOS);
- }
-
- public void processDescriptor(Descriptor descriptor, boolean relay) {
- if (descriptor instanceof ExtraInfoDescriptor && !relay) {
- this.processBridgeExtraInfoDescriptor(
- (ExtraInfoDescriptor) descriptor);
- }
- }
-
- private SortedMap<String, SortedSet<ClientsHistory>> newResponses =
- new TreeMap<String, SortedSet<ClientsHistory>>();
-
- private void processBridgeExtraInfoDescriptor(
- ExtraInfoDescriptor descriptor) {
- long dirreqStatsEndMillis = descriptor.getDirreqStatsEndMillis();
- long dirreqStatsIntervalLengthMillis =
- descriptor.getDirreqStatsIntervalLength()
- * DateTimeHelper.ONE_SECOND;
- SortedMap<String, Integer> responses = descriptor.getDirreqV3Resp();
- if (dirreqStatsEndMillis < 0L ||
- dirreqStatsIntervalLengthMillis != DateTimeHelper.ONE_DAY ||
- responses == null || !responses.containsKey("ok")) {
- return;
- }
- double okResponses = (double) (responses.get("ok") - 4);
- if (okResponses < 0.0) {
- return;
- }
- String hashedFingerprint = descriptor.getFingerprint().toUpperCase();
- long dirreqStatsStartMillis = dirreqStatsEndMillis
- - dirreqStatsIntervalLengthMillis;
- long utcBreakMillis = (dirreqStatsEndMillis / DateTimeHelper.ONE_DAY)
- * DateTimeHelper.ONE_DAY;
- for (int i = 0; i < 2; i++) {
- long startMillis = i == 0 ? dirreqStatsStartMillis : utcBreakMillis;
- long endMillis = i == 0 ? utcBreakMillis : dirreqStatsEndMillis;
- if (startMillis >= endMillis) {
- continue;
- }
- double totalResponses = okResponses
- * ((double) (endMillis - startMillis))
- / ((double) DateTimeHelper.ONE_DAY);
- SortedMap<String, Double> responsesByCountry =
- this.weightResponsesWithUniqueIps(totalResponses,
- descriptor.getBridgeIps(), "??");
- SortedMap<String, Double> responsesByTransport =
- this.weightResponsesWithUniqueIps(totalResponses,
- descriptor.getBridgeIpTransports(), "<??>");
- SortedMap<String, Double> responsesByVersion =
- this.weightResponsesWithUniqueIps(totalResponses,
- descriptor.getBridgeIpVersions(), "");
- ClientsHistory newResponseHistory = new ClientsHistory(
- startMillis, endMillis, totalResponses, responsesByCountry,
- responsesByTransport, responsesByVersion);
- if (!this.newResponses.containsKey(hashedFingerprint)) {
- this.newResponses.put(hashedFingerprint,
- new TreeSet<ClientsHistory>());
- }
- this.newResponses.get(hashedFingerprint).add(
- newResponseHistory);
- }
- }
-
- private SortedMap<String, Double> weightResponsesWithUniqueIps(
- double totalResponses, SortedMap<String, Integer> uniqueIps,
- String omitString) {
- SortedMap<String, Double> weightedResponses =
- new TreeMap<String, Double>();
- int totalUniqueIps = 0;
- if (uniqueIps != null) {
- for (Map.Entry<String, Integer> e : uniqueIps.entrySet()) {
- if (e.getValue() > 4) {
- totalUniqueIps += e.getValue() - 4;
- }
- }
- }
- if (totalUniqueIps > 0) {
- for (Map.Entry<String, Integer> e : uniqueIps.entrySet()) {
- if (!e.getKey().equals(omitString) && e.getValue() > 4) {
- weightedResponses.put(e.getKey(),
- (((double) (e.getValue() - 4)) * totalResponses)
- / ((double) totalUniqueIps));
- }
- }
- }
- return weightedResponses;
- }
-
- public void updateStatuses() {
- for (Map.Entry<String, SortedSet<ClientsHistory>> e :
- this.newResponses.entrySet()) {
- String hashedFingerprint = e.getKey();
- ClientsStatus clientsStatus = this.documentStore.retrieve(
- ClientsStatus.class, true, hashedFingerprint);
- if (clientsStatus == null) {
- clientsStatus = new ClientsStatus();
- }
- this.addToHistory(clientsStatus, e.getValue());
- this.compressHistory(clientsStatus);
- this.documentStore.store(clientsStatus, hashedFingerprint);
- }
- }
-
- private void addToHistory(ClientsStatus clientsStatus,
- SortedSet<ClientsHistory> newIntervals) {
- SortedSet<ClientsHistory> history = clientsStatus.getHistory();
- for (ClientsHistory interval : newIntervals) {
- if ((history.headSet(interval).isEmpty() ||
- history.headSet(interval).last().getEndMillis() <=
- interval.getStartMillis()) &&
- (history.tailSet(interval).isEmpty() ||
- history.tailSet(interval).first().getStartMillis() >=
- interval.getEndMillis())) {
- history.add(interval);
- }
- }
- }
-
- private void compressHistory(ClientsStatus clientsStatus) {
- SortedSet<ClientsHistory> history = clientsStatus.getHistory();
- SortedSet<ClientsHistory> compressedHistory =
- new TreeSet<ClientsHistory>();
- ClientsHistory lastResponses = null;
- String lastMonthString = "1970-01";
- for (ClientsHistory responses : history) {
- long intervalLengthMillis;
- if (this.now - responses.getEndMillis() <=
- DateTimeHelper.ROUGHLY_THREE_MONTHS) {
- intervalLengthMillis = DateTimeHelper.ONE_DAY;
- } else if (this.now - responses.getEndMillis() <=
- DateTimeHelper.ROUGHLY_ONE_YEAR) {
- intervalLengthMillis = DateTimeHelper.TWO_DAYS;
- } else {
- intervalLengthMillis = DateTimeHelper.TEN_DAYS;
- }
- String monthString = DateTimeHelper.format(
- responses.getStartMillis(),
- DateTimeHelper.ISO_YEARMONTH_FORMAT);
- if (lastResponses != null &&
- lastResponses.getEndMillis() == responses.getStartMillis() &&
- ((lastResponses.getEndMillis() - 1L) / intervalLengthMillis) ==
- ((responses.getEndMillis() - 1L) / intervalLengthMillis) &&
- lastMonthString.equals(monthString)) {
- lastResponses.addResponses(responses);
- } else {
- if (lastResponses != null) {
- compressedHistory.add(lastResponses);
- }
- lastResponses = responses;
- }
- lastMonthString = monthString;
- }
- if (lastResponses != null) {
- compressedHistory.add(lastResponses);
- }
- clientsStatus.setHistory(compressedHistory);
- }
-
- public String getStatsString() {
- int newIntervals = 0;
- for (SortedSet<ClientsHistory> hist : this.newResponses.values()) {
- newIntervals += hist.size();
- }
- StringBuilder sb = new StringBuilder();
- sb.append(" "
- + Logger.formatDecimalNumber(newIntervals / 2)
- + " client statistics processed from extra-info descriptors\n");
- sb.append(" "
- + Logger.formatDecimalNumber(this.newResponses.size())
- + " client status files updated\n");
- return sb.toString();
- }
-}
-
diff --git a/src/org/torproject/onionoo/DateTimeHelper.java b/src/org/torproject/onionoo/DateTimeHelper.java
deleted file mode 100644
index 41ac70b..0000000
--- a/src/org/torproject/onionoo/DateTimeHelper.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/* Copyright 2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.TimeZone;
-
-public class DateTimeHelper {
-
- private DateTimeHelper() {
- }
-
- public static final long ONE_SECOND = 1000L,
- TEN_SECONDS = 10L * ONE_SECOND,
- ONE_MINUTE = 60L * ONE_SECOND,
- FIVE_MINUTES = 5L * ONE_MINUTE,
- FIFTEEN_MINUTES = 15L * ONE_MINUTE,
- ONE_HOUR = 60L * ONE_MINUTE,
- FOUR_HOURS = 4L * ONE_HOUR,
- SIX_HOURS = 6L * ONE_HOUR,
- TWELVE_HOURS = 12L * ONE_HOUR,
- ONE_DAY = 24L * ONE_HOUR,
- TWO_DAYS = 2L * ONE_DAY,
- THREE_DAYS = 3L * ONE_DAY,
- ONE_WEEK = 7L * ONE_DAY,
- TEN_DAYS = 10L * ONE_DAY,
- ROUGHLY_ONE_MONTH = 31L * ONE_DAY,
- ROUGHLY_THREE_MONTHS = 92L * ONE_DAY,
- ROUGHLY_ONE_YEAR = 366L * ONE_DAY,
- ROUGHLY_FIVE_YEARS = 5L * ROUGHLY_ONE_YEAR;
-
- public static final String ISO_DATETIME_FORMAT = "yyyy-MM-dd HH:mm:ss";
-
- public static final String ISO_DATETIME_TAB_FORMAT =
- "yyyy-MM-dd\tHH:mm:ss";
-
- public static final String ISO_YEARMONTH_FORMAT = "yyyy-MM";
-
- public static final String DATEHOUR_NOSPACE_FORMAT = "yyyy-MM-dd-HH";
-
- private static ThreadLocal<Map<String, DateFormat>> dateFormats =
- new ThreadLocal<Map<String, DateFormat>> () {
- public Map<String, DateFormat> get() {
- return super.get();
- }
- protected Map<String, DateFormat> initialValue() {
- return new HashMap<String, DateFormat>();
- }
- public void remove() {
- super.remove();
- }
- public void set(Map<String, DateFormat> value) {
- super.set(value);
- }
- };
-
- private static DateFormat getDateFormat(String format) {
- Map<String, DateFormat> threadDateFormats = dateFormats.get();
- if (!threadDateFormats.containsKey(format)) {
- DateFormat dateFormat = new SimpleDateFormat(format);
- dateFormat.setLenient(false);
- dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- threadDateFormats.put(format, dateFormat);
- }
- return threadDateFormats.get(format);
- }
-
- public static String format(long millis, String format) {
- return getDateFormat(format).format(millis);
- }
-
- public static String format(long millis) {
- return format(millis, ISO_DATETIME_FORMAT);
- }
-
- public static long parse(String string, String format) {
- try {
- return getDateFormat(format).parse(string).getTime();
- } catch (ParseException e) {
- return -1L;
- }
- }
-
- public static long parse(String string) {
- return parse(string, ISO_DATETIME_FORMAT);
- }
-}
-
diff --git a/src/org/torproject/onionoo/DescriptorSource.java b/src/org/torproject/onionoo/DescriptorSource.java
deleted file mode 100644
index 8246bba..0000000
--- a/src/org/torproject/onionoo/DescriptorSource.java
+++ /dev/null
@@ -1,665 +0,0 @@
-/* Copyright 2013, 2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.HttpURLConnection;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.zip.GZIPInputStream;
-
-import org.torproject.descriptor.BridgeNetworkStatus;
-import org.torproject.descriptor.BridgePoolAssignment;
-import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
-import org.torproject.descriptor.DescriptorReader;
-import org.torproject.descriptor.DescriptorSourceFactory;
-import org.torproject.descriptor.ExitList;
-import org.torproject.descriptor.ExitListEntry;
-import org.torproject.descriptor.ExtraInfoDescriptor;
-import org.torproject.descriptor.RelayNetworkStatusConsensus;
-import org.torproject.descriptor.ServerDescriptor;
-
-enum DescriptorType {
- RELAY_CONSENSUSES,
- RELAY_SERVER_DESCRIPTORS,
- RELAY_EXTRA_INFOS,
- EXIT_LISTS,
- BRIDGE_STATUSES,
- BRIDGE_SERVER_DESCRIPTORS,
- BRIDGE_EXTRA_INFOS,
- BRIDGE_POOL_ASSIGNMENTS,
-}
-
-interface DescriptorListener {
- abstract void processDescriptor(Descriptor descriptor, boolean relay);
-}
-
-interface FingerprintListener {
- abstract void processFingerprints(SortedSet<String> fingerprints,
- boolean relay);
-}
-
-enum DescriptorHistory {
- RELAY_CONSENSUS_HISTORY,
- RELAY_SERVER_HISTORY,
- RELAY_EXTRAINFO_HISTORY,
- EXIT_LIST_HISTORY,
- BRIDGE_STATUS_HISTORY,
- BRIDGE_SERVER_HISTORY,
- BRIDGE_EXTRAINFO_HISTORY,
- BRIDGE_POOLASSIGN_HISTORY,
-}
-
-class DescriptorDownloader {
-
- private final String protocolHostNameResourcePrefix =
- "https://collector.torproject.org/recent/";;
-
- private String directory;
-
- private final File inDir = new File("in/recent");
-
- public DescriptorDownloader(DescriptorType descriptorType) {
- switch (descriptorType) {
- case RELAY_CONSENSUSES:
- this.directory = "relay-descriptors/consensuses/";
- break;
- case RELAY_SERVER_DESCRIPTORS:
- this.directory = "relay-descriptors/server-descriptors/";
- break;
- case RELAY_EXTRA_INFOS:
- this.directory = "relay-descriptors/extra-infos/";
- break;
- case EXIT_LISTS:
- this.directory = "exit-lists/";
- break;
- case BRIDGE_STATUSES:
- this.directory = "bridge-descriptors/statuses/";
- break;
- case BRIDGE_SERVER_DESCRIPTORS:
- this.directory = "bridge-descriptors/server-descriptors/";
- break;
- case BRIDGE_EXTRA_INFOS:
- this.directory = "bridge-descriptors/extra-infos/";
- break;
- case BRIDGE_POOL_ASSIGNMENTS:
- this.directory = "bridge-pool-assignments/";
- break;
- default:
- System.err.println("Unknown descriptor type.");
- return;
- }
- }
-
- private SortedSet<String> localFiles = new TreeSet<String>();
-
- public int statLocalFiles() {
- File localDirectory = new File(this.inDir, this.directory);
- if (localDirectory.exists()) {
- for (File file : localDirectory.listFiles()) {
- this.localFiles.add(file.getName());
- }
- }
- return this.localFiles.size();
- }
-
- private SortedSet<String> remoteFiles = new TreeSet<String>();
-
- public int fetchRemoteDirectory() {
- String directoryUrl = this.protocolHostNameResourcePrefix
- + this.directory;
- try {
- URL u = new URL(directoryUrl);
- HttpURLConnection huc = (HttpURLConnection) u.openConnection();
- huc.setRequestMethod("GET");
- huc.connect();
- if (huc.getResponseCode() != 200) {
- System.err.println("Could not fetch " + directoryUrl
- + ": " + huc.getResponseCode() + " "
- + huc.getResponseMessage() + ". Skipping.");
- return 0;
- }
- BufferedReader br = new BufferedReader(new InputStreamReader(
- huc.getInputStream()));
- String line;
- while ((line = br.readLine()) != null) {
- if (!line.trim().startsWith("<tr>") ||
- !line.contains("<a href=\"")) {
- continue;
- }
- String linePart = line.substring(
- line.indexOf("<a href=\"") + "<a href=\"".length());
- if (!linePart.contains("\"")) {
- continue;
- }
- linePart = linePart.substring(0, linePart.indexOf("\""));
- if (linePart.endsWith("/")) {
- continue;
- }
- this.remoteFiles.add(linePart);
- }
- br.close();
- } catch (IOException e) {
- System.err.println("Could not fetch or parse " + directoryUrl
- + ". Skipping.");
- }
- return this.remoteFiles.size();
- }
-
- public int fetchRemoteFiles() {
- int fetchedFiles = 0;
- for (String remoteFile : this.remoteFiles) {
- if (this.localFiles.contains(remoteFile)) {
- continue;
- }
- String fileUrl = this.protocolHostNameResourcePrefix
- + this.directory + remoteFile;
- File localTempFile = new File(this.inDir, this.directory
- + remoteFile + ".tmp");
- File localFile = new File(this.inDir, this.directory + remoteFile);
- try {
- localFile.getParentFile().mkdirs();
- URL u = new URL(fileUrl);
- HttpURLConnection huc = (HttpURLConnection) u.openConnection();
- huc.setRequestMethod("GET");
- huc.addRequestProperty("Accept-Encoding", "gzip");
- huc.connect();
- if (huc.getResponseCode() != 200) {
- System.err.println("Could not fetch " + fileUrl
- + ": " + huc.getResponseCode() + " "
- + huc.getResponseMessage() + ". Skipping.");
- continue;
- }
- long lastModified = huc.getHeaderFieldDate("Last-Modified", -1L);
- InputStream is;
- if (huc.getContentEncoding() != null &&
- huc.getContentEncoding().equalsIgnoreCase("gzip")) {
- is = new GZIPInputStream(huc.getInputStream());
- } else {
- is = huc.getInputStream();
- }
- BufferedInputStream bis = new BufferedInputStream(is);
- BufferedOutputStream bos = new BufferedOutputStream(
- new FileOutputStream(localTempFile));
- int len;
- byte[] data = new byte[1024];
- while ((len = bis.read(data, 0, 1024)) >= 0) {
- bos.write(data, 0, len);
- }
- bis.close();
- bos.close();
- localTempFile.renameTo(localFile);
- if (lastModified >= 0) {
- localFile.setLastModified(lastModified);
- }
- fetchedFiles++;
- } catch (IOException e) {
- System.err.println("Could not fetch or store " + fileUrl
- + ". Skipping.");
- }
- }
- return fetchedFiles;
- }
-
- public int deleteOldLocalFiles() {
- int deletedFiles = 0;
- for (String localFile : this.localFiles) {
- if (!this.remoteFiles.contains(localFile)) {
- new File(this.inDir, this.directory + localFile).delete();
- deletedFiles++;
- }
- }
- return deletedFiles;
- }
-}
-
-class DescriptorQueue {
-
- private File inDir;
-
- private File statusDir;
-
- private DescriptorReader descriptorReader;
-
- private File historyFile;
-
- private Iterator<DescriptorFile> descriptorFiles;
-
- private List<Descriptor> descriptors;
-
- private int historySizeBefore;
- public int getHistorySizeBefore() {
- return this.historySizeBefore;
- }
-
- private int historySizeAfter;
- public int getHistorySizeAfter() {
- return this.historySizeAfter;
- }
-
- private long returnedDescriptors = 0L;
- public long getReturnedDescriptors() {
- return this.returnedDescriptors;
- }
-
- private long returnedBytes = 0L;
- public long getReturnedBytes() {
- return this.returnedBytes;
- }
-
- public DescriptorQueue(File inDir, File statusDir) {
- this.inDir = inDir;
- this.statusDir = statusDir;
- this.descriptorReader =
- DescriptorSourceFactory.createDescriptorReader();
- }
-
- public void addDirectory(DescriptorType descriptorType) {
- String directoryName = null;
- switch (descriptorType) {
- case RELAY_CONSENSUSES:
- directoryName = "relay-descriptors/consensuses";
- break;
- case RELAY_SERVER_DESCRIPTORS:
- directoryName = "relay-descriptors/server-descriptors";
- break;
- case RELAY_EXTRA_INFOS:
- directoryName = "relay-descriptors/extra-infos";
- break;
- case BRIDGE_STATUSES:
- directoryName = "bridge-descriptors/statuses";
- break;
- case BRIDGE_SERVER_DESCRIPTORS:
- directoryName = "bridge-descriptors/server-descriptors";
- break;
- case BRIDGE_EXTRA_INFOS:
- directoryName = "bridge-descriptors/extra-infos";
- break;
- case BRIDGE_POOL_ASSIGNMENTS:
- directoryName = "bridge-pool-assignments";
- break;
- case EXIT_LISTS:
- directoryName = "exit-lists";
- break;
- default:
- System.err.println("Unknown descriptor type. Not adding directory "
- + "to descriptor reader.");
- return;
- }
- File directory = new File(this.inDir, directoryName);
- if (directory.exists() && directory.isDirectory()) {
- this.descriptorReader.addDirectory(directory);
- this.descriptorReader.setMaxDescriptorFilesInQueue(1);
- } else {
- System.err.println("Directory " + directory.getAbsolutePath()
- + " either does not exist or is not a directory. Not adding "
- + "to descriptor reader.");
- }
- }
-
- public void readHistoryFile(DescriptorHistory descriptorHistory) {
- String historyFileName = null;
- switch (descriptorHistory) {
- case RELAY_EXTRAINFO_HISTORY:
- historyFileName = "relay-extrainfo-history";
- break;
- case BRIDGE_EXTRAINFO_HISTORY:
- historyFileName = "bridge-extrainfo-history";
- break;
- case EXIT_LIST_HISTORY:
- historyFileName = "exit-list-history";
- break;
- case BRIDGE_POOLASSIGN_HISTORY:
- historyFileName = "bridge-poolassign-history";
- break;
- case RELAY_CONSENSUS_HISTORY:
- historyFileName = "relay-consensus-history";
- break;
- case BRIDGE_STATUS_HISTORY:
- historyFileName = "bridge-status-history";
- break;
- case RELAY_SERVER_HISTORY:
- historyFileName = "relay-server-history";
- break;
- case BRIDGE_SERVER_HISTORY:
- historyFileName = "bridge-server-history";
- break;
- default:
- System.err.println("Unknown descriptor history. Not excluding "
- + "files.");
- return;
- }
- this.historyFile = new File(this.statusDir, historyFileName);
- if (this.historyFile.exists() && this.historyFile.isFile()) {
- SortedMap<String, Long> excludedFiles = new TreeMap<String, Long>();
- try {
- BufferedReader br = new BufferedReader(new FileReader(
- this.historyFile));
- String line;
- while ((line = br.readLine()) != null) {
- try {
- String[] parts = line.split(" ", 2);
- excludedFiles.put(parts[1], Long.parseLong(parts[0]));
- } catch (NumberFormatException e) {
- System.err.println("Illegal line '" + line + "' in parse "
- + "history. Skipping line.");
- }
- }
- br.close();
- } catch (IOException e) {
- System.err.println("Could not read history file '"
- + this.historyFile.getAbsolutePath() + "'. Not excluding "
- + "descriptors in this execution.");
- e.printStackTrace();
- return;
- }
- this.historySizeBefore = excludedFiles.size();
- this.descriptorReader.setExcludedFiles(excludedFiles);
- }
- }
-
- public void writeHistoryFile() {
- if (this.historyFile == null) {
- return;
- }
- SortedMap<String, Long> excludedAndParsedFiles =
- new TreeMap<String, Long>();
- excludedAndParsedFiles.putAll(
- this.descriptorReader.getExcludedFiles());
- excludedAndParsedFiles.putAll(this.descriptorReader.getParsedFiles());
- this.historySizeAfter = excludedAndParsedFiles.size();
- try {
- this.historyFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.historyFile));
- for (Map.Entry<String, Long> e : excludedAndParsedFiles.entrySet()) {
- String absolutePath = e.getKey();
- long lastModifiedMillis = e.getValue();
- bw.write(String.valueOf(lastModifiedMillis) + " " + absolutePath
- + "\n");
- }
- bw.close();
- } catch (IOException e) {
- System.err.println("Could not write history file '"
- + this.historyFile.getAbsolutePath() + "'. Not excluding "
- + "descriptors in next execution.");
- return;
- }
- }
-
- public Descriptor nextDescriptor() {
- Descriptor nextDescriptor = null;
- if (this.descriptorFiles == null) {
- this.descriptorFiles = this.descriptorReader.readDescriptors();
- }
- while (this.descriptors == null && this.descriptorFiles.hasNext()) {
- DescriptorFile descriptorFile = this.descriptorFiles.next();
- if (descriptorFile.getException() != null) {
- System.err.println("Could not parse "
- + descriptorFile.getFileName());
- descriptorFile.getException().printStackTrace();
- }
- if (descriptorFile.getDescriptors() != null &&
- !descriptorFile.getDescriptors().isEmpty()) {
- this.descriptors = descriptorFile.getDescriptors();
- }
- }
- if (this.descriptors != null) {
- nextDescriptor = this.descriptors.remove(0);
- this.returnedDescriptors++;
- this.returnedBytes += nextDescriptor.getRawDescriptorBytes().length;
- if (this.descriptors.isEmpty()) {
- this.descriptors = null;
- }
- }
- return nextDescriptor;
- }
-}
-
-public class DescriptorSource {
-
- private final File inDir = new File("in/recent");
-
- private final File statusDir = new File("status");
-
- private List<DescriptorQueue> descriptorQueues;
-
- public DescriptorSource() {
- this.descriptorQueues = new ArrayList<DescriptorQueue>();
- this.descriptorListeners =
- new HashMap<DescriptorType, Set<DescriptorListener>>();
- this.fingerprintListeners =
- new HashMap<DescriptorType, Set<FingerprintListener>>();
- }
-
- private DescriptorQueue getDescriptorQueue(
- DescriptorType descriptorType,
- DescriptorHistory descriptorHistory) {
- DescriptorQueue descriptorQueue = new DescriptorQueue(this.inDir,
- this.statusDir);
- descriptorQueue.addDirectory(descriptorType);
- if (descriptorHistory != null) {
- descriptorQueue.readHistoryFile(descriptorHistory);
- }
- this.descriptorQueues.add(descriptorQueue);
- return descriptorQueue;
- }
-
- private Map<DescriptorType, Set<DescriptorListener>>
- descriptorListeners;
-
- private Map<DescriptorType, Set<FingerprintListener>>
- fingerprintListeners;
-
- public void registerDescriptorListener(DescriptorListener listener,
- DescriptorType descriptorType) {
- if (!this.descriptorListeners.containsKey(descriptorType)) {
- this.descriptorListeners.put(descriptorType,
- new HashSet<DescriptorListener>());
- }
- this.descriptorListeners.get(descriptorType).add(listener);
- }
-
- public void registerFingerprintListener(FingerprintListener listener,
- DescriptorType descriptorType) {
- if (!this.fingerprintListeners.containsKey(descriptorType)) {
- this.fingerprintListeners.put(descriptorType,
- new HashSet<FingerprintListener>());
- }
- this.fingerprintListeners.get(descriptorType).add(listener);
- }
-
- public void downloadDescriptors() {
- for (DescriptorType descriptorType : DescriptorType.values()) {
- this.downloadDescriptors(descriptorType);
- }
- }
-
- private int localFilesBefore = 0, foundRemoteFiles = 0,
- downloadedFiles = 0, deletedLocalFiles = 0;
-
- private void downloadDescriptors(DescriptorType descriptorType) {
- if (!this.descriptorListeners.containsKey(descriptorType) &&
- !this.fingerprintListeners.containsKey(descriptorType)) {
- return;
- }
- DescriptorDownloader descriptorDownloader =
- new DescriptorDownloader(descriptorType);
- this.localFilesBefore += descriptorDownloader.statLocalFiles();
- this.foundRemoteFiles +=
- descriptorDownloader.fetchRemoteDirectory();
- this.downloadedFiles += descriptorDownloader.fetchRemoteFiles();
- this.deletedLocalFiles += descriptorDownloader.deleteOldLocalFiles();
- }
-
- public void readDescriptors() {
- /* Careful when changing the order of parsing descriptor types! The
- * various status updaters may base assumptions on this order. */
- this.readDescriptors(DescriptorType.RELAY_SERVER_DESCRIPTORS,
- DescriptorHistory.RELAY_SERVER_HISTORY, true);
- this.readDescriptors(DescriptorType.RELAY_EXTRA_INFOS,
- DescriptorHistory.RELAY_EXTRAINFO_HISTORY, true);
- this.readDescriptors(DescriptorType.EXIT_LISTS,
- DescriptorHistory.EXIT_LIST_HISTORY, true);
- this.readDescriptors(DescriptorType.RELAY_CONSENSUSES,
- DescriptorHistory.RELAY_CONSENSUS_HISTORY, true);
- this.readDescriptors(DescriptorType.BRIDGE_SERVER_DESCRIPTORS,
- DescriptorHistory.BRIDGE_SERVER_HISTORY, false);
- this.readDescriptors(DescriptorType.BRIDGE_EXTRA_INFOS,
- DescriptorHistory.BRIDGE_EXTRAINFO_HISTORY, false);
- this.readDescriptors(DescriptorType.BRIDGE_POOL_ASSIGNMENTS,
- DescriptorHistory.BRIDGE_POOLASSIGN_HISTORY, false);
- this.readDescriptors(DescriptorType.BRIDGE_STATUSES,
- DescriptorHistory.BRIDGE_STATUS_HISTORY, false);
- }
-
- private void readDescriptors(DescriptorType descriptorType,
- DescriptorHistory descriptorHistory, boolean relay) {
- if (!this.descriptorListeners.containsKey(descriptorType) &&
- !this.fingerprintListeners.containsKey(descriptorType)) {
- return;
- }
- Set<DescriptorListener> descriptorListeners =
- this.descriptorListeners.get(descriptorType);
- Set<FingerprintListener> fingerprintListeners =
- this.fingerprintListeners.get(descriptorType);
- DescriptorQueue descriptorQueue = this.getDescriptorQueue(
- descriptorType, descriptorHistory);
- Descriptor descriptor;
- while ((descriptor = descriptorQueue.nextDescriptor()) != null) {
- for (DescriptorListener descriptorListener : descriptorListeners) {
- descriptorListener.processDescriptor(descriptor, relay);
- }
- if (fingerprintListeners == null) {
- continue;
- }
- SortedSet<String> fingerprints = new TreeSet<String>();
- if (descriptorType == DescriptorType.RELAY_CONSENSUSES &&
- descriptor instanceof RelayNetworkStatusConsensus) {
- fingerprints.addAll(((RelayNetworkStatusConsensus) descriptor).
- getStatusEntries().keySet());
- } else if (descriptorType
- == DescriptorType.RELAY_SERVER_DESCRIPTORS &&
- descriptor instanceof ServerDescriptor) {
- fingerprints.add(((ServerDescriptor) descriptor).
- getFingerprint());
- } else if (descriptorType == DescriptorType.RELAY_EXTRA_INFOS &&
- descriptor instanceof ExtraInfoDescriptor) {
- fingerprints.add(((ExtraInfoDescriptor) descriptor).
- getFingerprint());
- } else if (descriptorType == DescriptorType.EXIT_LISTS &&
- descriptor instanceof ExitList) {
- for (ExitListEntry entry :
- ((ExitList) descriptor).getExitListEntries()) {
- fingerprints.add(entry.getFingerprint());
- }
- } else if (descriptorType == DescriptorType.BRIDGE_STATUSES &&
- descriptor instanceof BridgeNetworkStatus) {
- fingerprints.addAll(((BridgeNetworkStatus) descriptor).
- getStatusEntries().keySet());
- } else if (descriptorType ==
- DescriptorType.BRIDGE_SERVER_DESCRIPTORS &&
- descriptor instanceof ServerDescriptor) {
- fingerprints.add(((ServerDescriptor) descriptor).
- getFingerprint());
- } else if (descriptorType == DescriptorType.BRIDGE_EXTRA_INFOS &&
- descriptor instanceof ExtraInfoDescriptor) {
- fingerprints.add(((ExtraInfoDescriptor) descriptor).
- getFingerprint());
- } else if (descriptorType ==
- DescriptorType.BRIDGE_POOL_ASSIGNMENTS &&
- descriptor instanceof BridgePoolAssignment) {
- fingerprints.addAll(((BridgePoolAssignment) descriptor).
- getEntries().keySet());
- }
- for (FingerprintListener fingerprintListener :
- fingerprintListeners) {
- fingerprintListener.processFingerprints(fingerprints, relay);
- }
- }
- switch (descriptorType) {
- case RELAY_CONSENSUSES:
- Logger.printStatusTime("Read relay network consensuses");
- break;
- case RELAY_SERVER_DESCRIPTORS:
- Logger.printStatusTime("Read relay server descriptors");
- break;
- case RELAY_EXTRA_INFOS:
- Logger.printStatusTime("Read relay extra-info descriptors");
- break;
- case EXIT_LISTS:
- Logger.printStatusTime("Read exit lists");
- break;
- case BRIDGE_STATUSES:
- Logger.printStatusTime("Read bridge network statuses");
- break;
- case BRIDGE_SERVER_DESCRIPTORS:
- Logger.printStatusTime("Read bridge server descriptors");
- break;
- case BRIDGE_EXTRA_INFOS:
- Logger.printStatusTime("Read bridge extra-info descriptors");
- break;
- case BRIDGE_POOL_ASSIGNMENTS:
- Logger.printStatusTime("Read bridge-pool assignments");
- break;
- }
- }
-
- public void writeHistoryFiles() {
- for (DescriptorQueue descriptorQueue : this.descriptorQueues) {
- descriptorQueue.writeHistoryFile();
- }
- }
-
- public String getStatsString() {
- StringBuilder sb = new StringBuilder();
- sb.append(" " + this.localFilesBefore + " descriptor files found "
- + "locally\n");
- sb.append(" " + this.foundRemoteFiles + " descriptor files found "
- + "remotely\n");
- sb.append(" " + this.downloadedFiles + " descriptor files "
- + "downloaded from remote\n");
- sb.append(" " + this.deletedLocalFiles + " descriptor files "
- + "deleted locally\n");
- sb.append(" " + this.descriptorQueues.size() + " descriptor "
- + "queues created\n");
- int historySizeBefore = 0, historySizeAfter = 0;
- long descriptors = 0L, bytes = 0L;
- for (DescriptorQueue descriptorQueue : descriptorQueues) {
- historySizeBefore += descriptorQueue.getHistorySizeBefore();
- historySizeAfter += descriptorQueue.getHistorySizeAfter();
- descriptors += descriptorQueue.getReturnedDescriptors();
- bytes += descriptorQueue.getReturnedBytes();
- }
- sb.append(" " + Logger.formatDecimalNumber(historySizeBefore)
- + " descriptors excluded from this execution\n");
- sb.append(" " + Logger.formatDecimalNumber(descriptors)
- + " descriptors provided\n");
- sb.append(" " + Logger.formatBytes(bytes) + " provided\n");
- sb.append(" " + Logger.formatDecimalNumber(historySizeAfter)
- + " descriptors excluded from next execution\n");
- return sb.toString();
- }
-}
-
diff --git a/src/org/torproject/onionoo/DetailsDocument.java b/src/org/torproject/onionoo/DetailsDocument.java
deleted file mode 100644
index 2df6e78..0000000
--- a/src/org/torproject/onionoo/DetailsDocument.java
+++ /dev/null
@@ -1,365 +0,0 @@
-/* Copyright 2013--2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang.StringEscapeUtils;
-
-public class DetailsDocument extends Document {
-
- /* We must ensure that details files only contain ASCII characters
- * and no UTF-8 characters. While UTF-8 characters are perfectly
- * valid in JSON, this would break compatibility with existing files
- * pretty badly. We do this by escaping non-ASCII characters, e.g.,
- * \u00F2. Gson won't treat this as UTF-8, but will think that we want
- * to write six characters '\', 'u', '0', '0', 'F', '2'. The only thing
- * we'll have to do is to change back the '\\' that Gson writes for the
- * '\'. */
- private static String escapeJSON(String s) {
- return s == null ? null :
- StringEscapeUtils.escapeJavaScript(s).replaceAll("\\\\'", "'");
- }
- private static String unescapeJSON(String s) {
- return s == null ? null :
- StringEscapeUtils.unescapeJavaScript(s.replaceAll("'", "\\'"));
- }
-
- private String nickname;
- public void setNickname(String nickname) {
- this.nickname = nickname;
- }
- public String getNickname() {
- return this.nickname;
- }
-
- private String fingerprint;
- public void setFingerprint(String fingerprint) {
- this.fingerprint = fingerprint;
- }
- public String getFingerprint() {
- return this.fingerprint;
- }
-
- private String hashed_fingerprint;
- public void setHashedFingerprint(String hashedFingerprint) {
- this.hashed_fingerprint = hashedFingerprint;
- }
- public String getHashedFingerprint() {
- return this.hashed_fingerprint;
- }
-
- private List<String> or_addresses;
- public void setOrAddresses(List<String> orAddresses) {
- this.or_addresses = orAddresses;
- }
- public List<String> getOrAddresses() {
- return this.or_addresses;
- }
-
- private List<String> exit_addresses;
- public void setExitAddresses(List<String> exitAddresses) {
- this.exit_addresses = exitAddresses;
- }
- public List<String> getExitAddresses() {
- return this.exit_addresses;
- }
-
- private String dir_address;
- public void setDirAddress(String dirAddress) {
- this.dir_address = dirAddress;
- }
- public String getDirAddress() {
- return this.dir_address;
- }
-
- private String last_seen;
- public void setLastSeen(String lastSeen) {
- this.last_seen = lastSeen;
- }
- public String getLastSeen() {
- return this.last_seen;
- }
-
- private String last_changed_address_or_port;
- public void setLastChangedAddressOrPort(
- String lastChangedAddressOrPort) {
- this.last_changed_address_or_port = lastChangedAddressOrPort;
- }
- public String getLastChangedAddressOrPort() {
- return this.last_changed_address_or_port;
- }
-
- private String first_seen;
- public void setFirstSeen(String firstSeen) {
- this.first_seen = firstSeen;
- }
- public String getFirstSeen() {
- return this.first_seen;
- }
-
- private Boolean running;
- public void setRunning(Boolean running) {
- this.running = running;
- }
- public Boolean getRunning() {
- return this.running;
- }
-
- private List<String> flags;
- public void setFlags(List<String> flags) {
- this.flags = flags;
- }
- public List<String> getFlags() {
- return this.flags;
- }
-
- private String country;
- public void setCountry(String country) {
- this.country = country;
- }
- public String getCountry() {
- return this.country;
- }
-
- private String country_name;
- public void setCountryName(String countryName) {
- this.country_name = escapeJSON(countryName);
- }
- public String getCountryName() {
- return unescapeJSON(this.country_name);
- }
-
- private String region_name;
- public void setRegionName(String regionName) {
- this.region_name = escapeJSON(regionName);
- }
- public String getRegionName() {
- return unescapeJSON(this.region_name);
- }
-
- private String city_name;
- public void setCityName(String cityName) {
- this.city_name = escapeJSON(cityName);
- }
- public String getCityName() {
- return unescapeJSON(this.city_name);
- }
-
- private Float latitude;
- public void setLatitude(Float latitude) {
- this.latitude = latitude;
- }
- public Float getLatitude() {
- return this.latitude;
- }
-
- private Float longitude;
- public void setLongitude(Float longitude) {
- this.longitude = longitude;
- }
- public Float getLongitude() {
- return this.longitude;
- }
-
- private String as_number;
- public void setAsNumber(String asNumber) {
- this.as_number = escapeJSON(asNumber);
- }
- public String getAsNumber() {
- return unescapeJSON(this.as_number);
- }
-
- private String as_name;
- public void setAsName(String asName) {
- this.as_name = escapeJSON(asName);
- }
- public String getAsName() {
- return unescapeJSON(this.as_name);
- }
-
- private Long consensus_weight;
- public void setConsensusWeight(Long consensusWeight) {
- this.consensus_weight = consensusWeight;
- }
- public Long getConsensusWeight() {
- return this.consensus_weight;
- }
-
- private String host_name;
- public void setHostName(String hostName) {
- this.host_name = escapeJSON(hostName);
- }
- public String getHostName() {
- return unescapeJSON(this.host_name);
- }
-
- private String last_restarted;
- public void setLastRestarted(String lastRestarted) {
- this.last_restarted = lastRestarted;
- }
- public String getLastRestarted() {
- return this.last_restarted;
- }
-
- private Integer bandwidth_rate;
- public void setBandwidthRate(Integer bandwidthRate) {
- this.bandwidth_rate = bandwidthRate;
- }
- public Integer getBandwidthRate() {
- return this.bandwidth_rate;
- }
-
- private Integer bandwidth_burst;
- public void setBandwidthBurst(Integer bandwidthBurst) {
- this.bandwidth_burst = bandwidthBurst;
- }
- public Integer getBandwidthBurst() {
- return this.bandwidth_burst;
- }
-
- private Integer observed_bandwidth;
- public void setObservedBandwidth(Integer observedBandwidth) {
- this.observed_bandwidth = observedBandwidth;
- }
- public Integer getObservedBandwidth() {
- return this.observed_bandwidth;
- }
-
- private Integer advertised_bandwidth;
- public void setAdvertisedBandwidth(Integer advertisedBandwidth) {
- this.advertised_bandwidth = advertisedBandwidth;
- }
- public Integer getAdvertisedBandwidth() {
- return this.advertised_bandwidth;
- }
-
- private List<String> exit_policy;
- public void setExitPolicy(List<String> exitPolicy) {
- this.exit_policy = exitPolicy;
- }
- public List<String> getExitPolicy() {
- return this.exit_policy;
- }
-
- private Map<String, List<String>> exit_policy_summary;
- public void setExitPolicySummary(
- Map<String, List<String>> exitPolicySummary) {
- this.exit_policy_summary = exitPolicySummary;
- }
- public Map<String, List<String>> getExitPolicySummary() {
- return this.exit_policy_summary;
- }
-
- private Map<String, List<String>> exit_policy_v6_summary;
- public void setExitPolicyV6Summary(
- Map<String, List<String>> exitPolicyV6Summary) {
- this.exit_policy_v6_summary = exitPolicyV6Summary;
- }
- public Map<String, List<String>> getExitPolicyV6Summary() {
- return this.exit_policy_v6_summary;
- }
-
- private String contact;
- public void setContact(String contact) {
- this.contact = escapeJSON(contact);
- }
- public String getContact() {
- return unescapeJSON(this.contact);
- }
-
- private String platform;
- public void setPlatform(String platform) {
- this.platform = escapeJSON(platform);
- }
- public String getPlatform() {
- return unescapeJSON(this.platform);
- }
-
- private List<String> family;
- public void setFamily(List<String> family) {
- this.family = family;
- }
- public List<String> getFamily() {
- return this.family;
- }
-
- private Float advertised_bandwidth_fraction;
- public void setAdvertisedBandwidthFraction(
- Float advertisedBandwidthFraction) {
- if (advertisedBandwidthFraction == null ||
- advertisedBandwidthFraction >= 0.0) {
- this.advertised_bandwidth_fraction = advertisedBandwidthFraction;
- }
- }
- public Float getAdvertisedBandwidthFraction() {
- return this.advertised_bandwidth_fraction;
- }
-
- private Float consensus_weight_fraction;
- public void setConsensusWeightFraction(Float consensusWeightFraction) {
- if (consensusWeightFraction == null ||
- consensusWeightFraction >= 0.0) {
- this.consensus_weight_fraction = consensusWeightFraction;
- }
- }
- public Float getConsensusWeightFraction() {
- return this.consensus_weight_fraction;
- }
-
- private Float guard_probability;
- public void setGuardProbability(Float guardProbability) {
- if (guardProbability == null || guardProbability >= 0.0) {
- this.guard_probability = guardProbability;
- }
- }
- public Float getGuardProbability() {
- return this.guard_probability;
- }
-
- private Float middle_probability;
- public void setMiddleProbability(Float middleProbability) {
- if (middleProbability == null || middleProbability >= 0.0) {
- this.middle_probability = middleProbability;
- }
- }
- public Float getMiddleProbability() {
- return this.middle_probability;
- }
-
- private Float exit_probability;
- public void setExitProbability(Float exitProbability) {
- if (exitProbability == null || exitProbability >= 0.0) {
- this.exit_probability = exitProbability;
- }
- }
- public Float getExitProbability() {
- return this.exit_probability;
- }
-
- private Boolean recommended_version;
- public void setRecommendedVersion(Boolean recommendedVersion) {
- this.recommended_version = recommendedVersion;
- }
- public Boolean getRecommendedVersion() {
- return this.recommended_version;
- }
-
- private Boolean hibernating;
- public void setHibernating(Boolean hibernating) {
- this.hibernating = hibernating;
- }
- public Boolean getHibernating() {
- return this.hibernating;
- }
-
- private String pool_assignment;
- public void setPoolAssignment(String poolAssignment) {
- this.pool_assignment = poolAssignment;
- }
- public String getPoolAssignment() {
- return this.pool_assignment;
- }
-}
-
diff --git a/src/org/torproject/onionoo/DetailsDocumentWriter.java b/src/org/torproject/onionoo/DetailsDocumentWriter.java
deleted file mode 100644
index a70efd0..0000000
--- a/src/org/torproject/onionoo/DetailsDocumentWriter.java
+++ /dev/null
@@ -1,222 +0,0 @@
-package org.torproject.onionoo;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-public class DetailsDocumentWriter implements FingerprintListener,
- DocumentWriter {
-
- private DescriptorSource descriptorSource;
-
- private DocumentStore documentStore;
-
- private long now;
-
- public DetailsDocumentWriter() {
- this.descriptorSource = ApplicationFactory.getDescriptorSource();
- this.documentStore = ApplicationFactory.getDocumentStore();
- this.now = ApplicationFactory.getTime().currentTimeMillis();
- this.registerFingerprintListeners();
- }
-
- private void registerFingerprintListeners() {
- this.descriptorSource.registerFingerprintListener(this,
- DescriptorType.RELAY_CONSENSUSES);
- this.descriptorSource.registerFingerprintListener(this,
- DescriptorType.RELAY_SERVER_DESCRIPTORS);
- this.descriptorSource.registerFingerprintListener(this,
- DescriptorType.BRIDGE_STATUSES);
- this.descriptorSource.registerFingerprintListener(this,
- DescriptorType.BRIDGE_SERVER_DESCRIPTORS);
- this.descriptorSource.registerFingerprintListener(this,
- DescriptorType.BRIDGE_POOL_ASSIGNMENTS);
- this.descriptorSource.registerFingerprintListener(this,
- DescriptorType.EXIT_LISTS);
- }
-
- private SortedSet<String> newRelays = new TreeSet<String>(),
- newBridges = new TreeSet<String>();
-
- public void processFingerprints(SortedSet<String> fingerprints,
- boolean relay) {
- if (relay) {
- this.newRelays.addAll(fingerprints);
- } else {
- this.newBridges.addAll(fingerprints);
- }
- }
-
- public void writeDocuments() {
- this.updateRelayDetailsFiles();
- this.updateBridgeDetailsFiles();
- Logger.printStatusTime("Wrote details document files");
- }
-
- private void updateRelayDetailsFiles() {
- for (String fingerprint : this.newRelays) {
-
- /* Generate network-status-specific part. */
- NodeStatus entry = this.documentStore.retrieve(NodeStatus.class,
- true, fingerprint);
- if (entry == null) {
- continue;
- }
- DetailsDocument detailsDocument = new DetailsDocument();
- detailsDocument.setNickname(entry.getNickname());
- detailsDocument.setFingerprint(fingerprint);
- List<String> orAddresses = new ArrayList<String>();
- orAddresses.add(entry.getAddress() + ":" + entry.getOrPort());
- for (String orAddress : entry.getOrAddressesAndPorts()) {
- orAddresses.add(orAddress.toLowerCase());
- }
- detailsDocument.setOrAddresses(orAddresses);
- if (entry.getDirPort() != 0) {
- detailsDocument.setDirAddress(entry.getAddress() + ":"
- + entry.getDirPort());
- }
- detailsDocument.setLastSeen(DateTimeHelper.format(
- entry.getLastSeenMillis()));
- detailsDocument.setFirstSeen(DateTimeHelper.format(
- entry.getFirstSeenMillis()));
- detailsDocument.setLastChangedAddressOrPort(
- DateTimeHelper.format(entry.getLastChangedOrAddress()));
- detailsDocument.setRunning(entry.getRunning());
- if (!entry.getRelayFlags().isEmpty()) {
- detailsDocument.setFlags(new ArrayList<String>(
- entry.getRelayFlags()));
- }
- detailsDocument.setCountry(entry.getCountryCode());
- detailsDocument.setLatitude(entry.getLatitude());
- detailsDocument.setLongitude(entry.getLongitude());
- detailsDocument.setCountryName(entry.getCountryName());
- detailsDocument.setRegionName(entry.getRegionName());
- detailsDocument.setCityName(entry.getCityName());
- detailsDocument.setAsNumber(entry.getASNumber());
- detailsDocument.setAsName(entry.getASName());
- detailsDocument.setConsensusWeight(entry.getConsensusWeight());
- detailsDocument.setHostName(entry.getHostName());
- detailsDocument.setAdvertisedBandwidthFraction(
- (float) entry.getAdvertisedBandwidthFraction());
- detailsDocument.setConsensusWeightFraction(
- (float) entry.getConsensusWeightFraction());
- detailsDocument.setGuardProbability(
- (float) entry.getGuardProbability());
- detailsDocument.setMiddleProbability(
- (float) entry.getMiddleProbability());
- detailsDocument.setExitProbability(
- (float) entry.getExitProbability());
- String defaultPolicy = entry.getDefaultPolicy();
- String portList = entry.getPortList();
- if (defaultPolicy != null && (defaultPolicy.equals("accept") ||
- defaultPolicy.equals("reject")) && portList != null) {
- Map<String, List<String>> exitPolicySummary =
- new HashMap<String, List<String>>();
- List<String> portsOrPortRanges = Arrays.asList(
- portList.split(","));
- exitPolicySummary.put(defaultPolicy, portsOrPortRanges);
- detailsDocument.setExitPolicySummary(exitPolicySummary);
- }
- detailsDocument.setRecommendedVersion(
- entry.getRecommendedVersion());
-
- /* Append descriptor-specific part and exit addresses from details
- * status file. */
- DetailsStatus detailsStatus = this.documentStore.retrieve(
- DetailsStatus.class, true, fingerprint);
- if (detailsStatus != null) {
- detailsDocument.setLastRestarted(
- detailsStatus.getLastRestarted());
- detailsDocument.setBandwidthRate(
- detailsStatus.getBandwidthRate());
- detailsDocument.setBandwidthBurst(
- detailsStatus.getBandwidthBurst());
- detailsDocument.setObservedBandwidth(
- detailsStatus.getObservedBandwidth());
- detailsDocument.setAdvertisedBandwidth(
- detailsStatus.getAdvertisedBandwidth());
- detailsDocument.setExitPolicy(detailsStatus.getExitPolicy());
- detailsDocument.setContact(detailsStatus.getContact());
- detailsDocument.setPlatform(detailsStatus.getPlatform());
- detailsDocument.setFamily(detailsStatus.getFamily());
- detailsDocument.setExitPolicyV6Summary(
- detailsStatus.getExitPolicyV6Summary());
- detailsDocument.setHibernating(detailsStatus.getHibernating());
- if (detailsStatus.getExitAddresses() != null) {
- SortedSet<String> exitAddresses = new TreeSet<String>();
- for (Map.Entry<String, Long> e :
- detailsStatus.getExitAddresses().entrySet()) {
- String exitAddress = e.getKey().toLowerCase();
- long scanMillis = e.getValue();
- if (!entry.getAddress().equals(exitAddress) &&
- !entry.getOrAddresses().contains(exitAddress) &&
- scanMillis >= this.now - DateTimeHelper.ONE_DAY) {
- exitAddresses.add(exitAddress);
- }
- }
- if (!exitAddresses.isEmpty()) {
- detailsDocument.setExitAddresses(new ArrayList<String>(
- exitAddresses));
- }
- }
- }
-
- /* Write details file to disk. */
- this.documentStore.store(detailsDocument, fingerprint);
- }
- }
-
- private void updateBridgeDetailsFiles() {
- for (String fingerprint : this.newBridges) {
-
- /* Generate network-status-specific part. */
- NodeStatus entry = this.documentStore.retrieve(NodeStatus.class,
- true, fingerprint);
- if (entry == null) {
- continue;
- }
- DetailsDocument detailsDocument = new DetailsDocument();
- detailsDocument.setNickname(entry.getNickname());
- detailsDocument.setHashedFingerprint(fingerprint);
- String address = entry.getAddress();
- List<String> orAddresses = new ArrayList<String>();
- orAddresses.add(address + ":" + entry.getOrPort());
- for (String orAddress : entry.getOrAddressesAndPorts()) {
- orAddresses.add(orAddress.toLowerCase());
- }
- detailsDocument.setOrAddresses(orAddresses);
- detailsDocument.setLastSeen(DateTimeHelper.format(
- entry.getLastSeenMillis()));
- detailsDocument.setFirstSeen(DateTimeHelper.format(
- entry.getFirstSeenMillis()));
- detailsDocument.setRunning(entry.getRunning());
- detailsDocument.setFlags(new ArrayList<String>(
- entry.getRelayFlags()));
-
- /* Append descriptor-specific part from details status file. */
- DetailsStatus detailsStatus = this.documentStore.retrieve(
- DetailsStatus.class, true, fingerprint);
- if (detailsStatus != null) {
- detailsDocument.setLastRestarted(
- detailsStatus.getLastRestarted());
- detailsDocument.setAdvertisedBandwidth(
- detailsStatus.getAdvertisedBandwidth());
- detailsDocument.setPlatform(detailsStatus.getPlatform());
- detailsDocument.setPoolAssignment(
- detailsStatus.getPoolAssignment());
- }
-
- /* Write details file to disk. */
- this.documentStore.store(detailsDocument, fingerprint);
- }
- }
-
- public String getStatsString() {
- /* TODO Add statistics string. */
- return null;
- }
-}
diff --git a/src/org/torproject/onionoo/DetailsStatus.java b/src/org/torproject/onionoo/DetailsStatus.java
deleted file mode 100644
index 1951c89..0000000
--- a/src/org/torproject/onionoo/DetailsStatus.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/* Copyright 2013--2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang.StringEscapeUtils;
-
-public class DetailsStatus extends Document {
-
- /* We must ensure that details files only contain ASCII characters
- * and no UTF-8 characters. While UTF-8 characters are perfectly
- * valid in JSON, this would break compatibility with existing files
- * pretty badly. We do this by escaping non-ASCII characters, e.g.,
- * \u00F2. Gson won't treat this as UTF-8, but will think that we want
- * to write six characters '\', 'u', '0', '0', 'F', '2'. The only thing
- * we'll have to do is to change back the '\\' that Gson writes for the
- * '\'. */
- private static String escapeJSON(String s) {
- return s == null ? null :
- StringEscapeUtils.escapeJavaScript(s).replaceAll("\\\\'", "'");
- }
- private static String unescapeJSON(String s) {
- return s == null ? null :
- StringEscapeUtils.unescapeJavaScript(s.replaceAll("'", "\\'"));
- }
-
- private String desc_published;
- public void setDescPublished(String descPublished) {
- this.desc_published = descPublished;
- }
- public String getDescPublished() {
- return this.desc_published;
- }
-
- private String last_restarted;
- public void setLastRestarted(String lastRestarted) {
- this.last_restarted = lastRestarted;
- }
- public String getLastRestarted() {
- return this.last_restarted;
- }
-
- private Integer bandwidth_rate;
- public void setBandwidthRate(Integer bandwidthRate) {
- this.bandwidth_rate = bandwidthRate;
- }
- public Integer getBandwidthRate() {
- return this.bandwidth_rate;
- }
-
- private Integer bandwidth_burst;
- public void setBandwidthBurst(Integer bandwidthBurst) {
- this.bandwidth_burst = bandwidthBurst;
- }
- public Integer getBandwidthBurst() {
- return this.bandwidth_burst;
- }
-
- private Integer observed_bandwidth;
- public void setObservedBandwidth(Integer observedBandwidth) {
- this.observed_bandwidth = observedBandwidth;
- }
- public Integer getObservedBandwidth() {
- return this.observed_bandwidth;
- }
-
- private Integer advertised_bandwidth;
- public void setAdvertisedBandwidth(Integer advertisedBandwidth) {
- this.advertised_bandwidth = advertisedBandwidth;
- }
- public Integer getAdvertisedBandwidth() {
- return this.advertised_bandwidth;
- }
-
- private List<String> exit_policy;
- public void setExitPolicy(List<String> exitPolicy) {
- this.exit_policy = exitPolicy;
- }
- public List<String> getExitPolicy() {
- return this.exit_policy;
- }
-
- private String contact;
- public void setContact(String contact) {
- this.contact = escapeJSON(contact);
- }
- public String getContact() {
- return unescapeJSON(this.contact);
- }
-
- private String platform;
- public void setPlatform(String platform) {
- this.platform = escapeJSON(platform);
- }
- public String getPlatform() {
- return unescapeJSON(this.platform);
- }
-
- private List<String> family;
- public void setFamily(List<String> family) {
- this.family = family;
- }
- public List<String> getFamily() {
- return this.family;
- }
-
- private Map<String, List<String>> exit_policy_v6_summary;
- public void setExitPolicyV6Summary(
- Map<String, List<String>> exitPolicyV6Summary) {
- this.exit_policy_v6_summary = exitPolicyV6Summary;
- }
- public Map<String, List<String>> getExitPolicyV6Summary() {
- return this.exit_policy_v6_summary;
- }
-
- private Boolean hibernating;
- public void setHibernating(Boolean hibernating) {
- this.hibernating = hibernating;
- }
- public Boolean getHibernating() {
- return this.hibernating;
- }
-
- private String pool_assignment;
- public void setPoolAssignment(String poolAssignment) {
- this.pool_assignment = poolAssignment;
- }
- public String getPoolAssignment() {
- return this.pool_assignment;
- }
-
- private Map<String, Long> exit_addresses;
- public void setExitAddresses(Map<String, Long> exitAddresses) {
- this.exit_addresses = exitAddresses;
- }
- public Map<String, Long> getExitAddresses() {
- return this.exit_addresses;
- }
-}
diff --git a/src/org/torproject/onionoo/Document.java b/src/org/torproject/onionoo/Document.java
deleted file mode 100644
index f49574c..0000000
--- a/src/org/torproject/onionoo/Document.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/* Copyright 2013 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-public abstract class Document {
-
- private transient String documentString;
- public void setDocumentString(String documentString) {
- this.documentString = documentString;
- }
- public String getDocumentString() {
- return this.documentString;
- }
-
- public void fromDocumentString(String documentString) {
- /* Subclasses may override this method to parse documentString. */
- }
-
- public String toDocumentString() {
- /* Subclasses may override this method to write documentString. */
- return null;
- }
-}
-
diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java
deleted file mode 100644
index 434ecb6..0000000
--- a/src/org/torproject/onionoo/DocumentStore.java
+++ /dev/null
@@ -1,744 +0,0 @@
-/* Copyright 2013 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.Stack;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonParseException;
-
-// TODO For later migration from disk to database, do the following:
-// - read from database and then from disk if not found
-// - write only to database, delete from disk once in database
-// - move entirely to database once disk is "empty"
-// TODO Also look into simple key-value stores instead of real databases.
-public class DocumentStore {
-
- private final File statusDir = new File("status");
-
- private File outDir = new File("out");
- public void setOutDir(File outDir) {
- this.outDir = outDir;
- }
-
- private Time time;
-
- public DocumentStore() {
- this.time = ApplicationFactory.getTime();
- }
-
- private long listOperations = 0L, listedFiles = 0L, storedFiles = 0L,
- storedBytes = 0L, retrievedFiles = 0L, retrievedBytes = 0L,
- removedFiles = 0L;
-
- /* Node statuses and summary documents are cached in memory, as opposed
- * to all other document types. These caches are initialized when first
- * accessing or modifying a NodeStatus or SummaryDocument document,
- * respectively. */
- private SortedMap<String, NodeStatus> cachedNodeStatuses;
- private SortedMap<String, SummaryDocument> cachedSummaryDocuments;
-
- public <T extends Document> SortedSet<String> list(
- Class<T> documentType) {
- if (documentType.equals(NodeStatus.class)) {
- return this.listNodeStatuses();
- } else if (documentType.equals(SummaryDocument.class)) {
- return this.listSummaryDocuments();
- } else {
- return this.listDocumentFiles(documentType);
- }
- }
-
- private SortedSet<String> listNodeStatuses() {
- if (this.cachedNodeStatuses == null) {
- this.cacheNodeStatuses();
- }
- return new TreeSet<String>(this.cachedNodeStatuses.keySet());
- }
-
- private void cacheNodeStatuses() {
- SortedMap<String, NodeStatus> parsedNodeStatuses =
- new TreeMap<String, NodeStatus>();
- File directory = this.statusDir;
- if (directory != null) {
- File summaryFile = new File(directory, "summary");
- if (summaryFile.exists()) {
- try {
- BufferedReader br = new BufferedReader(new FileReader(
- summaryFile));
- String line;
- while ((line = br.readLine()) != null) {
- if (line.length() == 0) {
- continue;
- }
- NodeStatus node = NodeStatus.fromString(line);
- if (node != null) {
- parsedNodeStatuses.put(node.getFingerprint(), node);
- }
- }
- br.close();
- this.listedFiles += parsedNodeStatuses.size();
- this.listOperations++;
- } catch (IOException e) {
- System.err.println("Could not read file '"
- + summaryFile.getAbsolutePath() + "'.");
- e.printStackTrace();
- }
- }
- }
- this.cachedNodeStatuses = parsedNodeStatuses;
- }
-
- private SortedSet<String> listSummaryDocuments() {
- if (this.cachedSummaryDocuments == null) {
- this.cacheSummaryDocuments();
- }
- return new TreeSet<String>(this.cachedSummaryDocuments.keySet());
- }
-
- private void cacheSummaryDocuments() {
- SortedMap<String, SummaryDocument> parsedSummaryDocuments =
- new TreeMap<String, SummaryDocument>();
- File directory = this.outDir;
- if (directory != null) {
- File summaryFile = new File(directory, "summary");
- if (summaryFile.exists()) {
- String line = null;
- try {
- Gson gson = new Gson();
- BufferedReader br = new BufferedReader(new FileReader(
- summaryFile));
- while ((line = br.readLine()) != null) {
- if (line.length() == 0) {
- continue;
- }
- SummaryDocument summaryDocument = gson.fromJson(line,
- SummaryDocument.class);
- if (summaryDocument != null) {
- parsedSummaryDocuments.put(summaryDocument.getFingerprint(),
- summaryDocument);
- }
- }
- br.close();
- this.listedFiles += parsedSummaryDocuments.size();
- this.listOperations++;
- } catch (IOException e) {
- System.err.println("Could not read file '"
- + summaryFile.getAbsolutePath() + "'.");
- e.printStackTrace();
- } catch (JsonParseException e) {
- System.err.println("Could not parse summary document '" + line
- + "' in file '" + summaryFile.getAbsolutePath() + "'.");
- e.printStackTrace();
- }
- }
- }
- this.cachedSummaryDocuments = parsedSummaryDocuments;
- }
-
- private <T extends Document> SortedSet<String> listDocumentFiles(
- Class<T> documentType) {
- SortedSet<String> fingerprints = new TreeSet<String>();
- File directory = null;
- String subdirectory = null;
- if (documentType.equals(DetailsStatus.class)) {
- directory = this.statusDir;
- subdirectory = "details";
- } else if (documentType.equals(BandwidthStatus.class)) {
- directory = this.statusDir;
- subdirectory = "bandwidth";
- } else if (documentType.equals(WeightsStatus.class)) {
- directory = this.statusDir;
- subdirectory = "weights";
- } else if (documentType.equals(ClientsStatus.class)) {
- directory = this.statusDir;
- subdirectory = "clients";
- } else if (documentType.equals(UptimeStatus.class)) {
- directory = this.statusDir;
- subdirectory = "uptimes";
- } else if (documentType.equals(DetailsDocument.class)) {
- directory = this.outDir;
- subdirectory = "details";
- } else if (documentType.equals(BandwidthDocument.class)) {
- directory = this.outDir;
- subdirectory = "bandwidth";
- } else if (documentType.equals(WeightsDocument.class)) {
- directory = this.outDir;
- subdirectory = "weights";
- } else if (documentType.equals(ClientsDocument.class)) {
- directory = this.outDir;
- subdirectory = "clients";
- } else if (documentType.equals(UptimeDocument.class)) {
- directory = this.outDir;
- subdirectory = "uptimes";
- }
- if (directory != null && subdirectory != null) {
- Stack<File> files = new Stack<File>();
- files.add(new File(directory, subdirectory));
- while (!files.isEmpty()) {
- File file = files.pop();
- if (file.isDirectory()) {
- files.addAll(Arrays.asList(file.listFiles()));
- } else if (file.getName().length() == 40) {
- fingerprints.add(file.getName());
- }
- }
- }
- this.listOperations++;
- this.listedFiles += fingerprints.size();
- return fingerprints;
- }
-
- public <T extends Document> boolean store(T document) {
- return this.store(document, null);
- }
-
- public <T extends Document> boolean store(T document,
- String fingerprint) {
- if (document instanceof NodeStatus) {
- return this.storeNodeStatus((NodeStatus) document, fingerprint);
- } else if (document instanceof SummaryDocument) {
- return this.storeSummaryDocument((SummaryDocument) document,
- fingerprint);
- } else {
- return this.storeDocumentFile(document, fingerprint);
- }
- }
-
- private <T extends Document> boolean storeNodeStatus(
- NodeStatus nodeStatus, String fingerprint) {
- if (this.cachedNodeStatuses == null) {
- this.cacheNodeStatuses();
- }
- this.cachedNodeStatuses.put(fingerprint, nodeStatus);
- return true;
- }
-
- private <T extends Document> boolean storeSummaryDocument(
- SummaryDocument summaryDocument, String fingerprint) {
- if (this.cachedSummaryDocuments == null) {
- this.cacheSummaryDocuments();
- }
- this.cachedSummaryDocuments.put(fingerprint, summaryDocument);
- return true;
- }
-
- private <T extends Document> boolean storeDocumentFile(T document,
- String fingerprint) {
- File documentFile = this.getDocumentFile(document.getClass(),
- fingerprint);
- if (documentFile == null) {
- return false;
- }
- String documentString;
- if (document.getDocumentString() != null) {
- documentString = document.getDocumentString();
- } else if (document instanceof BandwidthDocument ||
- document instanceof WeightsDocument ||
- document instanceof ClientsDocument ||
- document instanceof UptimeDocument) {
- Gson gson = new Gson();
- documentString = gson.toJson(document);
- } else if (document instanceof DetailsStatus ||
- document instanceof DetailsDocument) {
- /* Don't escape HTML characters, like < and >, contained in
- * strings. */
- Gson gson = new GsonBuilder().disableHtmlEscaping().create();
- /* We must ensure that details files only contain ASCII characters
- * and no UTF-8 characters. While UTF-8 characters are perfectly
- * valid in JSON, this would break compatibility with existing files
- * pretty badly. We already make sure that all strings in details
- * objects are escaped JSON, e.g., \u00F2. When Gson serlializes
- * this string, it escapes the \ to \\, hence writes \\u00F2. We
- * need to undo this and change \\u00F2 back to \u00F2. */
- documentString = gson.toJson(document).replaceAll("\\\\\\\\u",
- "\\\\u");
- /* Existing details statuses don't contain opening and closing curly
- * brackets, so we should remove them from new details statuses,
- * too. */
- if (document instanceof DetailsStatus) {
- documentString = documentString.substring(
- documentString.indexOf("{") + 1,
- documentString.lastIndexOf("}"));
- }
- } else if (document instanceof BandwidthStatus ||
- document instanceof WeightsStatus ||
- document instanceof ClientsStatus ||
- document instanceof UptimeStatus) {
- documentString = document.toDocumentString();
- } else {
- System.err.println("Serializing is not supported for type "
- + document.getClass().getName() + ".");
- return false;
- }
- try {
- documentFile.getParentFile().mkdirs();
- File documentTempFile = new File(
- documentFile.getAbsolutePath() + ".tmp");
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- documentTempFile));
- bw.write(documentString);
- bw.close();
- documentFile.delete();
- documentTempFile.renameTo(documentFile);
- this.storedFiles++;
- this.storedBytes += documentString.length();
- } catch (IOException e) {
- System.err.println("Could not write file '"
- + documentFile.getAbsolutePath() + "'.");
- e.printStackTrace();
- return false;
- }
- return true;
- }
-
- public <T extends Document> T retrieve(Class<T> documentType,
- boolean parse) {
- return this.retrieve(documentType, parse, null);
- }
-
- public <T extends Document> T retrieve(Class<T> documentType,
- boolean parse, String fingerprint) {
- if (documentType.equals(NodeStatus.class)) {
- return documentType.cast(this.retrieveNodeStatus(fingerprint));
- } else if (documentType.equals(SummaryDocument.class)) {
- return documentType.cast(this.retrieveSummaryDocument(fingerprint));
- } else {
- return this.retrieveDocumentFile(documentType, parse, fingerprint);
- }
- }
-
- private NodeStatus retrieveNodeStatus(String fingerprint) {
- if (this.cachedNodeStatuses == null) {
- this.cacheNodeStatuses();
- }
- return this.cachedNodeStatuses.get(fingerprint);
- }
-
- private SummaryDocument retrieveSummaryDocument(String fingerprint) {
- if (this.cachedSummaryDocuments == null) {
- this.cacheSummaryDocuments();
- }
- if (this.cachedSummaryDocuments.containsKey(fingerprint)) {
- return this.cachedSummaryDocuments.get(fingerprint);
- }
- /* TODO This is an evil hack to support looking up relays or bridges
- * that haven't been running for a week without having to load
- * 500,000 NodeStatus instances into memory. Maybe there's a better
- * way? Or do we need to switch to a real database for this? */
- DetailsDocument detailsDocument = this.retrieveDocumentFile(
- DetailsDocument.class, true, fingerprint);
- if (detailsDocument == null) {
- return null;
- }
- boolean isRelay = detailsDocument.getHashedFingerprint() == null;
- boolean running = false;
- String nickname = detailsDocument.getNickname();
- List<String> addresses = new ArrayList<String>();
- String countryCode = null, aSNumber = null, contact = null;
- for (String orAddressAndPort : detailsDocument.getOrAddresses()) {
- if (!orAddressAndPort.contains(":")) {
- return null;
- }
- String orAddress = orAddressAndPort.substring(0,
- orAddressAndPort.lastIndexOf(":"));
- if (!addresses.contains(orAddress)) {
- addresses.add(orAddress);
- }
- }
- if (detailsDocument.getExitAddresses() != null) {
- for (String exitAddress : detailsDocument.getExitAddresses()) {
- if (!addresses.contains(exitAddress)) {
- addresses.add(exitAddress);
- }
- }
- }
- SortedSet<String> relayFlags = new TreeSet<String>(), family = null;
- long lastSeenMillis = -1L, consensusWeight = -1L,
- firstSeenMillis = -1L;
- SummaryDocument summaryDocument = new SummaryDocument(isRelay,
- nickname, fingerprint, addresses, lastSeenMillis, running,
- relayFlags, consensusWeight, countryCode, firstSeenMillis,
- aSNumber, contact, family);
- return summaryDocument;
- }
-
- private <T extends Document> T retrieveDocumentFile(
- Class<T> documentType, boolean parse, String fingerprint) {
- File documentFile = this.getDocumentFile(documentType, fingerprint);
- if (documentFile == null || !documentFile.exists()) {
- return null;
- } else if (documentFile.isDirectory()) {
- System.err.println("Could not read file '"
- + documentFile.getAbsolutePath() + "', because it is a "
- + "directory.");
- return null;
- }
- String documentString = null;
- try {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- BufferedInputStream bis = new BufferedInputStream(
- new FileInputStream(documentFile));
- int len;
- byte[] data = new byte[1024];
- while ((len = bis.read(data, 0, 1024)) >= 0) {
- baos.write(data, 0, len);
- }
- bis.close();
- byte[] allData = baos.toByteArray();
- if (allData.length == 0) {
- return null;
- }
- documentString = new String(allData, "US-ASCII");
- this.retrievedFiles++;
- this.retrievedBytes += documentString.length();
- } catch (IOException e) {
- System.err.println("Could not read file '"
- + documentFile.getAbsolutePath() + "'.");
- e.printStackTrace();
- return null;
- }
- T result = null;
- if (!parse) {
- return this.retrieveUnparsedDocumentFile(documentType,
- documentString);
- } else if (documentType.equals(DetailsDocument.class) ||
- documentType.equals(BandwidthDocument.class) ||
- documentType.equals(WeightsDocument.class) ||
- documentType.equals(ClientsDocument.class) ||
- documentType.equals(UptimeDocument.class)) {
- return this.retrieveParsedDocumentFile(documentType,
- documentString);
- } else if (documentType.equals(BandwidthStatus.class) ||
- documentType.equals(WeightsStatus.class) ||
- documentType.equals(ClientsStatus.class) ||
- documentType.equals(UptimeStatus.class)) {
- return this.retrieveParsedStatusFile(documentType, documentString);
- } else if (documentType.equals(DetailsStatus.class)) {
- return this.retrieveParsedDocumentFile(documentType, "{"
- + documentString + "}");
- } else {
- System.err.println("Parsing is not supported for type "
- + documentType.getName() + ".");
- }
- return result;
- }
-
- private <T extends Document> T retrieveParsedStatusFile(
- Class<T> documentType, String documentString) {
- T result = null;
- try {
- result = documentType.newInstance();
- result.fromDocumentString(documentString);
- } catch (InstantiationException e) {
- /* Handle below. */
- e.printStackTrace();
- } catch (IllegalAccessException e) {
- /* Handle below. */
- e.printStackTrace();
- }
- if (result == null) {
- System.err.println("Could not initialize parsed status file of "
- + "type " + documentType.getName() + ".");
- }
- return result;
- }
-
- private <T extends Document> T retrieveParsedDocumentFile(
- Class<T> documentType, String documentString) {
- T result = null;
- Gson gson = new Gson();
- try {
- result = gson.fromJson(documentString, documentType);
- } catch (JsonParseException e) {
- /* Handle below. */
- e.printStackTrace();
- }
- if (result == null) {
- System.err.println("Could not initialize parsed document of type "
- + documentType.getName() + ".");
- }
- return result;
- }
-
- private <T extends Document> T retrieveUnparsedDocumentFile(
- Class<T> documentType, String documentString) {
- T result = null;
- try {
- result = documentType.newInstance();
- result.setDocumentString(documentString);
- } catch (InstantiationException e) {
- /* Handle below. */
- e.printStackTrace();
- } catch (IllegalAccessException e) {
- /* Handle below. */
- e.printStackTrace();
- }
- if (result == null) {
- System.err.println("Could not initialize unparsed document of type "
- + documentType.getName() + ".");
- }
- return result;
- }
-
- public <T extends Document> boolean remove(Class<T> documentType) {
- return this.remove(documentType, null);
- }
-
- public <T extends Document> boolean remove(Class<T> documentType,
- String fingerprint) {
- if (documentType.equals(NodeStatus.class)) {
- return this.removeNodeStatus(fingerprint);
- } else if (documentType.equals(SummaryDocument.class)) {
- return this.removeSummaryDocument(fingerprint);
- } else {
- return this.removeDocumentFile(documentType, fingerprint);
- }
- }
-
- private boolean removeNodeStatus(String fingerprint) {
- if (this.cachedNodeStatuses == null) {
- this.cacheNodeStatuses();
- }
- return this.cachedNodeStatuses.remove(fingerprint) != null;
- }
-
- private boolean removeSummaryDocument(String fingerprint) {
- if (this.cachedSummaryDocuments == null) {
- this.cacheSummaryDocuments();
- }
- return this.cachedSummaryDocuments.remove(fingerprint) != null;
- }
-
- private <T extends Document> boolean removeDocumentFile(
- Class<T> documentType, String fingerprint) {
- File documentFile = this.getDocumentFile(documentType, fingerprint);
- if (documentFile == null || !documentFile.delete()) {
- System.err.println("Could not delete file '"
- + documentFile.getAbsolutePath() + "'.");
- return false;
- }
- this.removedFiles++;
- return true;
- }
-
- private <T extends Document> File getDocumentFile(Class<T> documentType,
- String fingerprint) {
- File documentFile = null;
- if (fingerprint == null && !documentType.equals(UpdateStatus.class) &&
- !documentType.equals(UptimeStatus.class)) {
- // TODO Instead of using the update file workaround, add new method
- // lastModified(Class<T> documentType) that serves a similar
- // purpose.
- return null;
- }
- File directory = null;
- String fileName = null;
- if (documentType.equals(DetailsStatus.class)) {
- directory = this.statusDir;
- fileName = String.format("details/%s/%s/%s",
- fingerprint.substring(0, 1), fingerprint.substring(1, 2),
- fingerprint);
- } else if (documentType.equals(BandwidthStatus.class)) {
- directory = this.statusDir;
- fileName = String.format("bandwidth/%s/%s/%s",
- fingerprint.substring(0, 1), fingerprint.substring(1, 2),
- fingerprint);
- } else if (documentType.equals(WeightsStatus.class)) {
- directory = this.statusDir;
- fileName = String.format("weights/%s/%s/%s",
- fingerprint.substring(0, 1), fingerprint.substring(1, 2),
- fingerprint);
- } else if (documentType.equals(ClientsStatus.class)) {
- directory = this.statusDir;
- fileName = String.format("clients/%s/%s/%s",
- fingerprint.substring(0, 1), fingerprint.substring(1, 2),
- fingerprint);
- } else if (documentType.equals(UptimeStatus.class)) {
- directory = this.statusDir;
- if (fingerprint == null) {
- fileName = "uptime";
- } else {
- fileName = String.format("uptimes/%s/%s/%s",
- fingerprint.substring(0, 1), fingerprint.substring(1, 2),
- fingerprint);
- }
- } else if (documentType.equals(UpdateStatus.class)) {
- directory = this.outDir;
- fileName = "update";
- } else if (documentType.equals(DetailsDocument.class)) {
- directory = this.outDir;
- fileName = String.format("details/%s", fingerprint);
- } else if (documentType.equals(BandwidthDocument.class)) {
- directory = this.outDir;
- fileName = String.format("bandwidth/%s", fingerprint);
- } else if (documentType.equals(WeightsDocument.class)) {
- directory = this.outDir;
- fileName = String.format("weights/%s", fingerprint);
- } else if (documentType.equals(ClientsDocument.class)) {
- directory = this.outDir;
- fileName = String.format("clients/%s", fingerprint);
- } else if (documentType.equals(UptimeDocument.class)) {
- directory = this.outDir;
- fileName = String.format("uptimes/%s", fingerprint);
- }
- if (directory != null && fileName != null) {
- documentFile = new File(directory, fileName);
- }
- return documentFile;
- }
-
- public void flushDocumentCache() {
- /* Write cached node statuses to disk, and write update file
- * containing current time. It's important to write the update file
- * now, not earlier, because the front-end should not read new node
- * statuses until all details, bandwidths, and weights are ready. */
- if (this.cachedNodeStatuses != null ||
- this.cachedSummaryDocuments != null) {
- if (this.cachedNodeStatuses != null) {
- this.writeNodeStatuses();
- }
- if (this.cachedSummaryDocuments != null) {
- this.writeSummaryDocuments();
- }
- this.writeUpdateStatus();
- }
- }
-
- private void writeNodeStatuses() {
- File directory = this.statusDir;
- if (directory == null) {
- return;
- }
- File summaryFile = new File(directory, "summary");
- SortedMap<String, NodeStatus>
- cachedRelays = new TreeMap<String, NodeStatus>(),
- cachedBridges = new TreeMap<String, NodeStatus>();
- for (Map.Entry<String, NodeStatus> e :
- this.cachedNodeStatuses.entrySet()) {
- if (e.getValue().isRelay()) {
- cachedRelays.put(e.getKey(), e.getValue());
- } else {
- cachedBridges.put(e.getKey(), e.getValue());
- }
- }
- StringBuilder sb = new StringBuilder();
- for (NodeStatus relay : cachedRelays.values()) {
- String line = relay.toString();
- if (line != null) {
- sb.append(line + "\n");
- } else {
- System.err.println("Could not serialize relay node status '"
- + relay.getFingerprint() + "'");
- }
- }
- for (NodeStatus bridge : cachedBridges.values()) {
- String line = bridge.toString();
- if (line != null) {
- sb.append(line + "\n");
- } else {
- System.err.println("Could not serialize bridge node status '"
- + bridge.getFingerprint() + "'");
- }
- }
- String documentString = sb.toString();
- try {
- summaryFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(summaryFile));
- bw.write(documentString);
- bw.close();
- this.storedFiles++;
- this.storedBytes += documentString.length();
- } catch (IOException e) {
- System.err.println("Could not write file '"
- + summaryFile.getAbsolutePath() + "'.");
- e.printStackTrace();
- }
- }
-
- private void writeSummaryDocuments() {
- StringBuilder sb = new StringBuilder();
- Gson gson = new Gson();
- for (SummaryDocument summaryDocument :
- this.cachedSummaryDocuments.values()) {
- String line = gson.toJson(summaryDocument);
- if (line != null) {
- sb.append(line + "\n");
- } else {
- System.err.println("Could not serialize relay summary document '"
- + summaryDocument.getFingerprint() + "'");
- }
- }
- String documentString = sb.toString();
- File summaryFile = new File(this.outDir, "summary");
- try {
- summaryFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(summaryFile));
- bw.write(documentString);
- bw.close();
- this.storedFiles++;
- this.storedBytes += documentString.length();
- } catch (IOException e) {
- System.err.println("Could not write file '"
- + summaryFile.getAbsolutePath() + "'.");
- e.printStackTrace();
- }
- }
-
- private void writeUpdateStatus() {
- if (this.outDir == null) {
- return;
- }
- File updateFile = new File(this.outDir, "update");
- String documentString = String.valueOf(this.time.currentTimeMillis());
- try {
- updateFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(updateFile));
- bw.write(documentString);
- bw.close();
- this.storedFiles++;
- this.storedBytes += documentString.length();
- } catch (IOException e) {
- System.err.println("Could not write file '"
- + updateFile.getAbsolutePath() + "'.");
- e.printStackTrace();
- }
- }
-
- public String getStatsString() {
- StringBuilder sb = new StringBuilder();
- sb.append(" " + Logger.formatDecimalNumber(listOperations)
- + " list operations performed\n");
- sb.append(" " + Logger.formatDecimalNumber(listedFiles)
- + " files listed\n");
- sb.append(" " + Logger.formatDecimalNumber(storedFiles)
- + " files stored\n");
- sb.append(" " + Logger.formatBytes(storedBytes) + " stored\n");
- sb.append(" " + Logger.formatDecimalNumber(retrievedFiles)
- + " files retrieved\n");
- sb.append(" " + Logger.formatBytes(retrievedBytes)
- + " retrieved\n");
- sb.append(" " + Logger.formatDecimalNumber(removedFiles)
- + " files removed\n");
- return sb.toString();
- }
-}
-
diff --git a/src/org/torproject/onionoo/DocumentWriter.java b/src/org/torproject/onionoo/DocumentWriter.java
deleted file mode 100644
index 4cdeef9..0000000
--- a/src/org/torproject/onionoo/DocumentWriter.java
+++ /dev/null
@@ -1,11 +0,0 @@
-/* Copyright 2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-public interface DocumentWriter {
-
- public abstract void writeDocuments();
-
- public abstract String getStatsString();
-}
-
diff --git a/src/org/torproject/onionoo/GraphHistory.java b/src/org/torproject/onionoo/GraphHistory.java
deleted file mode 100644
index f03be58..0000000
--- a/src/org/torproject/onionoo/GraphHistory.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/* Copyright 2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.List;
-
-public class GraphHistory {
-
- private String first;
- public void setFirst(String first) {
- this.first = first;
- }
- public String getFirst() {
- return this.first;
- }
-
- private String last;
- public void setLast(String last) {
- this.last = last;
- }
- public String getLast() {
- return this.last;
- }
-
- private Integer interval;
- public void setInterval(Integer interval) {
- this.interval = interval;
- }
- public Integer getInterval() {
- return this.interval;
- }
-
- private Double factor;
- public void setFactor(Double factor) {
- this.factor = factor;
- }
- public Double getFactor() {
- return this.factor;
- }
-
- private Integer count;
- public void setCount(Integer count) {
- this.count = count;
- }
- public Integer getCount() {
- return this.count;
- }
-
- private List<Integer> values;
- public void setValues(List<Integer> values) {
- this.values = values;
- }
- public List<Integer> getValues() {
- return this.values;
- }
-}
diff --git a/src/org/torproject/onionoo/LockFile.java b/src/org/torproject/onionoo/LockFile.java
deleted file mode 100644
index 768db53..0000000
--- a/src/org/torproject/onionoo/LockFile.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/* Copyright 2013 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-
-public class LockFile {
-
- private final File lockFile = new File("lock");
-
- public boolean acquireLock() {
- Time time = ApplicationFactory.getTime();
- try {
- if (this.lockFile.exists()) {
- return false;
- }
- if (this.lockFile.getParentFile() != null) {
- this.lockFile.getParentFile().mkdirs();
- }
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.lockFile));
- bw.append("" + time.currentTimeMillis() + "\n");
- bw.close();
- return true;
- } catch (IOException e) {
- System.err.println("Caught exception while trying to acquire "
- + "lock!");
- e.printStackTrace();
- return false;
- }
- }
-
- public boolean releaseLock() {
- if (this.lockFile.exists()) {
- this.lockFile.delete();
- }
- return !this.lockFile.exists();
- }
-}
-
diff --git a/src/org/torproject/onionoo/Logger.java b/src/org/torproject/onionoo/Logger.java
deleted file mode 100644
index 6906a6a..0000000
--- a/src/org/torproject/onionoo/Logger.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/* Copyright 2013 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.Date;
-
-public class Logger {
-
- private Logger() {
- }
-
- private static Time time;
-
- public static void setTime() {
- time = ApplicationFactory.getTime();
- }
-
- private static long currentTimeMillis() {
- if (time == null) {
- return System.currentTimeMillis();
- } else {
- return time.currentTimeMillis();
- }
- }
-
- public static String formatDecimalNumber(long decimalNumber) {
- return String.format("%,d", decimalNumber);
- }
-
- public static String formatMillis(long millis) {
- return String.format("%02d:%02d.%03d minutes",
- millis / DateTimeHelper.ONE_MINUTE,
- (millis % DateTimeHelper.ONE_MINUTE) / DateTimeHelper.ONE_SECOND,
- millis % DateTimeHelper.ONE_SECOND);
- }
-
- public static String formatBytes(long bytes) {
- if (bytes < 1024) {
- return bytes + " B";
- } else {
- int exp = (int) (Math.log(bytes) / Math.log(1024));
- return String.format("%.1f %siB", bytes / Math.pow(1024, exp),
- "KMGTPE".charAt(exp-1));
- }
- }
-
- private static long printedLastStatusMessage = -1L;
-
- public static void printStatus(String message) {
- System.out.println(new Date() + ": " + message);
- printedLastStatusMessage = currentTimeMillis();
- }
-
- public static void printStatistics(String component, String message) {
- System.out.print(" " + component + " statistics:\n" + message);
- }
-
- public static void printStatusTime(String message) {
- printStatusOrErrorTime(message, false);
- }
-
- public static void printErrorTime(String message) {
- printStatusOrErrorTime(message, true);
- }
-
- private static void printStatusOrErrorTime(String message,
- boolean printToSystemErr) {
- long now = currentTimeMillis();
- long millis = printedLastStatusMessage < 0 ? 0 :
- now - printedLastStatusMessage;
- String line = " " + message + " (" + Logger.formatMillis(millis)
- + ").";
- if (printToSystemErr) {
- System.err.println(line);
- } else {
- System.out.println(line);
- }
- printedLastStatusMessage = now;
- }
-}
-
diff --git a/src/org/torproject/onionoo/LookupService.java b/src/org/torproject/onionoo/LookupService.java
deleted file mode 100644
index a968bc0..0000000
--- a/src/org/torproject/onionoo/LookupService.java
+++ /dev/null
@@ -1,408 +0,0 @@
-/* Copyright 2013 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.regex.Pattern;
-
-class LookupResult {
-
- private String countryCode;
- public void setCountryCode(String countryCode) {
- this.countryCode = countryCode;
- }
- public String getCountryCode() {
- return this.countryCode;
- }
-
- private String countryName;
- public void setCountryName(String countryName) {
- this.countryName = countryName;
- }
- public String getCountryName() {
- return this.countryName;
- }
-
- private String regionName;
- public void setRegionName(String regionName) {
- this.regionName = regionName;
- }
- public String getRegionName() {
- return this.regionName;
- }
-
- private String cityName;
- public void setCityName(String cityName) {
- this.cityName = cityName;
- }
- public String getCityName() {
- return this.cityName;
- }
-
- private Float latitude;
- public void setLatitude(Float latitude) {
- this.latitude = latitude;
- }
- public Float getLatitude() {
- return this.latitude;
- }
-
- private Float longitude;
- public void setLongitude(Float longitude) {
- this.longitude = longitude;
- }
- public Float getLongitude() {
- return this.longitude;
- }
-
- private String asNumber;
- public void setAsNumber(String asNumber) {
- this.asNumber = asNumber;
- }
- public String getAsNumber() {
- return this.asNumber;
- }
-
- private String asName;
- public void setAsName(String asName) {
- this.asName = asName;
- }
- public String getAsName() {
- return this.asName;
- }
-}
-
-public class LookupService {
-
- private File geoipDir;
- private File geoLite2CityBlocksCsvFile;
- private File geoLite2CityLocationsCsvFile;
- private File geoIPASNum2CsvFile;
- private boolean hasAllFiles = false;
- public LookupService(File geoipDir) {
- this.geoipDir = geoipDir;
- this.findRequiredCsvFiles();
- }
-
- /* Make sure we have all required .csv files. */
- private void findRequiredCsvFiles() {
- this.geoLite2CityBlocksCsvFile = new File(this.geoipDir,
- "GeoLite2-City-Blocks.csv");
- if (!this.geoLite2CityBlocksCsvFile.exists()) {
- System.err.println("No GeoLite2-City-Blocks.csv file in geoip/.");
- return;
- }
- this.geoLite2CityLocationsCsvFile = new File(this.geoipDir,
- "GeoLite2-City-Locations.csv");
- if (!this.geoLite2CityLocationsCsvFile.exists()) {
- System.err.println("No GeoLite2-City-Locations.csv file in "
- + "geoip/.");
- return;
- }
- this.geoIPASNum2CsvFile = new File(this.geoipDir, "GeoIPASNum2.csv");
- if (!this.geoIPASNum2CsvFile.exists()) {
- System.err.println("No GeoIPASNum2.csv file in geoip/.");
- return;
- }
- this.hasAllFiles = true;
- }
-
- private Pattern ipv4Pattern = Pattern.compile("^[0-9\\.]{7,15}$");
- private long parseAddressString(String addressString) {
- long addressNumber = -1L;
- if (ipv4Pattern.matcher(addressString).matches()) {
- String[] parts = addressString.split("\\.", 4);
- if (parts.length == 4) {
- addressNumber = 0L;
- for (int i = 0; i < 4; i++) {
- addressNumber *= 256L;
- int octetValue = -1;
- try {
- octetValue = Integer.parseInt(parts[i]);
- } catch (NumberFormatException e) {
- }
- if (octetValue < 0 || octetValue > 255) {
- addressNumber = -1L;
- break;
- }
- addressNumber += octetValue;
- }
- }
- }
- return addressNumber;
- }
-
- public SortedMap<String, LookupResult> lookup(
- SortedSet<String> addressStrings) {
-
- SortedMap<String, LookupResult> lookupResults =
- new TreeMap<String, LookupResult>();
-
- if (!this.hasAllFiles) {
- return lookupResults;
- }
-
- /* Obtain a map from relay IP address strings to numbers. */
- Map<String, Long> addressStringNumbers = new HashMap<String, Long>();
- for (String addressString : addressStrings) {
- long addressNumber = this.parseAddressString(addressString);
- if (addressNumber >= 0L) {
- addressStringNumbers.put(addressString, addressNumber);
- }
- }
- if (addressStringNumbers.isEmpty()) {
- return lookupResults;
- }
-
- /* Obtain a map from IP address numbers to blocks and to latitudes and
- longitudes. */
- Map<Long, Long> addressNumberBlocks = new HashMap<Long, Long>();
- Map<Long, Float[]> addressNumberLatLong =
- new HashMap<Long, Float[]>();
- try {
- SortedSet<Long> sortedAddressNumbers = new TreeSet<Long>(
- addressStringNumbers.values());
- BufferedReader br = new BufferedReader(new InputStreamReader(
- new FileInputStream(geoLite2CityBlocksCsvFile), "ISO-8859-1"));
- String line = br.readLine();
- while ((line = br.readLine()) != null) {
- if (!line.startsWith("::ffff:")) {
- /* TODO Make this less hacky and IPv6-ready at some point. */
- continue;
- }
- String[] parts = line.replaceAll("\"", "").split(",", 10);
- if (parts.length != 10) {
- System.err.println("Illegal line '" + line + "' in "
- + geoLite2CityBlocksCsvFile.getAbsolutePath() + ".");
- br.close();
- return lookupResults;
- }
- try {
- String startAddressString = parts[0].substring(7); /* ::ffff: */
- long startIpNum = this.parseAddressString(startAddressString);
- if (startIpNum < 0L) {
- System.err.println("Illegal IP address in '" + line
- + "' in " + geoLite2CityBlocksCsvFile.getAbsolutePath()
- + ".");
- br.close();
- return lookupResults;
- }
- int networkMaskLength = Integer.parseInt(parts[1]);
- if (networkMaskLength < 96 || networkMaskLength > 128) {
- System.err.println("Illegal network mask in '" + line
- + "' in " + geoLite2CityBlocksCsvFile.getAbsolutePath()
- + ".");
- br.close();
- return lookupResults;
- }
- if (parts[2].length() == 0 && parts[3].length() == 0) {
- continue;
- }
- long endIpNum = startIpNum + (1 << (128 - networkMaskLength))
- - 1;
- for (long addressNumber : sortedAddressNumbers.
- tailSet(startIpNum).headSet(endIpNum + 1L)) {
- String blockString = parts[2].length() > 0 ? parts[2] :
- parts[3];
- long blockNumber = Long.parseLong(blockString);
- addressNumberBlocks.put(addressNumber, blockNumber);
- if (parts[6].length() > 0 && parts[7].length() > 0) {
- addressNumberLatLong.put(addressNumber,
- new Float[] { Float.parseFloat(parts[6]),
- Float.parseFloat(parts[7]) });
- }
- }
- } catch (NumberFormatException e) {
- System.err.println("Number format exception while parsing line "
- + "'" + line + "' in "
- + geoLite2CityBlocksCsvFile.getAbsolutePath() + ".");
- br.close();
- return lookupResults;
- }
- }
- br.close();
- } catch (IOException e) {
- System.err.println("I/O exception while reading "
- + geoLite2CityBlocksCsvFile.getAbsolutePath() + ".");
- return lookupResults;
- }
-
- /* Obtain a map from relevant blocks to location lines. */
- Map<Long, String> blockLocations = new HashMap<Long, String>();
- try {
- Set<Long> blockNumbers = new HashSet<Long>(
- addressNumberBlocks.values());
- BufferedReader br = new BufferedReader(new InputStreamReader(
- new FileInputStream(geoLite2CityLocationsCsvFile),
- "ISO-8859-1"));
- String line = br.readLine();
- while ((line = br.readLine()) != null) {
- String[] parts = line.replaceAll("\"", "").split(",", 10);
- if (parts.length != 10) {
- System.err.println("Illegal line '" + line + "' in "
- + geoLite2CityLocationsCsvFile.getAbsolutePath() + ".");
- br.close();
- return lookupResults;
- }
- try {
- long locId = Long.parseLong(parts[0]);
- if (blockNumbers.contains(locId)) {
- blockLocations.put(locId, line);
- }
- } catch (NumberFormatException e) {
- System.err.println("Number format exception while parsing line "
- + "'" + line + "' in "
- + geoLite2CityLocationsCsvFile.getAbsolutePath() + ".");
- br.close();
- return lookupResults;
- }
- }
- br.close();
- } catch (IOException e) {
- System.err.println("I/O exception while reading "
- + geoLite2CityLocationsCsvFile.getAbsolutePath() + ".");
- return lookupResults;
- }
-
- /* Obtain a map from IP address numbers to ASN. */
- Map<Long, String> addressNumberASN = new HashMap<Long, String>();
- try {
- SortedSet<Long> sortedAddressNumbers = new TreeSet<Long>(
- addressStringNumbers.values());
- long firstAddressNumber = sortedAddressNumbers.first();
- BufferedReader br = new BufferedReader(new InputStreamReader(
- new FileInputStream(geoIPASNum2CsvFile), "ISO-8859-1"));
- String line;
- long previousStartIpNum = -1L;
- while ((line = br.readLine()) != null) {
- String[] parts = line.replaceAll("\"", "").split(",", 3);
- if (parts.length != 3) {
- System.err.println("Illegal line '" + line + "' in "
- + geoIPASNum2CsvFile.getAbsolutePath() + ".");
- br.close();
- return lookupResults;
- }
- try {
- long startIpNum = Long.parseLong(parts[0]);
- if (startIpNum <= previousStartIpNum) {
- System.err.println("Line '" + line + "' not sorted in "
- + geoIPASNum2CsvFile.getAbsolutePath() + ".");
- br.close();
- return lookupResults;
- }
- previousStartIpNum = startIpNum;
- while (firstAddressNumber < startIpNum &&
- firstAddressNumber != -1L) {
- sortedAddressNumbers.remove(firstAddressNumber);
- if (sortedAddressNumbers.isEmpty()) {
- firstAddressNumber = -1L;
- } else {
- firstAddressNumber = sortedAddressNumbers.first();
- }
- }
- long endIpNum = Long.parseLong(parts[1]);
- while (firstAddressNumber <= endIpNum &&
- firstAddressNumber != -1L) {
- if (parts[2].startsWith("AS") &&
- parts[2].split(" ", 2).length == 2) {
- addressNumberASN.put(firstAddressNumber, parts[2]);
- }
- sortedAddressNumbers.remove(firstAddressNumber);
- if (sortedAddressNumbers.isEmpty()) {
- firstAddressNumber = -1L;
- } else {
- firstAddressNumber = sortedAddressNumbers.first();
- }
- }
- if (firstAddressNumber == -1L) {
- break;
- }
- }
- catch (NumberFormatException e) {
- System.err.println("Number format exception while parsing line "
- + "'" + line + "' in "
- + geoIPASNum2CsvFile.getAbsolutePath() + ".");
- br.close();
- return lookupResults;
- }
- }
- br.close();
- } catch (IOException e) {
- System.err.println("I/O exception while reading "
- + geoIPASNum2CsvFile.getAbsolutePath() + ".");
- return lookupResults;
- }
-
- /* Finally, put together lookup results. */
- for (String addressString : addressStrings) {
- if (!addressStringNumbers.containsKey(addressString)) {
- continue;
- }
- long addressNumber = addressStringNumbers.get(addressString);
- if (!addressNumberBlocks.containsKey(addressNumber) &&
- !addressNumberLatLong.containsKey(addressNumber) &&
- !addressNumberASN.containsKey(addressNumber)) {
- continue;
- }
- LookupResult lookupResult = new LookupResult();
- if (addressNumberBlocks.containsKey(addressNumber)) {
- long blockNumber = addressNumberBlocks.get(addressNumber);
- if (blockLocations.containsKey(blockNumber)) {
- String[] parts = blockLocations.get(blockNumber).
- replaceAll("\"", "").split(",", -1);
- lookupResult.setCountryCode(parts[3].toLowerCase());
- if (parts[4].length() > 0) {
- lookupResult.setCountryName(parts[4]);
- }
- if (parts[6].length() > 0) {
- lookupResult.setRegionName(parts[6]);
- }
- if (parts[7].length() > 0) {
- lookupResult.setCityName(parts[7]);
- }
- }
- }
- if (addressNumberLatLong.containsKey(addressNumber)) {
- Float[] latLong = addressNumberLatLong.get(addressNumber);
- lookupResult.setLatitude(latLong[0]);
- lookupResult.setLongitude(latLong[1]);
- }
- if (addressNumberASN.containsKey(addressNumber)) {
- String[] parts = addressNumberASN.get(addressNumber).split(" ",
- 2);
- lookupResult.setAsNumber(parts[0]);
- lookupResult.setAsName(parts[1]);
- }
- lookupResults.put(addressString, lookupResult);
- }
-
- /* Keep statistics. */
- this.addressesLookedUp += addressStrings.size();
- this.addressesResolved += lookupResults.size();
-
- return lookupResults;
- }
-
- private int addressesLookedUp = 0, addressesResolved = 0;
-
- public String getStatsString() {
- StringBuilder sb = new StringBuilder();
- sb.append(" " + Logger.formatDecimalNumber(addressesLookedUp)
- + " addresses looked up\n");
- sb.append(" " + Logger.formatDecimalNumber(addressesResolved)
- + " addresses resolved\n");
- return sb.toString();
- }
-}
diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java
deleted file mode 100644
index e9afe2f..0000000
--- a/src/org/torproject/onionoo/Main.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/* Copyright 2011, 2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.io.File;
-
-/* Update search data and status data files. */
-public class Main {
-
- private Main() {
- }
-
- public static void main(String[] args) {
-
- LockFile lf = new LockFile();
- Logger.setTime();
- Logger.printStatus("Initializing.");
- if (lf.acquireLock()) {
- Logger.printStatusTime("Acquired lock");
- } else {
- Logger.printErrorTime("Could not acquire lock. Is Onionoo "
- + "already running? Terminating");
- return;
- }
-
- DescriptorSource dso = ApplicationFactory.getDescriptorSource();
- Logger.printStatusTime("Initialized descriptor source");
- DocumentStore ds = ApplicationFactory.getDocumentStore();
- Logger.printStatusTime("Initialized document store");
- LookupService ls = new LookupService(new File("geoip"));
- Logger.printStatusTime("Initialized Geoip lookup service");
- ReverseDomainNameResolver rdnr = new ReverseDomainNameResolver();
- Logger.printStatusTime("Initialized reverse domain name resolver");
- NodeDetailsStatusUpdater ndsu = new NodeDetailsStatusUpdater(rdnr,
- ls);
- Logger.printStatusTime("Initialized node data writer");
- BandwidthStatusUpdater bsu = new BandwidthStatusUpdater();
- Logger.printStatusTime("Initialized bandwidth status updater");
- WeightsStatusUpdater wsu = new WeightsStatusUpdater();
- Logger.printStatusTime("Initialized weights status updater");
- ClientsStatusUpdater csu = new ClientsStatusUpdater();
- Logger.printStatusTime("Initialized clients status updater");
- UptimeStatusUpdater usu = new UptimeStatusUpdater();
- Logger.printStatusTime("Initialized uptime status updater");
- StatusUpdater[] sus = new StatusUpdater[] { ndsu, bsu, wsu, csu,
- usu };
-
- SummaryDocumentWriter sdw = new SummaryDocumentWriter();
- Logger.printStatusTime("Initialized summary document writer");
- DetailsDocumentWriter ddw = new DetailsDocumentWriter();
- Logger.printStatusTime("Initialized details document writer");
- BandwidthDocumentWriter bdw = new BandwidthDocumentWriter();
- Logger.printStatusTime("Initialized bandwidth document writer");
- WeightsDocumentWriter wdw = new WeightsDocumentWriter();
- Logger.printStatusTime("Initialized weights document writer");
- ClientsDocumentWriter cdw = new ClientsDocumentWriter();
- Logger.printStatusTime("Initialized clients document writer");
- UptimeDocumentWriter udw = new UptimeDocumentWriter();
- Logger.printStatusTime("Initialized uptime document writer");
- DocumentWriter[] dws = new DocumentWriter[] { sdw, ddw, bdw, wdw, cdw,
- udw };
-
- Logger.printStatus("Downloading descriptors.");
- dso.downloadDescriptors();
-
- Logger.printStatus("Reading descriptors.");
- dso.readDescriptors();
-
- Logger.printStatus("Updating internal status files.");
- for (StatusUpdater su : sus) {
- su.updateStatuses();
- Logger.printStatusTime(su.getClass().getSimpleName()
- + " updated status files");
- }
-
- Logger.printStatus("Updating document files.");
- for (DocumentWriter dw : dws) {
- dw.writeDocuments();
- }
-
- Logger.printStatus("Shutting down.");
- dso.writeHistoryFiles();
- Logger.printStatusTime("Wrote parse histories");
- ds.flushDocumentCache();
- Logger.printStatusTime("Flushed document cache");
-
- Logger.printStatus("Gathering statistics.");
- for (StatusUpdater su : sus) {
- String statsString = su.getStatsString();
- if (statsString != null) {
- Logger.printStatistics(su.getClass().getSimpleName(),
- statsString);
- }
- }
- for (DocumentWriter dw : dws) {
- String statsString = dw.getStatsString();
- if (statsString != null) {
- Logger.printStatistics(dw.getClass().getSimpleName(),
- statsString);
- }
- }
- Logger.printStatistics("Descriptor source", dso.getStatsString());
- Logger.printStatistics("Document store", ds.getStatsString());
- Logger.printStatistics("GeoIP lookup service", ls.getStatsString());
- Logger.printStatistics("Reverse domain name resolver",
- rdnr.getStatsString());
-
- Logger.printStatus("Releasing lock.");
- if (lf.releaseLock()) {
- Logger.printStatusTime("Released lock");
- } else {
- Logger.printErrorTime("Could not release lock. The next "
- + "execution may not start as expected");
- }
-
- Logger.printStatus("Terminating.");
- }
-}
-
diff --git a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java
deleted file mode 100644
index a1149a5..0000000
--- a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java
+++ /dev/null
@@ -1,620 +0,0 @@
-/* Copyright 2011--2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.torproject.descriptor.BridgeNetworkStatus;
-import org.torproject.descriptor.BridgePoolAssignment;
-import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.ExitList;
-import org.torproject.descriptor.ExitListEntry;
-import org.torproject.descriptor.NetworkStatusEntry;
-import org.torproject.descriptor.RelayNetworkStatusConsensus;
-import org.torproject.descriptor.ServerDescriptor;
-
-public class NodeDetailsStatusUpdater implements DescriptorListener,
- StatusUpdater {
-
- private DescriptorSource descriptorSource;
-
- private ReverseDomainNameResolver reverseDomainNameResolver;
-
- private LookupService lookupService;
-
- private DocumentStore documentStore;
-
- private long now;
-
- private SortedMap<String, NodeStatus> knownNodes =
- new TreeMap<String, NodeStatus>();
-
- private SortedMap<String, NodeStatus> relays;
-
- private SortedMap<String, NodeStatus> bridges;
-
- private long relaysLastValidAfterMillis = -1L;
-
- private long bridgesLastPublishedMillis = -1L;
-
- private SortedMap<String, Integer> lastBandwidthWeights = null;
-
- private int relayConsensusesProcessed = 0, bridgeStatusesProcessed = 0;
-
- public NodeDetailsStatusUpdater(
- ReverseDomainNameResolver reverseDomainNameResolver,
- LookupService lookupService) {
- this.descriptorSource = ApplicationFactory.getDescriptorSource();
- this.reverseDomainNameResolver = reverseDomainNameResolver;
- this.lookupService = lookupService;
- this.documentStore = ApplicationFactory.getDocumentStore();
- this.now = ApplicationFactory.getTime().currentTimeMillis();
- this.registerDescriptorListeners();
- }
-
- private void registerDescriptorListeners() {
- this.descriptorSource.registerDescriptorListener(this,
- DescriptorType.RELAY_CONSENSUSES);
- this.descriptorSource.registerDescriptorListener(this,
- DescriptorType.RELAY_SERVER_DESCRIPTORS);
- this.descriptorSource.registerDescriptorListener(this,
- DescriptorType.BRIDGE_STATUSES);
- this.descriptorSource.registerDescriptorListener(this,
- DescriptorType.BRIDGE_SERVER_DESCRIPTORS);
- this.descriptorSource.registerDescriptorListener(this,
- DescriptorType.BRIDGE_POOL_ASSIGNMENTS);
- this.descriptorSource.registerDescriptorListener(this,
- DescriptorType.EXIT_LISTS);
- }
-
- public void processDescriptor(Descriptor descriptor, boolean relay) {
- if (descriptor instanceof ServerDescriptor && relay) {
- this.processRelayServerDescriptor((ServerDescriptor) descriptor);
- } else if (descriptor instanceof ExitList) {
- this.processExitList((ExitList) descriptor);
- } else if (descriptor instanceof RelayNetworkStatusConsensus) {
- this.processRelayNetworkStatusConsensus(
- (RelayNetworkStatusConsensus) descriptor);
- } else if (descriptor instanceof ServerDescriptor && !relay) {
- this.processBridgeServerDescriptor((ServerDescriptor) descriptor);
- } else if (descriptor instanceof BridgePoolAssignment) {
- this.processBridgePoolAssignment((BridgePoolAssignment) descriptor);
- } else if (descriptor instanceof BridgeNetworkStatus) {
- this.processBridgeNetworkStatus((BridgeNetworkStatus) descriptor);
- }
- }
-
- private void processRelayServerDescriptor(
- ServerDescriptor descriptor) {
- String fingerprint = descriptor.getFingerprint();
- DetailsStatus detailsStatus = this.documentStore.retrieve(
- DetailsStatus.class, true, fingerprint);
- String publishedDateTime =
- DateTimeHelper.format(descriptor.getPublishedMillis());
- if (detailsStatus == null) {
- detailsStatus = new DetailsStatus();
- } else if (detailsStatus.getDescPublished() != null &&
- publishedDateTime.compareTo(
- detailsStatus.getDescPublished()) < 0) {
- return;
- }
- String lastRestartedString = DateTimeHelper.format(
- descriptor.getPublishedMillis() - descriptor.getUptime()
- * DateTimeHelper.ONE_SECOND);
- int bandwidthRate = descriptor.getBandwidthRate();
- int bandwidthBurst = descriptor.getBandwidthBurst();
- int observedBandwidth = descriptor.getBandwidthObserved();
- int advertisedBandwidth = Math.min(bandwidthRate,
- Math.min(bandwidthBurst, observedBandwidth));
- detailsStatus.setDescPublished(publishedDateTime);
- detailsStatus.setLastRestarted(lastRestartedString);
- detailsStatus.setBandwidthRate(bandwidthRate);
- detailsStatus.setBandwidthBurst(bandwidthBurst);
- detailsStatus.setObservedBandwidth(observedBandwidth);
- detailsStatus.setAdvertisedBandwidth(advertisedBandwidth);
- detailsStatus.setExitPolicy(descriptor.getExitPolicyLines());
- detailsStatus.setContact(descriptor.getContact());
- detailsStatus.setPlatform(descriptor.getPlatform());
- detailsStatus.setFamily(descriptor.getFamilyEntries());
- if (descriptor.getIpv6DefaultPolicy() != null &&
- (descriptor.getIpv6DefaultPolicy().equals("accept") ||
- descriptor.getIpv6DefaultPolicy().equals("reject")) &&
- descriptor.getIpv6PortList() != null) {
- Map<String, List<String>> exitPolicyV6Summary =
- new HashMap<String, List<String>>();
- List<String> portsOrPortRanges = Arrays.asList(
- descriptor.getIpv6PortList().split(","));
- exitPolicyV6Summary.put(descriptor.getIpv6DefaultPolicy(),
- portsOrPortRanges);
- detailsStatus.setExitPolicyV6Summary(exitPolicyV6Summary);
- }
- if (descriptor.isHibernating()) {
- detailsStatus.setHibernating(true);
- }
- this.documentStore.store(detailsStatus, fingerprint);
- }
-
- private Map<String, Map<String, Long>> exitListEntries =
- new HashMap<String, Map<String, Long>>();
-
- private void processExitList(ExitList exitList) {
- for (ExitListEntry exitListEntry : exitList.getExitListEntries()) {
- String fingerprint = exitListEntry.getFingerprint();
- if (exitListEntry.getScanMillis() <
- this.now - DateTimeHelper.ONE_DAY) {
- continue;
- }
- if (!this.exitListEntries.containsKey(fingerprint)) {
- this.exitListEntries.put(fingerprint,
- new HashMap<String, Long>());
- }
- String exitAddress = exitListEntry.getExitAddress();
- long scanMillis = exitListEntry.getScanMillis();
- if (!this.exitListEntries.get(fingerprint).containsKey(exitAddress)
- || this.exitListEntries.get(fingerprint).get(exitAddress)
- < scanMillis) {
- this.exitListEntries.get(fingerprint).put(exitAddress,
- scanMillis);
- }
- }
- }
-
- private void processRelayNetworkStatusConsensus(
- RelayNetworkStatusConsensus consensus) {
- long validAfterMillis = consensus.getValidAfterMillis();
- if (validAfterMillis > this.relaysLastValidAfterMillis) {
- this.relaysLastValidAfterMillis = validAfterMillis;
- }
- Set<String> recommendedVersions = null;
- if (consensus.getRecommendedServerVersions() != null) {
- recommendedVersions = new HashSet<String>();
- for (String recommendedVersion :
- consensus.getRecommendedServerVersions()) {
- recommendedVersions.add("Tor " + recommendedVersion);
- }
- }
- for (NetworkStatusEntry entry :
- consensus.getStatusEntries().values()) {
- String nickname = entry.getNickname();
- String fingerprint = entry.getFingerprint();
- String address = entry.getAddress();
- SortedSet<String> orAddressesAndPorts = new TreeSet<String>(
- entry.getOrAddresses());
- int orPort = entry.getOrPort();
- int dirPort = entry.getDirPort();
- SortedSet<String> relayFlags = entry.getFlags();
- long consensusWeight = entry.getBandwidth();
- String defaultPolicy = entry.getDefaultPolicy();
- String portList = entry.getPortList();
- Boolean recommendedVersion = (recommendedVersions == null ||
- entry.getVersion() == null) ? null :
- recommendedVersions.contains(entry.getVersion());
- NodeStatus newNodeStatus = new NodeStatus(true, nickname,
- fingerprint, address, orAddressesAndPorts, null,
- validAfterMillis, orPort, dirPort, relayFlags, consensusWeight,
- null, null, -1L, defaultPolicy, portList, validAfterMillis,
- validAfterMillis, null, null, recommendedVersion, null);
- if (this.knownNodes.containsKey(fingerprint)) {
- this.knownNodes.get(fingerprint).update(newNodeStatus);
- } else {
- this.knownNodes.put(fingerprint, newNodeStatus);
- }
- }
- this.relayConsensusesProcessed++;
- if (this.relaysLastValidAfterMillis == validAfterMillis) {
- this.lastBandwidthWeights = consensus.getBandwidthWeights();
- }
- }
-
- private void processBridgeServerDescriptor(
- ServerDescriptor descriptor) {
- String fingerprint = descriptor.getFingerprint();
- DetailsStatus detailsStatus = this.documentStore.retrieve(
- DetailsStatus.class, true, fingerprint);
- String publishedDateTime =
- DateTimeHelper.format(descriptor.getPublishedMillis());
- if (detailsStatus == null) {
- detailsStatus = new DetailsStatus();
- } else if (detailsStatus.getDescPublished() != null &&
- publishedDateTime.compareTo(
- detailsStatus.getDescPublished()) < 0) {
- return;
- }
- String lastRestartedString = DateTimeHelper.format(
- descriptor.getPublishedMillis() - descriptor.getUptime()
- * DateTimeHelper.ONE_SECOND);
- int advertisedBandwidth = Math.min(descriptor.getBandwidthRate(),
- Math.min(descriptor.getBandwidthBurst(),
- descriptor.getBandwidthObserved()));
- detailsStatus.setDescPublished(publishedDateTime);
- detailsStatus.setLastRestarted(lastRestartedString);
- detailsStatus.setAdvertisedBandwidth(advertisedBandwidth);
- detailsStatus.setPlatform(descriptor.getPlatform());
- this.documentStore.store(detailsStatus, fingerprint);
- }
-
- private void processBridgePoolAssignment(
- BridgePoolAssignment bridgePoolAssignment) {
- for (Map.Entry<String, String> e :
- bridgePoolAssignment.getEntries().entrySet()) {
- String fingerprint = e.getKey();
- String details = e.getValue();
- DetailsStatus detailsStatus = this.documentStore.retrieve(
- DetailsStatus.class, true, fingerprint);
- if (detailsStatus == null) {
- detailsStatus = new DetailsStatus();
- } else if (details.equals(detailsStatus.getPoolAssignment())) {
- continue;
- }
- detailsStatus.setPoolAssignment(details);
- this.documentStore.store(detailsStatus, fingerprint);
- }
- }
-
- private void processBridgeNetworkStatus(BridgeNetworkStatus status) {
- long publishedMillis = status.getPublishedMillis();
- if (publishedMillis > this.bridgesLastPublishedMillis) {
- this.bridgesLastPublishedMillis = publishedMillis;
- }
- for (NetworkStatusEntry entry : status.getStatusEntries().values()) {
- String nickname = entry.getNickname();
- String fingerprint = entry.getFingerprint();
- String address = entry.getAddress();
- SortedSet<String> orAddressesAndPorts = new TreeSet<String>(
- entry.getOrAddresses());
- int orPort = entry.getOrPort();
- int dirPort = entry.getDirPort();
- SortedSet<String> relayFlags = entry.getFlags();
- NodeStatus newNodeStatus = new NodeStatus(false, nickname,
- fingerprint, address, orAddressesAndPorts, null,
- publishedMillis, orPort, dirPort, relayFlags, -1L, "??", null,
- -1L, null, null, publishedMillis, -1L, null, null, null, null);
- if (this.knownNodes.containsKey(fingerprint)) {
- this.knownNodes.get(fingerprint).update(newNodeStatus);
- } else {
- this.knownNodes.put(fingerprint, newNodeStatus);
- }
- }
- this.bridgeStatusesProcessed++;
- }
-
- public void updateStatuses() {
- this.readStatusSummary();
- Logger.printStatusTime("Read status summary");
- this.setCurrentNodes();
- Logger.printStatusTime("Set current node fingerprints");
- this.startReverseDomainNameLookups();
- Logger.printStatusTime("Started reverse domain name lookups");
- this.lookUpCitiesAndASes();
- Logger.printStatusTime("Looked up cities and ASes");
- this.setDescriptorPartsOfNodeStatus();
- Logger.printStatusTime("Set descriptor parts of node statuses.");
- this.calculatePathSelectionProbabilities();
- Logger.printStatusTime("Calculated path selection probabilities");
- this.finishReverseDomainNameLookups();
- Logger.printStatusTime("Finished reverse domain name lookups");
- this.writeStatusSummary();
- Logger.printStatusTime("Wrote status summary");
- this.updateDetailsStatuses();
- Logger.printStatusTime("Updated exit addresses in details statuses");
- }
-
- private void readStatusSummary() {
- SortedSet<String> fingerprints = this.documentStore.list(
- NodeStatus.class);
- for (String fingerprint : fingerprints) {
- NodeStatus node = this.documentStore.retrieve(NodeStatus.class,
- true, fingerprint);
- if (node.isRelay()) {
- this.relaysLastValidAfterMillis = Math.max(
- this.relaysLastValidAfterMillis, node.getLastSeenMillis());
- } else {
- this.bridgesLastPublishedMillis = Math.max(
- this.bridgesLastPublishedMillis, node.getLastSeenMillis());
- }
- if (this.knownNodes.containsKey(fingerprint)) {
- this.knownNodes.get(fingerprint).update(node);
- } else {
- this.knownNodes.put(fingerprint, node);
- }
- }
- }
-
- private void setCurrentNodes() {
- long cutoff = Math.max(this.relaysLastValidAfterMillis,
- this.bridgesLastPublishedMillis) - 7L * 24L * 60L * 60L * 1000L;
- SortedMap<String, NodeStatus> currentNodes =
- new TreeMap<String, NodeStatus>();
- for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) {
- if (e.getValue().getLastSeenMillis() >= cutoff) {
- currentNodes.put(e.getKey(), e.getValue());
- }
- }
- this.relays = new TreeMap<String, NodeStatus>();
- this.bridges = new TreeMap<String, NodeStatus>();
- for (Map.Entry<String, NodeStatus> e : currentNodes.entrySet()) {
- if (e.getValue().isRelay()) {
- this.relays.put(e.getKey(), e.getValue());
- } else {
- this.bridges.put(e.getKey(), e.getValue());
- }
- }
- }
-
- private void startReverseDomainNameLookups() {
- Map<String, Long> addressLastLookupTimes =
- new HashMap<String, Long>();
- for (NodeStatus relay : relays.values()) {
- addressLastLookupTimes.put(relay.getAddress(),
- relay.getLastRdnsLookup());
- }
- this.reverseDomainNameResolver.setAddresses(addressLastLookupTimes);
- this.reverseDomainNameResolver.startReverseDomainNameLookups();
- }
-
- private void lookUpCitiesAndASes() {
- SortedSet<String> addressStrings = new TreeSet<String>();
- for (NodeStatus node : this.knownNodes.values()) {
- if (node.isRelay()) {
- addressStrings.add(node.getAddress());
- }
- }
- if (addressStrings.isEmpty()) {
- System.err.println("No relay IP addresses to resolve to cities or "
- + "ASN.");
- return;
- }
- SortedMap<String, LookupResult> lookupResults =
- this.lookupService.lookup(addressStrings);
- for (NodeStatus node : knownNodes.values()) {
- if (!node.isRelay()) {
- continue;
- }
- String addressString = node.getAddress();
- if (lookupResults.containsKey(addressString)) {
- LookupResult lookupResult = lookupResults.get(addressString);
- node.setCountryCode(lookupResult.getCountryCode());
- node.setCountryName(lookupResult.getCountryName());
- node.setRegionName(lookupResult.getRegionName());
- node.setCityName(lookupResult.getCityName());
- node.setLatitude(lookupResult.getLatitude());
- node.setLongitude(lookupResult.getLongitude());
- node.setASNumber(lookupResult.getAsNumber());
- node.setASName(lookupResult.getAsName());
- }
- }
- }
-
- private void setDescriptorPartsOfNodeStatus() {
- for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) {
- String fingerprint = e.getKey();
- NodeStatus node = e.getValue();
- if (node.isRelay()) {
- if (node.getRelayFlags().contains("Running") &&
- node.getLastSeenMillis() == this.relaysLastValidAfterMillis) {
- node.setRunning(true);
- }
- DetailsStatus detailsStatus = this.documentStore.retrieve(
- DetailsStatus.class, true, fingerprint);
- if (detailsStatus != null) {
- node.setContact(detailsStatus.getContact());
- if (detailsStatus.getExitAddresses() != null) {
- for (Map.Entry<String, Long> ea :
- detailsStatus.getExitAddresses().entrySet()) {
- if (ea.getValue() >= this.now - DateTimeHelper.ONE_DAY) {
- node.addExitAddress(ea.getKey());
- }
- }
- }
- if (detailsStatus.getFamily() != null &&
- !detailsStatus.getFamily().isEmpty()) {
- SortedSet<String> familyFingerprints = new TreeSet<String>();
- for (String familyMember : detailsStatus.getFamily()) {
- if (familyMember.startsWith("$") &&
- familyMember.length() == 41) {
- familyFingerprints.add(familyMember.substring(1));
- }
- }
- if (!familyFingerprints.isEmpty()) {
- node.setFamilyFingerprints(familyFingerprints);
- }
- }
- }
- }
- if (!node.isRelay() && node.getRelayFlags().contains("Running") &&
- node.getLastSeenMillis() == this.bridgesLastPublishedMillis) {
- node.setRunning(true);
- }
- }
- }
-
- private void calculatePathSelectionProbabilities() {
- boolean consensusContainsBandwidthWeights = false;
- double wgg = 0.0, wgd = 0.0, wmg = 0.0, wmm = 0.0, wme = 0.0,
- wmd = 0.0, wee = 0.0, wed = 0.0;
- if (this.lastBandwidthWeights != null) {
- SortedSet<String> weightKeys = new TreeSet<String>(Arrays.asList(
- "Wgg,Wgd,Wmg,Wmm,Wme,Wmd,Wee,Wed".split(",")));
- weightKeys.removeAll(this.lastBandwidthWeights.keySet());
- if (weightKeys.isEmpty()) {
- consensusContainsBandwidthWeights = true;
- wgg = ((double) this.lastBandwidthWeights.get("Wgg")) / 10000.0;
- wgd = ((double) this.lastBandwidthWeights.get("Wgd")) / 10000.0;
- wmg = ((double) this.lastBandwidthWeights.get("Wmg")) / 10000.0;
- wmm = ((double) this.lastBandwidthWeights.get("Wmm")) / 10000.0;
- wme = ((double) this.lastBandwidthWeights.get("Wme")) / 10000.0;
- wmd = ((double) this.lastBandwidthWeights.get("Wmd")) / 10000.0;
- wee = ((double) this.lastBandwidthWeights.get("Wee")) / 10000.0;
- wed = ((double) this.lastBandwidthWeights.get("Wed")) / 10000.0;
- }
- } else {
- System.err.println("Could not determine most recent Wxx parameter "
- + "values, probably because we didn't parse a consensus in "
- + "this execution. All relays' guard/middle/exit weights are "
- + "going to be 0.0.");
- }
- SortedMap<String, Double>
- advertisedBandwidths = new TreeMap<String, Double>(),
- consensusWeights = new TreeMap<String, Double>(),
- guardWeights = new TreeMap<String, Double>(),
- middleWeights = new TreeMap<String, Double>(),
- exitWeights = new TreeMap<String, Double>();
- double totalAdvertisedBandwidth = 0.0;
- double totalConsensusWeight = 0.0;
- double totalGuardWeight = 0.0;
- double totalMiddleWeight = 0.0;
- double totalExitWeight = 0.0;
- for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) {
- String fingerprint = e.getKey();
- NodeStatus relay = e.getValue();
- if (!relay.getRunning()) {
- continue;
- }
- boolean isExit = relay.getRelayFlags().contains("Exit") &&
- !relay.getRelayFlags().contains("BadExit");
- boolean isGuard = relay.getRelayFlags().contains("Guard");
- DetailsStatus detailsStatus = this.documentStore.retrieve(
- DetailsStatus.class, true, fingerprint);
- if (detailsStatus != null) {
- double advertisedBandwidth =
- detailsStatus.getAdvertisedBandwidth();
- if (advertisedBandwidth >= 0.0) {
- advertisedBandwidths.put(fingerprint, advertisedBandwidth);
- totalAdvertisedBandwidth += advertisedBandwidth;
- }
- }
- double consensusWeight = (double) relay.getConsensusWeight();
- consensusWeights.put(fingerprint, consensusWeight);
- totalConsensusWeight += consensusWeight;
- if (consensusContainsBandwidthWeights) {
- double guardWeight = consensusWeight,
- middleWeight = consensusWeight,
- exitWeight = consensusWeight;
- if (isGuard && isExit) {
- guardWeight *= wgd;
- middleWeight *= wmd;
- exitWeight *= wed;
- } else if (isGuard) {
- guardWeight *= wgg;
- middleWeight *= wmg;
- exitWeight = 0.0;
- } else if (isExit) {
- guardWeight = 0.0;
- middleWeight *= wme;
- exitWeight *= wee;
- } else {
- guardWeight = 0.0;
- middleWeight *= wmm;
- exitWeight = 0.0;
- }
- guardWeights.put(fingerprint, guardWeight);
- middleWeights.put(fingerprint, middleWeight);
- exitWeights.put(fingerprint, exitWeight);
- totalGuardWeight += guardWeight;
- totalMiddleWeight += middleWeight;
- totalExitWeight += exitWeight;
- }
- }
- for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) {
- String fingerprint = e.getKey();
- NodeStatus relay = e.getValue();
- if (advertisedBandwidths.containsKey(fingerprint)) {
- relay.setAdvertisedBandwidthFraction(advertisedBandwidths.get(
- fingerprint) / totalAdvertisedBandwidth);
- }
- if (consensusWeights.containsKey(fingerprint)) {
- relay.setConsensusWeightFraction(consensusWeights.get(fingerprint)
- / totalConsensusWeight);
- }
- if (guardWeights.containsKey(fingerprint)) {
- relay.setGuardProbability(guardWeights.get(fingerprint)
- / totalGuardWeight);
- }
- if (middleWeights.containsKey(fingerprint)) {
- relay.setMiddleProbability(middleWeights.get(fingerprint)
- / totalMiddleWeight);
- }
- if (exitWeights.containsKey(fingerprint)) {
- relay.setExitProbability(exitWeights.get(fingerprint)
- / totalExitWeight);
- }
- }
- }
-
- private void finishReverseDomainNameLookups() {
- this.reverseDomainNameResolver.finishReverseDomainNameLookups();
- Map<String, String> lookupResults =
- this.reverseDomainNameResolver.getLookupResults();
- long startedRdnsLookups =
- this.reverseDomainNameResolver.getLookupStartMillis();
- for (NodeStatus relay : relays.values()) {
- if (lookupResults.containsKey(relay.getAddress())) {
- relay.setHostName(lookupResults.get(relay.getAddress()));
- relay.setLastRdnsLookup(startedRdnsLookups);
- }
- }
- }
-
- private void writeStatusSummary() {
- for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) {
- this.documentStore.store(e.getValue(), e.getKey());
- }
- }
-
- private void updateDetailsStatuses() {
- SortedSet<String> fingerprints = new TreeSet<String>();
- fingerprints.addAll(this.exitListEntries.keySet());
- for (String fingerprint : fingerprints) {
- DetailsStatus detailsStatus = this.documentStore.retrieve(
- DetailsStatus.class, true, fingerprint);
- if (detailsStatus == null) {
- detailsStatus = new DetailsStatus();
- }
- Map<String, Long> exitAddresses = new HashMap<String, Long>();
- if (detailsStatus.getExitAddresses() != null) {
- for (Map.Entry<String, Long> e :
- detailsStatus.getExitAddresses().entrySet()) {
- if (e.getValue() >= this.now - DateTimeHelper.ONE_DAY) {
- exitAddresses.put(e.getKey(), e.getValue());
- }
- }
- }
- if (this.exitListEntries.containsKey(fingerprint)) {
- for (Map.Entry<String, Long> e :
- this.exitListEntries.get(fingerprint).entrySet()) {
- if (!exitAddresses.containsKey(e.getKey()) ||
- exitAddresses.get(e.getKey()) < e.getValue()) {
- exitAddresses.put(e.getKey(), e.getValue());
- }
- }
- }
- if (this.knownNodes.containsKey(fingerprint)) {
- for (String orAddress :
- this.knownNodes.get(fingerprint).getOrAddresses()) {
- this.exitListEntries.remove(orAddress);
- }
- }
- detailsStatus.setExitAddresses(exitAddresses);
- this.documentStore.store(detailsStatus, fingerprint);
- }
- }
-
- public String getStatsString() {
- StringBuilder sb = new StringBuilder();
- sb.append(" " + Logger.formatDecimalNumber(
- relayConsensusesProcessed) + " relay consensuses processed\n");
- sb.append(" " + Logger.formatDecimalNumber(bridgeStatusesProcessed)
- + " bridge statuses processed\n");
- return sb.toString();
- }
-}
-
diff --git a/src/org/torproject/onionoo/NodeIndexer.java b/src/org/torproject/onionoo/NodeIndexer.java
deleted file mode 100644
index d24284e..0000000
--- a/src/org/torproject/onionoo/NodeIndexer.java
+++ /dev/null
@@ -1,425 +0,0 @@
-package org.torproject.onionoo;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.TreeMap;
-
-import javax.servlet.ServletContext;
-import javax.servlet.ServletContextEvent;
-import javax.servlet.ServletContextListener;
-
-class NodeIndex {
-
- private String relaysPublishedString;
- public void setRelaysPublishedString(String relaysPublishedString) {
- this.relaysPublishedString = relaysPublishedString;
- }
- public String getRelaysPublishedString() {
- return relaysPublishedString;
- }
-
- private String bridgesPublishedString;
- public void setBridgesPublishedString(String bridgesPublishedString) {
- this.bridgesPublishedString = bridgesPublishedString;
- }
- public String getBridgesPublishedString() {
- return bridgesPublishedString;
- }
-
- private List<String> relaysByConsensusWeight;
- public void setRelaysByConsensusWeight(
- List<String> relaysByConsensusWeight) {
- this.relaysByConsensusWeight = relaysByConsensusWeight;
- }
- public List<String> getRelaysByConsensusWeight() {
- return relaysByConsensusWeight;
- }
-
-
- private Map<String, SummaryDocument> relayFingerprintSummaryLines;
- public void setRelayFingerprintSummaryLines(
- Map<String, SummaryDocument> relayFingerprintSummaryLines) {
- this.relayFingerprintSummaryLines = relayFingerprintSummaryLines;
- }
- public Map<String, SummaryDocument> getRelayFingerprintSummaryLines() {
- return this.relayFingerprintSummaryLines;
- }
-
- private Map<String, SummaryDocument> bridgeFingerprintSummaryLines;
- public void setBridgeFingerprintSummaryLines(
- Map<String, SummaryDocument> bridgeFingerprintSummaryLines) {
- this.bridgeFingerprintSummaryLines = bridgeFingerprintSummaryLines;
- }
- public Map<String, SummaryDocument> getBridgeFingerprintSummaryLines() {
- return this.bridgeFingerprintSummaryLines;
- }
-
- private Map<String, Set<String>> relaysByCountryCode = null;
- public void setRelaysByCountryCode(
- Map<String, Set<String>> relaysByCountryCode) {
- this.relaysByCountryCode = relaysByCountryCode;
- }
- public Map<String, Set<String>> getRelaysByCountryCode() {
- return relaysByCountryCode;
- }
-
- private Map<String, Set<String>> relaysByASNumber = null;
- public void setRelaysByASNumber(
- Map<String, Set<String>> relaysByASNumber) {
- this.relaysByASNumber = relaysByASNumber;
- }
- public Map<String, Set<String>> getRelaysByASNumber() {
- return relaysByASNumber;
- }
-
- private Map<String, Set<String>> relaysByFlag = null;
- public void setRelaysByFlag(Map<String, Set<String>> relaysByFlag) {
- this.relaysByFlag = relaysByFlag;
- }
- public Map<String, Set<String>> getRelaysByFlag() {
- return relaysByFlag;
- }
-
- private Map<String, Set<String>> bridgesByFlag = null;
- public void setBridgesByFlag(Map<String, Set<String>> bridgesByFlag) {
- this.bridgesByFlag = bridgesByFlag;
- }
- public Map<String, Set<String>> getBridgesByFlag() {
- return bridgesByFlag;
- }
-
- private Map<String, Set<String>> relaysByContact = null;
- public void setRelaysByContact(
- Map<String, Set<String>> relaysByContact) {
- this.relaysByContact = relaysByContact;
- }
- public Map<String, Set<String>> getRelaysByContact() {
- return relaysByContact;
- }
-
- private Map<String, Set<String>> relaysByFamily = null;
- public void setRelaysByFamily(Map<String, Set<String>> relaysByFamily) {
- this.relaysByFamily = relaysByFamily;
- }
- public Map<String, Set<String>> getRelaysByFamily() {
- return this.relaysByFamily;
- }
-
- private SortedMap<Integer, Set<String>> relaysByFirstSeenDays;
- public void setRelaysByFirstSeenDays(
- SortedMap<Integer, Set<String>> relaysByFirstSeenDays) {
- this.relaysByFirstSeenDays = relaysByFirstSeenDays;
- }
- public SortedMap<Integer, Set<String>> getRelaysByFirstSeenDays() {
- return relaysByFirstSeenDays;
- }
-
- private SortedMap<Integer, Set<String>> bridgesByFirstSeenDays;
- public void setBridgesByFirstSeenDays(
- SortedMap<Integer, Set<String>> bridgesByFirstSeenDays) {
- this.bridgesByFirstSeenDays = bridgesByFirstSeenDays;
- }
- public SortedMap<Integer, Set<String>> getBridgesByFirstSeenDays() {
- return bridgesByFirstSeenDays;
- }
-
- private SortedMap<Integer, Set<String>> relaysByLastSeenDays;
- public void setRelaysByLastSeenDays(
- SortedMap<Integer, Set<String>> relaysByLastSeenDays) {
- this.relaysByLastSeenDays = relaysByLastSeenDays;
- }
- public SortedMap<Integer, Set<String>> getRelaysByLastSeenDays() {
- return relaysByLastSeenDays;
- }
-
- private SortedMap<Integer, Set<String>> bridgesByLastSeenDays;
- public void setBridgesByLastSeenDays(
- SortedMap<Integer, Set<String>> bridgesByLastSeenDays) {
- this.bridgesByLastSeenDays = bridgesByLastSeenDays;
- }
- public SortedMap<Integer, Set<String>> getBridgesByLastSeenDays() {
- return bridgesByLastSeenDays;
- }
-}
-
-public class NodeIndexer implements ServletContextListener, Runnable {
-
- public void contextInitialized(ServletContextEvent contextEvent) {
- ServletContext servletContext = contextEvent.getServletContext();
- File outDir = new File(servletContext.getInitParameter("outDir"));
- DocumentStore documentStore = ApplicationFactory.getDocumentStore();
- documentStore.setOutDir(outDir);
- /* The servlet container created us, and we need to avoid that
- * ApplicationFactory creates another instance of us. */
- ApplicationFactory.setNodeIndexer(this);
- this.startIndexing();
- }
-
- public void contextDestroyed(ServletContextEvent contextEvent) {
- this.stopIndexing();
- }
-
- private long lastIndexed = -1L;
-
- private NodeIndex latestNodeIndex = null;
-
- private Thread nodeIndexerThread = null;
-
- public synchronized long getLastIndexed(long timeoutMillis) {
- if (this.lastIndexed == -1L && this.nodeIndexerThread != null &&
- timeoutMillis > 0L) {
- try {
- this.wait(timeoutMillis);
- } catch (InterruptedException e) {
- }
- }
- return this.lastIndexed;
- }
-
- public synchronized NodeIndex getLatestNodeIndex(long timeoutMillis) {
- if (this.latestNodeIndex == null && this.nodeIndexerThread != null &&
- timeoutMillis > 0L) {
- try {
- this.wait(timeoutMillis);
- } catch (InterruptedException e) {
- }
- }
- return this.latestNodeIndex;
- }
-
- public synchronized void startIndexing() {
- if (this.nodeIndexerThread == null) {
- this.nodeIndexerThread = new Thread(this);
- this.nodeIndexerThread.setDaemon(true);
- this.nodeIndexerThread.start();
- }
- }
-
- public void run() {
- while (this.nodeIndexerThread != null) {
- this.indexNodeStatuses();
- try {
- Thread.sleep(DateTimeHelper.ONE_MINUTE);
- } catch (InterruptedException e) {
- }
- }
- }
-
- public synchronized void stopIndexing() {
- Thread indexerThread = this.nodeIndexerThread;
- this.nodeIndexerThread = null;
- indexerThread.interrupt();
- }
-
- private void indexNodeStatuses() {
- long updateStatusMillis = -1L;
- DocumentStore documentStore = ApplicationFactory.getDocumentStore();
- UpdateStatus updateStatus = documentStore.retrieve(UpdateStatus.class,
- false);
- if (updateStatus != null &&
- updateStatus.getDocumentString() != null) {
- String updateString = updateStatus.getDocumentString();
- try {
- updateStatusMillis = Long.parseLong(updateString.trim());
- } catch (NumberFormatException e) {
- /* Handle below. */
- }
- }
- synchronized (this) {
- if (updateStatusMillis <= this.lastIndexed) {
- return;
- }
- }
- List<String> newRelaysByConsensusWeight = new ArrayList<String>();
- Map<String, SummaryDocument>
- newRelayFingerprintSummaryLines =
- new HashMap<String, SummaryDocument>(),
- newBridgeFingerprintSummaryLines =
- new HashMap<String, SummaryDocument>();
- Map<String, Set<String>>
- newRelaysByCountryCode = new HashMap<String, Set<String>>(),
- newRelaysByASNumber = new HashMap<String, Set<String>>(),
- newRelaysByFlag = new HashMap<String, Set<String>>(),
- newBridgesByFlag = new HashMap<String, Set<String>>(),
- newRelaysByContact = new HashMap<String, Set<String>>(),
- newRelaysByFamily = new HashMap<String, Set<String>>();
- SortedMap<Integer, Set<String>>
- newRelaysByFirstSeenDays = new TreeMap<Integer, Set<String>>(),
- newBridgesByFirstSeenDays = new TreeMap<Integer, Set<String>>(),
- newRelaysByLastSeenDays = new TreeMap<Integer, Set<String>>(),
- newBridgesByLastSeenDays = new TreeMap<Integer, Set<String>>();
- Set<SummaryDocument> currentRelays = new HashSet<SummaryDocument>(),
- currentBridges = new HashSet<SummaryDocument>();
- SortedSet<String> fingerprints = documentStore.list(
- SummaryDocument.class);
- long relaysLastValidAfterMillis = 0L, bridgesLastPublishedMillis = 0L;
- for (String fingerprint : fingerprints) {
- SummaryDocument node = documentStore.retrieve(SummaryDocument.class,
- true, fingerprint);
- if (node.isRelay()) {
- relaysLastValidAfterMillis = Math.max(
- relaysLastValidAfterMillis, node.getLastSeenMillis());
- currentRelays.add(node);
- } else {
- bridgesLastPublishedMillis = Math.max(
- bridgesLastPublishedMillis, node.getLastSeenMillis());
- currentBridges.add(node);
- }
- }
- Time time = ApplicationFactory.getTime();
- List<String> orderRelaysByConsensusWeight = new ArrayList<String>();
- for (SummaryDocument entry : currentRelays) {
- String fingerprint = entry.getFingerprint().toUpperCase();
- String hashedFingerprint = entry.getHashedFingerprint().
- toUpperCase();
- newRelayFingerprintSummaryLines.put(fingerprint, entry);
- newRelayFingerprintSummaryLines.put(hashedFingerprint, entry);
- long consensusWeight = entry.getConsensusWeight();
- orderRelaysByConsensusWeight.add(String.format("%020d %s",
- consensusWeight, fingerprint));
- orderRelaysByConsensusWeight.add(String.format("%020d %s",
- consensusWeight, hashedFingerprint));
- if (entry.getCountryCode() != null) {
- String countryCode = entry.getCountryCode();
- if (!newRelaysByCountryCode.containsKey(countryCode)) {
- newRelaysByCountryCode.put(countryCode,
- new HashSet<String>());
- }
- newRelaysByCountryCode.get(countryCode).add(fingerprint);
- newRelaysByCountryCode.get(countryCode).add(hashedFingerprint);
- }
- if (entry.getASNumber() != null) {
- String aSNumber = entry.getASNumber();
- if (!newRelaysByASNumber.containsKey(aSNumber)) {
- newRelaysByASNumber.put(aSNumber, new HashSet<String>());
- }
- newRelaysByASNumber.get(aSNumber).add(fingerprint);
- newRelaysByASNumber.get(aSNumber).add(hashedFingerprint);
- }
- for (String flag : entry.getRelayFlags()) {
- String flagLowerCase = flag.toLowerCase();
- if (!newRelaysByFlag.containsKey(flagLowerCase)) {
- newRelaysByFlag.put(flagLowerCase, new HashSet<String>());
- }
- newRelaysByFlag.get(flagLowerCase).add(fingerprint);
- newRelaysByFlag.get(flagLowerCase).add(hashedFingerprint);
- }
- if (entry.getFamilyFingerprints() != null) {
- newRelaysByFamily.put(fingerprint, entry.getFamilyFingerprints());
- }
- int daysSinceFirstSeen = (int) ((time.currentTimeMillis()
- - entry.getFirstSeenMillis()) / DateTimeHelper.ONE_DAY);
- if (!newRelaysByFirstSeenDays.containsKey(daysSinceFirstSeen)) {
- newRelaysByFirstSeenDays.put(daysSinceFirstSeen,
- new HashSet<String>());
- }
- newRelaysByFirstSeenDays.get(daysSinceFirstSeen).add(fingerprint);
- newRelaysByFirstSeenDays.get(daysSinceFirstSeen).add(
- hashedFingerprint);
- int daysSinceLastSeen = (int) ((time.currentTimeMillis()
- - entry.getLastSeenMillis()) / DateTimeHelper.ONE_DAY);
- if (!newRelaysByLastSeenDays.containsKey(daysSinceLastSeen)) {
- newRelaysByLastSeenDays.put(daysSinceLastSeen,
- new HashSet<String>());
- }
- newRelaysByLastSeenDays.get(daysSinceLastSeen).add(fingerprint);
- newRelaysByLastSeenDays.get(daysSinceLastSeen).add(
- hashedFingerprint);
- String contact = entry.getContact();
- if (!newRelaysByContact.containsKey(contact)) {
- newRelaysByContact.put(contact, new HashSet<String>());
- }
- newRelaysByContact.get(contact).add(fingerprint);
- newRelaysByContact.get(contact).add(hashedFingerprint);
- }
- Collections.sort(orderRelaysByConsensusWeight);
- newRelaysByConsensusWeight = new ArrayList<String>();
- for (String relay : orderRelaysByConsensusWeight) {
- newRelaysByConsensusWeight.add(relay.split(" ")[1]);
- }
- for (Map.Entry<String, Set<String>> e :
- newRelaysByFamily.entrySet()) {
- String fingerprint = e.getKey();
- Set<String> inMutualFamilyRelation = new HashSet<String>();
- for (String otherFingerprint : e.getValue()) {
- if (newRelaysByFamily.containsKey(otherFingerprint) &&
- newRelaysByFamily.get(otherFingerprint).contains(
- fingerprint)) {
- inMutualFamilyRelation.add(otherFingerprint);
- }
- }
- e.getValue().retainAll(inMutualFamilyRelation);
- }
- for (SummaryDocument entry : currentBridges) {
- String hashedFingerprint = entry.getFingerprint().toUpperCase();
- String hashedHashedFingerprint = entry.getHashedFingerprint().
- toUpperCase();
- newBridgeFingerprintSummaryLines.put(hashedFingerprint, entry);
- newBridgeFingerprintSummaryLines.put(hashedHashedFingerprint,
- entry);
- for (String flag : entry.getRelayFlags()) {
- String flagLowerCase = flag.toLowerCase();
- if (!newBridgesByFlag.containsKey(flagLowerCase)) {
- newBridgesByFlag.put(flagLowerCase, new HashSet<String>());
- }
- newBridgesByFlag.get(flagLowerCase).add(hashedFingerprint);
- newBridgesByFlag.get(flagLowerCase).add(
- hashedHashedFingerprint);
- }
- int daysSinceFirstSeen = (int) ((time.currentTimeMillis()
- - entry.getFirstSeenMillis()) / DateTimeHelper.ONE_DAY);
- if (!newBridgesByFirstSeenDays.containsKey(daysSinceFirstSeen)) {
- newBridgesByFirstSeenDays.put(daysSinceFirstSeen,
- new HashSet<String>());
- }
- newBridgesByFirstSeenDays.get(daysSinceFirstSeen).add(
- hashedFingerprint);
- newBridgesByFirstSeenDays.get(daysSinceFirstSeen).add(
- hashedHashedFingerprint);
- int daysSinceLastSeen = (int) ((time.currentTimeMillis()
- - entry.getLastSeenMillis()) / DateTimeHelper.ONE_DAY);
- if (!newBridgesByLastSeenDays.containsKey(daysSinceLastSeen)) {
- newBridgesByLastSeenDays.put(daysSinceLastSeen,
- new HashSet<String>());
- }
- newBridgesByLastSeenDays.get(daysSinceLastSeen).add(
- hashedFingerprint);
- newBridgesByLastSeenDays.get(daysSinceLastSeen).add(
- hashedHashedFingerprint);
- }
- NodeIndex newNodeIndex = new NodeIndex();
- newNodeIndex.setRelaysByConsensusWeight(newRelaysByConsensusWeight);
- newNodeIndex.setRelayFingerprintSummaryLines(
- newRelayFingerprintSummaryLines);
- newNodeIndex.setBridgeFingerprintSummaryLines(
- newBridgeFingerprintSummaryLines);
- newNodeIndex.setRelaysByCountryCode(newRelaysByCountryCode);
- newNodeIndex.setRelaysByASNumber(newRelaysByASNumber);
- newNodeIndex.setRelaysByFlag(newRelaysByFlag);
- newNodeIndex.setBridgesByFlag(newBridgesByFlag);
- newNodeIndex.setRelaysByContact(newRelaysByContact);
- newNodeIndex.setRelaysByFamily(newRelaysByFamily);
- newNodeIndex.setRelaysByFirstSeenDays(newRelaysByFirstSeenDays);
- newNodeIndex.setRelaysByLastSeenDays(newRelaysByLastSeenDays);
- newNodeIndex.setBridgesByFirstSeenDays(newBridgesByFirstSeenDays);
- newNodeIndex.setBridgesByLastSeenDays(newBridgesByLastSeenDays);
- newNodeIndex.setRelaysPublishedString(DateTimeHelper.format(
- relaysLastValidAfterMillis));
- newNodeIndex.setBridgesPublishedString(DateTimeHelper.format(
- bridgesLastPublishedMillis));
- synchronized (this) {
- this.lastIndexed = updateStatusMillis;
- this.latestNodeIndex = newNodeIndex;
- this.notifyAll();
- }
- }
-}
-
diff --git a/src/org/torproject/onionoo/NodeStatus.java b/src/org/torproject/onionoo/NodeStatus.java
deleted file mode 100644
index 1bc74f9..0000000
--- a/src/org/torproject/onionoo/NodeStatus.java
+++ /dev/null
@@ -1,581 +0,0 @@
-/* Copyright 2011, 2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.apache.commons.codec.DecoderException;
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.codec.digest.DigestUtils;
-
-/* Store search data of a single relay that was running in the past seven
- * days. */
-public class NodeStatus extends Document {
-
- private boolean isRelay;
- public boolean isRelay() {
- return this.isRelay;
- }
-
- private String fingerprint;
- public String getFingerprint() {
- return this.fingerprint;
- }
-
- private String hashedFingerprint;
- public String getHashedFingerprint() {
- return this.hashedFingerprint;
- }
-
- private String nickname;
- public String getNickname() {
- return this.nickname;
- }
-
- private String address;
- public String getAddress() {
- return this.address;
- }
-
- private SortedSet<String> orAddresses;
- private SortedSet<String> orAddressesAndPorts;
- public SortedSet<String> getOrAddresses() {
- return new TreeSet<String>(this.orAddresses);
- }
- public void addOrAddressAndPort(String orAddressAndPort) {
- if (orAddressAndPort.contains(":") && orAddressAndPort.length() > 0) {
- String orAddress = orAddressAndPort.substring(0,
- orAddressAndPort.lastIndexOf(':'));
- if (this.exitAddresses.contains(orAddress)) {
- this.exitAddresses.remove(orAddress);
- }
- this.orAddresses.add(orAddress);
- this.orAddressesAndPorts.add(orAddressAndPort);
- }
- }
- public SortedSet<String> getOrAddressesAndPorts() {
- return new TreeSet<String>(this.orAddressesAndPorts);
- }
-
- private SortedSet<String> exitAddresses;
- public void addExitAddress(String exitAddress) {
- if (exitAddress.length() > 0 && !this.address.equals(exitAddress) &&
- !this.orAddresses.contains(exitAddress)) {
- this.exitAddresses.add(exitAddress);
- }
- }
- public SortedSet<String> getExitAddresses() {
- return new TreeSet<String>(this.exitAddresses);
- }
-
- private Float latitude;
- public void setLatitude(Float latitude) {
- this.latitude = latitude;
- }
- public Float getLatitude() {
- return this.latitude;
- }
-
- private Float longitude;
- public void setLongitude(Float longitude) {
- this.longitude = longitude;
- }
- public Float getLongitude() {
- return this.longitude;
- }
-
- private String countryCode;
- public void setCountryCode(String countryCode) {
- this.countryCode = countryCode;
- }
- public String getCountryCode() {
- return this.countryCode;
- }
-
- private String countryName;
- public void setCountryName(String countryName) {
- this.countryName = countryName;
- }
- public String getCountryName() {
- return this.countryName;
- }
-
- private String regionName;
- public void setRegionName(String regionName) {
- this.regionName = regionName;
- }
- public String getRegionName() {
- return this.regionName;
- }
-
- private String cityName;
- public void setCityName(String cityName) {
- this.cityName = cityName;
- }
- public String getCityName() {
- return this.cityName;
- }
-
- private String aSName;
- public void setASName(String aSName) {
- this.aSName = aSName;
- }
- public String getASName() {
- return this.aSName;
- }
-
- private String aSNumber;
- public void setASNumber(String aSNumber) {
- this.aSNumber = aSNumber;
- }
- public String getASNumber() {
- return this.aSNumber;
- }
-
- private long firstSeenMillis;
- public long getFirstSeenMillis() {
- return this.firstSeenMillis;
- }
-
- private long lastSeenMillis;
- public long getLastSeenMillis() {
- return this.lastSeenMillis;
- }
-
- private int orPort;
- public int getOrPort() {
- return this.orPort;
- }
-
- private int dirPort;
- public int getDirPort() {
- return this.dirPort;
- }
-
- private SortedSet<String> relayFlags;
- public SortedSet<String> getRelayFlags() {
- return this.relayFlags;
- }
-
- private long consensusWeight;
- public long getConsensusWeight() {
- return this.consensusWeight;
- }
-
- private boolean running;
- public void setRunning(boolean running) {
- this.running = running;
- }
- public boolean getRunning() {
- return this.running;
- }
-
- private String hostName;
- public void setHostName(String hostName) {
- this.hostName = hostName;
- }
- public String getHostName() {
- return this.hostName;
- }
-
- private long lastRdnsLookup = -1L;
- public void setLastRdnsLookup(long lastRdnsLookup) {
- this.lastRdnsLookup = lastRdnsLookup;
- }
- public long getLastRdnsLookup() {
- return this.lastRdnsLookup;
- }
-
- private double advertisedBandwidthFraction = -1.0;
- public void setAdvertisedBandwidthFraction(
- double advertisedBandwidthFraction) {
- this.advertisedBandwidthFraction = advertisedBandwidthFraction;
- }
- public double getAdvertisedBandwidthFraction() {
- return this.advertisedBandwidthFraction;
- }
-
- private double consensusWeightFraction = -1.0;
- public void setConsensusWeightFraction(double consensusWeightFraction) {
- this.consensusWeightFraction = consensusWeightFraction;
- }
- public double getConsensusWeightFraction() {
- return this.consensusWeightFraction;
- }
-
- private double guardProbability = -1.0;
- public void setGuardProbability(double guardProbability) {
- this.guardProbability = guardProbability;
- }
- public double getGuardProbability() {
- return this.guardProbability;
- }
-
- private double middleProbability = -1.0;
- public void setMiddleProbability(double middleProbability) {
- this.middleProbability = middleProbability;
- }
- public double getMiddleProbability() {
- return this.middleProbability;
- }
-
- private double exitProbability = -1.0;
- public void setExitProbability(double exitProbability) {
- this.exitProbability = exitProbability;
- }
- public double getExitProbability() {
- return this.exitProbability;
- }
-
- private String defaultPolicy;
- public String getDefaultPolicy() {
- return this.defaultPolicy;
- }
-
- private String portList;
- public String getPortList() {
- return this.portList;
- }
-
- private SortedMap<Long, Set<String>> lastAddresses;
- public SortedMap<Long, Set<String>> getLastAddresses() {
- return this.lastAddresses == null ? null :
- new TreeMap<Long, Set<String>>(this.lastAddresses);
- }
- public long getLastChangedOrAddress() {
- long lastChangedAddressesMillis = -1L;
- if (this.lastAddresses != null) {
- Set<String> lastAddresses = null;
- for (Map.Entry<Long, Set<String>> e : this.lastAddresses.entrySet()) {
- if (lastAddresses != null) {
- for (String address : e.getValue()) {
- if (!lastAddresses.contains(address)) {
- return lastChangedAddressesMillis;
- }
- }
- }
- lastChangedAddressesMillis = e.getKey();
- lastAddresses = e.getValue();
- }
- }
- return lastChangedAddressesMillis;
- }
-
- private String contact;
- public void setContact(String contact) {
- if (contact == null) {
- this.contact = null;
- } else {
- StringBuilder sb = new StringBuilder();
- for (char c : contact.toLowerCase().toCharArray()) {
- if (c >= 32 && c < 127) {
- sb.append(c);
- } else {
- sb.append(" ");
- }
- }
- this.contact = sb.toString();
- }
- }
- public String getContact() {
- return this.contact;
- }
-
- private Boolean recommendedVersion;
- public Boolean getRecommendedVersion() {
- return this.recommendedVersion;
- }
-
- private SortedSet<String> familyFingerprints;
- public void setFamilyFingerprints(
- SortedSet<String> familyFingerprints) {
- this.familyFingerprints = familyFingerprints;
- }
- public SortedSet<String> getFamilyFingerprints() {
- return this.familyFingerprints;
- }
-
- public NodeStatus(boolean isRelay, String nickname, String fingerprint,
- String address, SortedSet<String> orAddressesAndPorts,
- SortedSet<String> exitAddresses, long lastSeenMillis, int orPort,
- int dirPort, SortedSet<String> relayFlags, long consensusWeight,
- String countryCode, String hostName, long lastRdnsLookup,
- String defaultPolicy, String portList, long firstSeenMillis,
- long lastChangedAddresses, String aSNumber, String contact,
- Boolean recommendedVersion, SortedSet<String> familyFingerprints) {
- this.isRelay = isRelay;
- this.nickname = nickname;
- this.fingerprint = fingerprint;
- try {
- this.hashedFingerprint = DigestUtils.shaHex(Hex.decodeHex(
- this.fingerprint.toCharArray())).toUpperCase();
- } catch (DecoderException e) {
- throw new IllegalArgumentException("Fingerprint '" + fingerprint
- + "' is not a valid fingerprint.", e);
- }
- this.address = address;
- this.exitAddresses = new TreeSet<String>();
- if (exitAddresses != null) {
- this.exitAddresses.addAll(exitAddresses);
- }
- this.exitAddresses.remove(this.address);
- this.orAddresses = new TreeSet<String>();
- this.orAddressesAndPorts = new TreeSet<String>();
- if (orAddressesAndPorts != null) {
- for (String orAddressAndPort : orAddressesAndPorts) {
- this.addOrAddressAndPort(orAddressAndPort);
- }
- }
- this.lastSeenMillis = lastSeenMillis;
- this.orPort = orPort;
- this.dirPort = dirPort;
- this.relayFlags = relayFlags;
- this.consensusWeight = consensusWeight;
- this.countryCode = countryCode;
- this.hostName = hostName;
- this.lastRdnsLookup = lastRdnsLookup;
- this.defaultPolicy = defaultPolicy;
- this.portList = portList;
- this.firstSeenMillis = firstSeenMillis;
- this.lastAddresses =
- new TreeMap<Long, Set<String>>(Collections.reverseOrder());
- Set<String> addresses = new HashSet<String>();
- addresses.add(address + ":" + orPort);
- if (dirPort > 0) {
- addresses.add(address + ":" + dirPort);
- }
- addresses.addAll(orAddressesAndPorts);
- this.lastAddresses.put(lastChangedAddresses, addresses);
- this.aSNumber = aSNumber;
- this.contact = contact;
- this.recommendedVersion = recommendedVersion;
- this.familyFingerprints = familyFingerprints;
- }
-
- public static NodeStatus fromString(String documentString) {
- boolean isRelay = false;
- String nickname = null, fingerprint = null, address = null,
- countryCode = null, hostName = null, defaultPolicy = null,
- portList = null, aSNumber = null, contact = null;
- SortedSet<String> orAddressesAndPorts = null, exitAddresses = null,
- relayFlags = null, familyFingerprints = null;
- long lastSeenMillis = -1L, consensusWeight = -1L,
- lastRdnsLookup = -1L, firstSeenMillis = -1L,
- lastChangedAddresses = -1L;
- int orPort = -1, dirPort = -1;
- Boolean recommendedVersion = null;
- try {
- String separator = documentString.contains("\t") ? "\t" : " ";
- String[] parts = documentString.trim().split(separator);
- isRelay = parts[0].equals("r");
- if (parts.length < 9) {
- System.err.println("Too few space-separated values in line '"
- + documentString.trim() + "'. Skipping.");
- return null;
- }
- nickname = parts[1];
- fingerprint = parts[2];
- orAddressesAndPorts = new TreeSet<String>();
- exitAddresses = new TreeSet<String>();
- String addresses = parts[3];
- if (addresses.contains(";")) {
- String[] addressParts = addresses.split(";", -1);
- if (addressParts.length != 3) {
- System.err.println("Invalid addresses entry in line '"
- + documentString.trim() + "'. Skipping.");
- return null;
- }
- address = addressParts[0];
- if (addressParts[1].length() > 0) {
- orAddressesAndPorts.addAll(Arrays.asList(
- addressParts[1].split("\\+")));
- }
- if (addressParts[2].length() > 0) {
- exitAddresses.addAll(Arrays.asList(
- addressParts[2].split("\\+")));
- }
- } else {
- address = addresses;
- }
- lastSeenMillis = DateTimeHelper.parse(parts[4] + " " + parts[5]);
- if (lastSeenMillis < 0L) {
- System.err.println("Parse exception while parsing node status "
- + "line '" + documentString + "'. Skipping.");
- return null;
- }
- orPort = Integer.parseInt(parts[6]);
- dirPort = Integer.parseInt(parts[7]);
- relayFlags = new TreeSet<String>();
- if (parts[8].length() > 0) {
- relayFlags.addAll(Arrays.asList(parts[8].split(",")));
- }
- if (parts.length > 9) {
- consensusWeight = Long.parseLong(parts[9]);
- }
- if (parts.length > 10) {
- countryCode = parts[10];
- }
- if (parts.length > 12) {
- hostName = parts[11].equals("null") ? null : parts[11];
- lastRdnsLookup = Long.parseLong(parts[12]);
- }
- if (parts.length > 14) {
- if (!parts[13].equals("null")) {
- defaultPolicy = parts[13];
- }
- if (!parts[14].equals("null")) {
- portList = parts[14];
- }
- }
- firstSeenMillis = lastSeenMillis;
- if (parts.length > 16) {
- firstSeenMillis = DateTimeHelper.parse(parts[15] + " "
- + parts[16]);
- if (firstSeenMillis < 0L) {
- System.err.println("Parse exception while parsing node status "
- + "line '" + documentString + "'. Skipping.");
- return null;
- }
- }
- lastChangedAddresses = lastSeenMillis;
- if (parts.length > 18 && !parts[17].equals("null")) {
- lastChangedAddresses = DateTimeHelper.parse(parts[17] + " "
- + parts[18]);
- if (lastChangedAddresses < 0L) {
- System.err.println("Parse exception while parsing node status "
- + "line '" + documentString + "'. Skipping.");
- return null;
- }
- }
- if (parts.length > 19) {
- aSNumber = parts[19];
- }
- if (parts.length > 20) {
- contact = parts[20];
- }
- if (parts.length > 21) {
- recommendedVersion = parts[21].equals("null") ? null :
- parts[21].equals("true");
- }
- if (parts.length > 22 && !parts[22].equals("null")) {
- familyFingerprints = new TreeSet<String>(Arrays.asList(
- parts[22].split(";")));
- }
- } catch (NumberFormatException e) {
- System.err.println("Number format exception while parsing node "
- + "status line '" + documentString + "': " + e.getMessage()
- + ". Skipping.");
- return null;
- } catch (Exception e) {
- /* This catch block is only here to handle yet unknown errors. It
- * should go away once we're sure what kind of errors can occur. */
- System.err.println("Unknown exception while parsing node status "
- + "line '" + documentString + "': " + e.getMessage() + ". "
- + "Skipping.");
- return null;
- }
- NodeStatus newNodeStatus = new NodeStatus(isRelay, nickname,
- fingerprint, address, orAddressesAndPorts, exitAddresses,
- lastSeenMillis, orPort, dirPort, relayFlags, consensusWeight,
- countryCode, hostName, lastRdnsLookup, defaultPolicy, portList,
- firstSeenMillis, lastChangedAddresses, aSNumber, contact,
- recommendedVersion, familyFingerprints);
- return newNodeStatus;
- }
-
- public void update(NodeStatus newNodeStatus) {
- if (newNodeStatus.lastSeenMillis > this.lastSeenMillis) {
- this.nickname = newNodeStatus.nickname;
- this.address = newNodeStatus.address;
- this.orAddressesAndPorts = newNodeStatus.orAddressesAndPorts;
- this.lastSeenMillis = newNodeStatus.lastSeenMillis;
- this.orPort = newNodeStatus.orPort;
- this.dirPort = newNodeStatus.dirPort;
- this.relayFlags = newNodeStatus.relayFlags;
- this.consensusWeight = newNodeStatus.consensusWeight;
- this.countryCode = newNodeStatus.countryCode;
- this.defaultPolicy = newNodeStatus.defaultPolicy;
- this.portList = newNodeStatus.portList;
- this.aSNumber = newNodeStatus.aSNumber;
- this.recommendedVersion = newNodeStatus.recommendedVersion;
- }
- if (this.isRelay && newNodeStatus.isRelay) {
- this.lastAddresses.putAll(newNodeStatus.lastAddresses);
- }
- this.firstSeenMillis = Math.min(newNodeStatus.firstSeenMillis,
- this.firstSeenMillis);
- }
-
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append(this.isRelay ? "r" : "b");
- sb.append("\t" + this.nickname);
- sb.append("\t" + this.fingerprint);
- sb.append("\t" + this.address + ";");
- int written = 0;
- for (String orAddressAndPort : this.orAddressesAndPorts) {
- sb.append((written++ > 0 ? "+" : "") + orAddressAndPort);
- }
- sb.append(";");
- if (this.isRelay) {
- written = 0;
- for (String exitAddress : this.exitAddresses) {
- sb.append((written++ > 0 ? "+" : "")
- + exitAddress);
- }
- }
- sb.append("\t" + DateTimeHelper.format(this.lastSeenMillis,
- DateTimeHelper.ISO_DATETIME_TAB_FORMAT));
- sb.append("\t" + this.orPort);
- sb.append("\t" + this.dirPort + "\t");
- written = 0;
- for (String relayFlag : this.relayFlags) {
- sb.append((written++ > 0 ? "," : "") + relayFlag);
- }
- if (this.isRelay) {
- sb.append("\t" + String.valueOf(this.consensusWeight));
- sb.append("\t"
- + (this.countryCode != null ? this.countryCode : "??"));
- sb.append("\t" + (this.hostName != null ? this.hostName : "null"));
- sb.append("\t" + String.valueOf(this.lastRdnsLookup));
- sb.append("\t" + (this.defaultPolicy != null ? this.defaultPolicy
- : "null"));
- sb.append("\t" + (this.portList != null ? this.portList : "null"));
- } else {
- sb.append("\t-1\t??\tnull\t-1\tnull\tnull");
- }
- sb.append("\t" + DateTimeHelper.format(this.firstSeenMillis,
- DateTimeHelper.ISO_DATETIME_TAB_FORMAT));
- if (this.isRelay) {
- sb.append("\t" + DateTimeHelper.format(
- this.getLastChangedOrAddress(),
- DateTimeHelper.ISO_DATETIME_TAB_FORMAT));
- sb.append("\t" + (this.aSNumber != null ? this.aSNumber : "null"));
- } else {
- sb.append("\tnull\tnull\tnull");
- }
- sb.append("\t" + (this.contact != null ? this.contact : ""));
- sb.append("\t" + (this.recommendedVersion == null ? "null" :
- this.recommendedVersion ? "true" : "false"));
- if (this.familyFingerprints == null ||
- this.familyFingerprints.isEmpty()) {
- sb.append("\tnull");
- } else {
- sb.append("\t");
- written = 0;
- for (String familyFingerprint : this.familyFingerprints) {
- sb.append((written++ > 0 ? ";" : "") + familyFingerprint);
- }
- }
- return sb.toString();
- }
-}
-
diff --git a/src/org/torproject/onionoo/RequestHandler.java b/src/org/torproject/onionoo/RequestHandler.java
deleted file mode 100644
index 58ec17a..0000000
--- a/src/org/torproject/onionoo/RequestHandler.java
+++ /dev/null
@@ -1,548 +0,0 @@
-/* Copyright 2011--2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-
-public class RequestHandler {
-
- private NodeIndex nodeIndex;
-
- private DocumentStore documentStore;
-
- public RequestHandler(NodeIndex nodeIndex) {
- this.nodeIndex = nodeIndex;
- this.documentStore = ApplicationFactory.getDocumentStore();
- }
-
- private String resourceType;
- public void setResourceType(String resourceType) {
- this.resourceType = resourceType;
- }
-
- private String type;
- public void setType(String type) {
- this.type = type;
- }
-
- private String running;
- public void setRunning(String running) {
- this.running = running;
- }
-
- private String[] search;
- public void setSearch(String[] search) {
- this.search = new String[search.length];
- System.arraycopy(search, 0, this.search, 0, search.length);
- }
-
- private String lookup;
- public void setLookup(String lookup) {
- this.lookup = lookup;
- }
-
- private String fingerprint;
- public void setFingerprint(String fingerprint) {
- this.fingerprint = fingerprint;
- }
-
- private String country;
- public void setCountry(String country) {
- this.country = country;
- }
-
- private String as;
- public void setAs(String as) {
- this.as = as;
- }
-
- private String flag;
- public void setFlag(String flag) {
- this.flag = flag;
- }
-
- private String[] contact;
- public void setContact(String[] contact) {
- this.contact = new String[contact.length];
- System.arraycopy(contact, 0, this.contact, 0, contact.length);
- }
-
- private String[] order;
- public void setOrder(String[] order) {
- this.order = new String[order.length];
- System.arraycopy(order, 0, this.order, 0, order.length);
- }
-
- private String offset;
- public void setOffset(String offset) {
- this.offset = offset;
- }
-
- private String limit;
- public void setLimit(String limit) {
- this.limit = limit;
- }
-
- private int[] firstSeenDays;
- public void setFirstSeenDays(int[] firstSeenDays) {
- this.firstSeenDays = new int[firstSeenDays.length];
- System.arraycopy(firstSeenDays, 0, this.firstSeenDays, 0,
- firstSeenDays.length);
- }
-
- private int[] lastSeenDays;
- public void setLastSeenDays(int[] lastSeenDays) {
- this.lastSeenDays = new int[lastSeenDays.length];
- System.arraycopy(lastSeenDays, 0, this.lastSeenDays, 0,
- lastSeenDays.length);
- }
-
- private String family;
- public void setFamily(String family) {
- this.family = family;
- }
-
- private Map<String, SummaryDocument> filteredRelays =
- new HashMap<String, SummaryDocument>();
-
- private Map<String, SummaryDocument> filteredBridges =
- new HashMap<String, SummaryDocument>();
-
- public void handleRequest() {
- this.filteredRelays.putAll(
- this.nodeIndex.getRelayFingerprintSummaryLines());
- this.filteredBridges.putAll(
- this.nodeIndex.getBridgeFingerprintSummaryLines());
- this.filterByResourceType();
- this.filterByType();
- this.filterByRunning();
- this.filterBySearchTerms();
- this.filterByLookup();
- this.filterByFingerprint();
- this.filterByCountryCode();
- this.filterByASNumber();
- this.filterByFlag();
- this.filterNodesByFirstSeenDays();
- this.filterNodesByLastSeenDays();
- this.filterByContact();
- this.filterByFamily();
- this.order();
- this.offset();
- this.limit();
- }
-
-
- private void filterByResourceType() {
- if (this.resourceType.equals("clients")) {
- this.filteredRelays.clear();
- }
- if (this.resourceType.equals("weights")) {
- this.filteredBridges.clear();
- }
- }
-
- private void filterByType() {
- if (this.type == null) {
- return;
- } else if (this.type.equals("relay")) {
- this.filteredBridges.clear();
- } else {
- this.filteredRelays.clear();
- }
- }
-
- private void filterByRunning() {
- if (this.running == null) {
- return;
- }
- boolean runningRequested = this.running.equals("true");
- Set<String> removeRelays = new HashSet<String>();
- for (Map.Entry<String, SummaryDocument> e :
- filteredRelays.entrySet()) {
- if (e.getValue().isRunning() != runningRequested) {
- removeRelays.add(e.getKey());
- }
- }
- for (String fingerprint : removeRelays) {
- this.filteredRelays.remove(fingerprint);
- }
- Set<String> removeBridges = new HashSet<String>();
- for (Map.Entry<String, SummaryDocument> e :
- filteredBridges.entrySet()) {
- if (e.getValue().isRunning() != runningRequested) {
- removeBridges.add(e.getKey());
- }
- }
- for (String fingerprint : removeBridges) {
- this.filteredBridges.remove(fingerprint);
- }
- }
-
- private void filterBySearchTerms() {
- if (this.search == null) {
- return;
- }
- for (String searchTerm : this.search) {
- filterBySearchTerm(searchTerm);
- }
- }
-
- private void filterBySearchTerm(String searchTerm) {
- Set<String> removeRelays = new HashSet<String>();
- for (Map.Entry<String, SummaryDocument> e :
- filteredRelays.entrySet()) {
- String fingerprint = e.getKey();
- SummaryDocument entry = e.getValue();
- boolean lineMatches = false;
- String nickname = entry.getNickname() != null ?
- entry.getNickname().toLowerCase() : "unnamed";
- if (searchTerm.startsWith("$")) {
- /* Search is for $-prefixed fingerprint. */
- if (fingerprint.startsWith(
- searchTerm.substring(1).toUpperCase())) {
- /* $-prefixed fingerprint matches. */
- lineMatches = true;
- }
- } else if (nickname.contains(searchTerm.toLowerCase())) {
- /* Nickname matches. */
- lineMatches = true;
- } else if (fingerprint.startsWith(searchTerm.toUpperCase())) {
- /* Non-$-prefixed fingerprint matches. */
- lineMatches = true;
- } else {
- List<String> addresses = entry.getAddresses();
- for (String address : addresses) {
- if (address.startsWith(searchTerm.toLowerCase())) {
- /* Address matches. */
- lineMatches = true;
- break;
- }
- }
- }
- if (!lineMatches) {
- removeRelays.add(e.getKey());
- }
- }
- for (String fingerprint : removeRelays) {
- this.filteredRelays.remove(fingerprint);
- }
- Set<String> removeBridges = new HashSet<String>();
- for (Map.Entry<String, SummaryDocument> e :
- filteredBridges.entrySet()) {
- String hashedFingerprint = e.getKey();
- SummaryDocument entry = e.getValue();
- boolean lineMatches = false;
- String nickname = entry.getNickname() != null ?
- entry.getNickname().toLowerCase() : "unnamed";
- if (searchTerm.startsWith("$")) {
- /* Search is for $-prefixed hashed fingerprint. */
- if (hashedFingerprint.startsWith(
- searchTerm.substring(1).toUpperCase())) {
- /* $-prefixed hashed fingerprint matches. */
- lineMatches = true;
- }
- } else if (nickname.contains(searchTerm.toLowerCase())) {
- /* Nickname matches. */
- lineMatches = true;
- } else if (hashedFingerprint.startsWith(searchTerm.toUpperCase())) {
- /* Non-$-prefixed hashed fingerprint matches. */
- lineMatches = true;
- }
- if (!lineMatches) {
- removeBridges.add(e.getKey());
- }
- }
- for (String fingerprint : removeBridges) {
- this.filteredBridges.remove(fingerprint);
- }
- }
-
- private void filterByLookup() {
- if (this.lookup == null) {
- return;
- }
- String fingerprint = this.lookup;
- SummaryDocument relayLine = this.filteredRelays.get(fingerprint);
- this.filteredRelays.clear();
- if (relayLine != null) {
- this.filteredRelays.put(fingerprint, relayLine);
- }
- SummaryDocument bridgeLine = this.filteredBridges.get(fingerprint);
- this.filteredBridges.clear();
- if (bridgeLine != null) {
- this.filteredBridges.put(fingerprint, bridgeLine);
- }
- }
-
- private void filterByFingerprint() {
- if (this.fingerprint == null) {
- return;
- }
- this.filteredRelays.clear();
- this.filteredBridges.clear();
- String fingerprint = this.fingerprint;
- SummaryDocument entry = this.documentStore.retrieve(
- SummaryDocument.class, true, fingerprint);
- if (entry != null) {
- if (entry.isRelay()) {
- this.filteredRelays.put(fingerprint, entry);
- } else {
- this.filteredBridges.put(fingerprint, entry);
- }
- }
- }
-
- private void filterByCountryCode() {
- if (this.country == null) {
- return;
- }
- String countryCode = this.country.toLowerCase();
- if (!this.nodeIndex.getRelaysByCountryCode().containsKey(
- countryCode)) {
- this.filteredRelays.clear();
- } else {
- Set<String> relaysWithCountryCode =
- this.nodeIndex.getRelaysByCountryCode().get(countryCode);
- Set<String> removeRelays = new HashSet<String>();
- for (String fingerprint : this.filteredRelays.keySet()) {
- if (!relaysWithCountryCode.contains(fingerprint)) {
- removeRelays.add(fingerprint);
- }
- }
- for (String fingerprint : removeRelays) {
- this.filteredRelays.remove(fingerprint);
- }
- }
- this.filteredBridges.clear();
- }
-
- private void filterByASNumber() {
- if (this.as == null) {
- return;
- }
- String aSNumber = this.as.toUpperCase();
- if (!aSNumber.startsWith("AS")) {
- aSNumber = "AS" + aSNumber;
- }
- if (!this.nodeIndex.getRelaysByASNumber().containsKey(aSNumber)) {
- this.filteredRelays.clear();
- } else {
- Set<String> relaysWithASNumber =
- this.nodeIndex.getRelaysByASNumber().get(aSNumber);
- Set<String> removeRelays = new HashSet<String>();
- for (String fingerprint : this.filteredRelays.keySet()) {
- if (!relaysWithASNumber.contains(fingerprint)) {
- removeRelays.add(fingerprint);
- }
- }
- for (String fingerprint : removeRelays) {
- this.filteredRelays.remove(fingerprint);
- }
- }
- this.filteredBridges.clear();
- }
-
- private void filterByFlag() {
- if (this.flag == null) {
- return;
- }
- String flag = this.flag.toLowerCase();
- if (!this.nodeIndex.getRelaysByFlag().containsKey(flag)) {
- this.filteredRelays.clear();
- } else {
- Set<String> relaysWithFlag = this.nodeIndex.getRelaysByFlag().get(
- flag);
- Set<String> removeRelays = new HashSet<String>();
- for (String fingerprint : this.filteredRelays.keySet()) {
- if (!relaysWithFlag.contains(fingerprint)) {
- removeRelays.add(fingerprint);
- }
- }
- for (String fingerprint : removeRelays) {
- this.filteredRelays.remove(fingerprint);
- }
- }
- if (!this.nodeIndex.getBridgesByFlag().containsKey(flag)) {
- this.filteredBridges.clear();
- } else {
- Set<String> bridgesWithFlag = this.nodeIndex.getBridgesByFlag().get(
- flag);
- Set<String> removeBridges = new HashSet<String>();
- for (String fingerprint : this.filteredBridges.keySet()) {
- if (!bridgesWithFlag.contains(fingerprint)) {
- removeBridges.add(fingerprint);
- }
- }
- for (String fingerprint : removeBridges) {
- this.filteredBridges.remove(fingerprint);
- }
- }
- }
-
- private void filterNodesByFirstSeenDays() {
- if (this.firstSeenDays == null) {
- return;
- }
- filterNodesByDays(this.filteredRelays,
- this.nodeIndex.getRelaysByFirstSeenDays(), this.firstSeenDays);
- filterNodesByDays(this.filteredBridges,
- this.nodeIndex.getBridgesByFirstSeenDays(), this.firstSeenDays);
- }
-
- private void filterNodesByLastSeenDays() {
- if (this.lastSeenDays == null) {
- return;
- }
- filterNodesByDays(this.filteredRelays,
- this.nodeIndex.getRelaysByLastSeenDays(), this.lastSeenDays);
- filterNodesByDays(this.filteredBridges,
- this.nodeIndex.getBridgesByLastSeenDays(), this.lastSeenDays);
- }
-
- private void filterNodesByDays(
- Map<String, SummaryDocument> filteredNodes,
- SortedMap<Integer, Set<String>> nodesByDays, int[] days) {
- Set<String> removeNodes = new HashSet<String>();
- for (Set<String> nodes : nodesByDays.headMap(days[0]).values()) {
- removeNodes.addAll(nodes);
- }
- if (days[1] < Integer.MAX_VALUE) {
- for (Set<String> nodes :
- nodesByDays.tailMap(days[1] + 1).values()) {
- removeNodes.addAll(nodes);
- }
- }
- for (String fingerprint : removeNodes) {
- filteredNodes.remove(fingerprint);
- }
- }
-
- private void filterByContact() {
- if (this.contact == null) {
- return;
- }
- Set<String> removeRelays = new HashSet<String>();
- for (Map.Entry<String, Set<String>> e :
- this.nodeIndex.getRelaysByContact().entrySet()) {
- String contact = e.getKey();
- for (String contactPart : this.contact) {
- if (contact == null ||
- !contact.contains(contactPart.toLowerCase())) {
- removeRelays.addAll(e.getValue());
- break;
- }
- }
- }
- for (String fingerprint : removeRelays) {
- this.filteredRelays.remove(fingerprint);
- }
- this.filteredBridges.clear();
- }
-
- private void filterByFamily() {
- if (this.family == null) {
- return;
- }
- Set<String> removeRelays = new HashSet<String>(
- this.filteredRelays.keySet());
- removeRelays.remove(this.family);
- if (this.nodeIndex.getRelaysByFamily().containsKey(this.family)) {
- removeRelays.removeAll(this.nodeIndex.getRelaysByFamily().
- get(this.family));
- }
- for (String fingerprint : removeRelays) {
- this.filteredRelays.remove(fingerprint);
- }
- this.filteredBridges.clear();
- }
-
- private void order() {
- if (this.order != null && this.order.length == 1) {
- List<String> orderBy = new ArrayList<String>(
- this.nodeIndex.getRelaysByConsensusWeight());
- if (this.order[0].startsWith("-")) {
- Collections.reverse(orderBy);
- }
- for (String relay : orderBy) {
- if (this.filteredRelays.containsKey(relay) &&
- !this.orderedRelays.contains(filteredRelays.get(relay))) {
- this.orderedRelays.add(this.filteredRelays.remove(relay));
- }
- }
- for (String relay : this.filteredRelays.keySet()) {
- if (!this.orderedRelays.contains(this.filteredRelays.get(relay))) {
- this.orderedRelays.add(this.filteredRelays.remove(relay));
- }
- }
- Set<SummaryDocument> uniqueBridges = new HashSet<SummaryDocument>(
- this.filteredBridges.values());
- this.orderedBridges.addAll(uniqueBridges);
- } else {
- Set<SummaryDocument> uniqueRelays = new HashSet<SummaryDocument>(
- this.filteredRelays.values());
- this.orderedRelays.addAll(uniqueRelays);
- Set<SummaryDocument> uniqueBridges = new HashSet<SummaryDocument>(
- this.filteredBridges.values());
- this.orderedBridges.addAll(uniqueBridges);
- }
- }
-
- private void offset() {
- if (this.offset == null) {
- return;
- }
- int offsetValue = Integer.parseInt(this.offset);
- while (offsetValue-- > 0 &&
- (!this.orderedRelays.isEmpty() ||
- !this.orderedBridges.isEmpty())) {
- if (!this.orderedRelays.isEmpty()) {
- this.orderedRelays.remove(0);
- } else {
- this.orderedBridges.remove(0);
- }
- }
- }
-
- private void limit() {
- if (this.limit == null) {
- return;
- }
- int limitValue = Integer.parseInt(this.limit);
- while (!this.orderedRelays.isEmpty() &&
- limitValue < this.orderedRelays.size()) {
- this.orderedRelays.remove(this.orderedRelays.size() - 1);
- }
- limitValue -= this.orderedRelays.size();
- while (!this.orderedBridges.isEmpty() &&
- limitValue < this.orderedBridges.size()) {
- this.orderedBridges.remove(this.orderedBridges.size() - 1);
- }
- }
-
- private List<SummaryDocument> orderedRelays =
- new ArrayList<SummaryDocument>();
- public List<SummaryDocument> getOrderedRelays() {
- return this.orderedRelays;
- }
-
- private List<SummaryDocument> orderedBridges =
- new ArrayList<SummaryDocument>();
- public List<SummaryDocument> getOrderedBridges() {
- return this.orderedBridges;
- }
-
- public String getRelaysPublishedString() {
- return this.nodeIndex.getRelaysPublishedString();
- }
-
- public String getBridgesPublishedString() {
- return this.nodeIndex.getBridgesPublishedString();
- }
-}
diff --git a/src/org/torproject/onionoo/ResourceServlet.java b/src/org/torproject/onionoo/ResourceServlet.java
deleted file mode 100644
index efce86e..0000000
--- a/src/org/torproject/onionoo/ResourceServlet.java
+++ /dev/null
@@ -1,448 +0,0 @@
-/* Copyright 2011, 2012 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Pattern;
-
-import javax.servlet.ServletConfig;
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-public class ResourceServlet extends HttpServlet {
-
- private static final long serialVersionUID = 7236658979947465319L;
-
- private boolean maintenanceMode = false;
-
- /* Called by servlet container, not by test class. */
- public void init(ServletConfig config) throws ServletException {
- super.init(config);
- this.maintenanceMode =
- config.getInitParameter("maintenance") != null &&
- config.getInitParameter("maintenance").equals("1");
- }
-
- public long getLastModified(HttpServletRequest request) {
- if (this.maintenanceMode) {
- return super.getLastModified(request);
- } else {
- return ApplicationFactory.getNodeIndexer().getLastIndexed(
- DateTimeHelper.TEN_SECONDS);
- }
- }
-
- protected static class HttpServletRequestWrapper {
- private HttpServletRequest request;
- protected HttpServletRequestWrapper(HttpServletRequest request) {
- this.request = request;
- }
- protected String getRequestURI() {
- return this.request.getRequestURI();
- }
- @SuppressWarnings("rawtypes")
- protected Map getParameterMap() {
- return this.request.getParameterMap();
- }
- protected String[] getParameterValues(String parameterKey) {
- return this.request.getParameterValues(parameterKey);
- }
- }
-
- protected static class HttpServletResponseWrapper {
- private HttpServletResponse response = null;
- protected HttpServletResponseWrapper(HttpServletResponse response) {
- this.response = response;
- }
- protected void sendError(int errorStatusCode) throws IOException {
- this.response.sendError(errorStatusCode);
- }
- protected void setHeader(String headerName, String headerValue) {
- this.response.setHeader(headerName, headerValue);
- }
- protected void setContentType(String contentType) {
- this.response.setContentType(contentType);
- }
- protected void setCharacterEncoding(String characterEncoding) {
- this.response.setCharacterEncoding(characterEncoding);
- }
- protected PrintWriter getWriter() throws IOException {
- return this.response.getWriter();
- }
- }
-
- public void doGet(HttpServletRequest request,
- HttpServletResponse response) throws IOException, ServletException {
- HttpServletRequestWrapper requestWrapper =
- new HttpServletRequestWrapper(request);
- HttpServletResponseWrapper responseWrapper =
- new HttpServletResponseWrapper(response);
- this.doGet(requestWrapper, responseWrapper);
- }
-
- public void doGet(HttpServletRequestWrapper request,
- HttpServletResponseWrapper response) throws IOException {
-
- if (this.maintenanceMode) {
- response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
- return;
- }
-
- if (ApplicationFactory.getNodeIndexer().getLastIndexed(
- DateTimeHelper.TEN_SECONDS) + DateTimeHelper.SIX_HOURS
- < ApplicationFactory.getTime().currentTimeMillis()) {
- response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
- return;
- }
-
- NodeIndex nodeIndex = ApplicationFactory.getNodeIndexer().
- getLatestNodeIndex(DateTimeHelper.TEN_SECONDS);
- if (nodeIndex == null) {
- response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
- return;
- }
-
- String uri = request.getRequestURI();
- if (uri.startsWith("/onionoo/")) {
- uri = uri.substring("/onionoo".length());
- }
- String resourceType = null;
- if (uri.startsWith("/summary")) {
- resourceType = "summary";
- } else if (uri.startsWith("/details")) {
- resourceType = "details";
- } else if (uri.startsWith("/bandwidth")) {
- resourceType = "bandwidth";
- } else if (uri.startsWith("/weights")) {
- resourceType = "weights";
- } else if (uri.startsWith("/clients")) {
- resourceType = "clients";
- } else if (uri.startsWith("/uptime")) {
- resourceType = "uptime";
- } else {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
-
- RequestHandler rh = new RequestHandler(nodeIndex);
- rh.setResourceType(resourceType);
-
- /* Extract parameters either from the old-style URI or from request
- * parameters. */
- Map<String, String> parameterMap = new HashMap<String, String>();
- for (Object parameterKey : request.getParameterMap().keySet()) {
- String[] parameterValues =
- request.getParameterValues((String) parameterKey);
- parameterMap.put((String) parameterKey, parameterValues[0]);
- }
-
- /* Make sure that the request doesn't contain any unknown
- * parameters. */
- Set<String> knownParameters = new HashSet<String>(Arrays.asList((
- "type,running,search,lookup,fingerprint,country,as,flag,"
- + "first_seen_days,last_seen_days,contact,order,limit,offset,"
- + "fields,family").split(",")));
- for (String parameterKey : parameterMap.keySet()) {
- if (!knownParameters.contains(parameterKey)) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
- }
-
- /* Filter relays and bridges matching the request. */
- if (parameterMap.containsKey("type")) {
- String typeParameterValue = parameterMap.get("type").toLowerCase();
- boolean relaysRequested = true;
- if (typeParameterValue.equals("bridge")) {
- relaysRequested = false;
- } else if (!typeParameterValue.equals("relay")) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
- rh.setType(relaysRequested ? "relay" : "bridge");
- }
- if (parameterMap.containsKey("running")) {
- String runningParameterValue =
- parameterMap.get("running").toLowerCase();
- boolean runningRequested = true;
- if (runningParameterValue.equals("false")) {
- runningRequested = false;
- } else if (!runningParameterValue.equals("true")) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
- rh.setRunning(runningRequested ? "true" : "false");
- }
- if (parameterMap.containsKey("search")) {
- String[] searchTerms = this.parseSearchParameters(
- parameterMap.get("search"));
- if (searchTerms == null) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
- rh.setSearch(searchTerms);
- }
- if (parameterMap.containsKey("lookup")) {
- String lookupParameter = this.parseFingerprintParameter(
- parameterMap.get("lookup"));
- if (lookupParameter == null) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
- String fingerprint = lookupParameter.toUpperCase();
- rh.setLookup(fingerprint);
- }
- if (parameterMap.containsKey("fingerprint")) {
- String fingerprintParameter = this.parseFingerprintParameter(
- parameterMap.get("fingerprint"));
- if (fingerprintParameter == null) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
- String fingerprint = fingerprintParameter.toUpperCase();
- rh.setFingerprint(fingerprint);
- }
- if (parameterMap.containsKey("country")) {
- String countryCodeParameter = this.parseCountryCodeParameter(
- parameterMap.get("country"));
- if (countryCodeParameter == null) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
- rh.setCountry(countryCodeParameter);
- }
- if (parameterMap.containsKey("as")) {
- String aSNumberParameter = this.parseASNumberParameter(
- parameterMap.get("as"));
- if (aSNumberParameter == null) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
- rh.setAs(aSNumberParameter);
- }
- if (parameterMap.containsKey("flag")) {
- String flagParameter = this.parseFlagParameter(
- parameterMap.get("flag"));
- if (flagParameter == null) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
- rh.setFlag(flagParameter);
- }
- if (parameterMap.containsKey("first_seen_days")) {
- int[] days = this.parseDaysParameter(
- parameterMap.get("first_seen_days"));
- if (days == null) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
- rh.setFirstSeenDays(days);
- }
- if (parameterMap.containsKey("last_seen_days")) {
- int[] days = this.parseDaysParameter(
- parameterMap.get("last_seen_days"));
- if (days == null) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
- rh.setLastSeenDays(days);
- }
- if (parameterMap.containsKey("contact")) {
- String[] contactParts = this.parseContactParameter(
- parameterMap.get("contact"));
- if (contactParts == null) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
- rh.setContact(contactParts);
- }
- if (parameterMap.containsKey("order")) {
- String orderParameter = parameterMap.get("order").toLowerCase();
- String orderByField = orderParameter;
- if (orderByField.startsWith("-")) {
- orderByField = orderByField.substring(1);
- }
- if (!orderByField.equals("consensus_weight")) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
- rh.setOrder(new String[] { orderParameter });
- }
- if (parameterMap.containsKey("offset")) {
- String offsetParameter = parameterMap.get("offset");
- if (offsetParameter.length() > 6) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
- try {
- Integer.parseInt(offsetParameter);
- } catch (NumberFormatException e) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
- rh.setOffset(offsetParameter);
- }
- if (parameterMap.containsKey("limit")) {
- String limitParameter = parameterMap.get("limit");
- if (limitParameter.length() > 6) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
- try {
- Integer.parseInt(limitParameter);
- } catch (NumberFormatException e) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
- rh.setLimit(limitParameter);
- }
- if (parameterMap.containsKey("family")) {
- String familyParameter = this.parseFingerprintParameter(
- parameterMap.get("family"));
- if (familyParameter == null) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
- String family = familyParameter.toUpperCase();
- rh.setFamily(family);
- }
- rh.handleRequest();
-
- ResponseBuilder rb = new ResponseBuilder();
- rb.setResourceType(resourceType);
- rb.setRelaysPublishedString(rh.getRelaysPublishedString());
- rb.setBridgesPublishedString(rh.getBridgesPublishedString());
- rb.setOrderedRelays(rh.getOrderedRelays());
- rb.setOrderedBridges(rh.getOrderedBridges());
- String[] fields = null;
- if (parameterMap.containsKey("fields")) {
- fields = this.parseFieldsParameter(parameterMap.get("fields"));
- if (fields == null) {
- response.sendError(HttpServletResponse.SC_BAD_REQUEST);
- return;
- }
- rb.setFields(fields);
- }
-
- response.setHeader("Access-Control-Allow-Origin", "*");
- response.setContentType("application/json");
- response.setCharacterEncoding("utf-8");
- PrintWriter pw = response.getWriter();
- rb.buildResponse(pw);
- pw.flush();
- pw.close();
- }
-
- private static Pattern searchParameterPattern =
- Pattern.compile("^\\$?[0-9a-fA-F]{1,40}$|" /* Fingerprint. */
- + "^[0-9a-zA-Z\\.]{1,19}$|" /* Nickname or IPv4 address. */
- + "^\\[[0-9a-fA-F:\\.]{1,39}\\]?$"); /* IPv6 address. */
- private String[] parseSearchParameters(String parameter) {
- String[] searchParameters;
- if (parameter.contains(" ")) {
- searchParameters = parameter.split(" ");
- } else {
- searchParameters = new String[] { parameter };
- }
- for (String searchParameter : searchParameters) {
- if (!searchParameterPattern.matcher(searchParameter).matches()) {
- return null;
- }
- }
- return searchParameters;
- }
-
- private static Pattern fingerprintParameterPattern =
- Pattern.compile("^[0-9a-zA-Z]{1,40}$");
- private String parseFingerprintParameter(String parameter) {
- if (!fingerprintParameterPattern.matcher(parameter).matches()) {
- return null;
- }
- if (parameter.length() != 40) {
- return null;
- }
- return parameter;
- }
-
- private static Pattern countryCodeParameterPattern =
- Pattern.compile("^[0-9a-zA-Z]{2}$");
- private String parseCountryCodeParameter(String parameter) {
- if (!countryCodeParameterPattern.matcher(parameter).matches()) {
- return null;
- }
- return parameter;
- }
-
- private static Pattern aSNumberParameterPattern =
- Pattern.compile("^[asAS]{0,2}[0-9]{1,10}$");
- private String parseASNumberParameter(String parameter) {
- if (!aSNumberParameterPattern.matcher(parameter).matches()) {
- return null;
- }
- return parameter;
- }
-
- private static Pattern flagPattern =
- Pattern.compile("^[a-zA-Z0-9]{1,20}$");
- private String parseFlagParameter(String parameter) {
- if (!flagPattern.matcher(parameter).matches()) {
- return null;
- }
- return parameter;
- }
-
- private static Pattern daysPattern = Pattern.compile("^[0-9-]{1,10}$");
- private int[] parseDaysParameter(String parameter) {
- if (!daysPattern.matcher(parameter).matches()) {
- return null;
- }
- int x = 0, y = Integer.MAX_VALUE;
- try {
- if (!parameter.contains("-")) {
- x = Integer.parseInt(parameter);
- y = x;
- } else {
- String[] parts = parameter.split("-", 2);
- if (parts[0].length() > 0) {
- x = Integer.parseInt(parts[0]);
- }
- if (parts.length > 1 && parts[1].length() > 0) {
- y = Integer.parseInt(parts[1]);
- }
- }
- } catch (NumberFormatException e) {
- return null;
- }
- if (x > y) {
- return null;
- }
- return new int[] { x, y };
- }
-
- private String[] parseContactParameter(String parameter) {
- for (char c : parameter.toCharArray()) {
- if (c < 32 || c >= 127) {
- return null;
- }
- }
- return parameter.split(" ");
- }
-
- private static Pattern fieldsParameterPattern =
- Pattern.compile("^[0-9a-zA-Z_,]*$");
- private String[] parseFieldsParameter(String parameter) {
- if (!fieldsParameterPattern.matcher(parameter).matches()) {
- return null;
- }
- return parameter.split(",");
- }
-}
-
diff --git a/src/org/torproject/onionoo/ResponseBuilder.java b/src/org/torproject/onionoo/ResponseBuilder.java
deleted file mode 100644
index 2086ea8..0000000
--- a/src/org/torproject/onionoo/ResponseBuilder.java
+++ /dev/null
@@ -1,311 +0,0 @@
-/* Copyright 2011--2013 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.io.PrintWriter;
-import java.util.ArrayList;
-import java.util.List;
-
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-
-public class ResponseBuilder {
-
- private DocumentStore documentStore;
-
- public ResponseBuilder() {
- this.documentStore = ApplicationFactory.getDocumentStore();
- }
-
- private String resourceType;
- public void setResourceType(String resourceType) {
- this.resourceType = resourceType;
- }
-
- private String relaysPublishedString;
- public void setRelaysPublishedString(String relaysPublishedString) {
- this.relaysPublishedString = relaysPublishedString;
- }
-
- private String bridgesPublishedString;
- public void setBridgesPublishedString(String bridgesPublishedString) {
- this.bridgesPublishedString = bridgesPublishedString;
- }
-
- private List<SummaryDocument> orderedRelays =
- new ArrayList<SummaryDocument>();
- public void setOrderedRelays(List<SummaryDocument> orderedRelays) {
- this.orderedRelays = orderedRelays;
- }
-
- private List<SummaryDocument> orderedBridges =
- new ArrayList<SummaryDocument>();
- public void setOrderedBridges(List<SummaryDocument> orderedBridges) {
- this.orderedBridges = orderedBridges;
- }
-
- private String[] fields;
- public void setFields(String[] fields) {
- this.fields = new String[fields.length];
- System.arraycopy(fields, 0, this.fields, 0, fields.length);
- }
-
- public void buildResponse(PrintWriter pw) {
- writeRelays(orderedRelays, pw);
- writeBridges(orderedBridges, pw);
- }
-
- private void writeRelays(List<SummaryDocument> relays, PrintWriter pw) {
- pw.write("{\"relays_published\":\"" + relaysPublishedString
- + "\",\n\"relays\":[");
- int written = 0;
- for (SummaryDocument entry : relays) {
- String lines = this.formatNodeStatus(entry);
- if (lines.length() > 0) {
- pw.print((written++ > 0 ? ",\n" : "\n") + lines);
- }
- }
- pw.print("\n],\n");
- }
-
- private void writeBridges(List<SummaryDocument> bridges,
- PrintWriter pw) {
- pw.write("\"bridges_published\":\"" + bridgesPublishedString
- + "\",\n\"bridges\":[");
- int written = 0;
- for (SummaryDocument entry : bridges) {
- String lines = this.formatNodeStatus(entry);
- if (lines.length() > 0) {
- pw.print((written++ > 0 ? ",\n" : "\n") + lines);
- }
- }
- pw.print("\n]}\n");
- }
-
- private String formatNodeStatus(SummaryDocument entry) {
- if (this.resourceType == null) {
- return "";
- } else if (this.resourceType.equals("summary")) {
- return this.writeSummaryLine(entry);
- } else if (this.resourceType.equals("details")) {
- return this.writeDetailsLines(entry);
- } else if (this.resourceType.equals("bandwidth")) {
- return this.writeBandwidthLines(entry);
- } else if (this.resourceType.equals("weights")) {
- return this.writeWeightsLines(entry);
- } else if (this.resourceType.equals("clients")) {
- return this.writeClientsLines(entry);
- } else if (this.resourceType.equals("uptime")) {
- return this.writeUptimeLines(entry);
- } else {
- return "";
- }
- }
-
- private String writeSummaryLine(SummaryDocument entry) {
- return entry.isRelay() ? writeRelaySummaryLine(entry)
- : writeBridgeSummaryLine(entry);
- }
-
- private String writeRelaySummaryLine(SummaryDocument entry) {
- String nickname = !entry.getNickname().equals("Unnamed") ?
- entry.getNickname() : null;
- String fingerprint = entry.getFingerprint();
- String running = entry.isRunning() ? "true" : "false";
- List<String> addresses = entry.getAddresses();
- StringBuilder addressesBuilder = new StringBuilder();
- int written = 0;
- for (String address : addresses) {
- addressesBuilder.append((written++ > 0 ? "," : "") + "\""
- + address.toLowerCase() + "\"");
- }
- return String.format("{%s\"f\":\"%s\",\"a\":[%s],\"r\":%s}",
- (nickname == null ? "" : "\"n\":\"" + nickname + "\","),
- fingerprint, addressesBuilder.toString(), running);
- }
-
- private String writeBridgeSummaryLine(SummaryDocument entry) {
- String nickname = !entry.getNickname().equals("Unnamed") ?
- entry.getNickname() : null;
- String hashedFingerprint = entry.getFingerprint();
- String running = entry.isRunning() ? "true" : "false";
- return String.format("{%s\"h\":\"%s\",\"r\":%s}",
- (nickname == null ? "" : "\"n\":\"" + nickname + "\","),
- hashedFingerprint, running);
- }
-
- private String writeDetailsLines(SummaryDocument entry) {
- String fingerprint = entry.getFingerprint();
- if (this.fields != null) {
- /* TODO Maybe there's a more elegant way (more maintainable, more
- * efficient, etc.) to implement this? */
- DetailsDocument detailsDocument = documentStore.retrieve(
- DetailsDocument.class, true, fingerprint);
- if (detailsDocument != null) {
- DetailsDocument dd = new DetailsDocument();
- for (String field : this.fields) {
- if (field.equals("nickname")) {
- dd.setNickname(detailsDocument.getNickname());
- } else if (field.equals("fingerprint")) {
- dd.setFingerprint(detailsDocument.getFingerprint());
- } else if (field.equals("hashed_fingerprint")) {
- dd.setHashedFingerprint(
- detailsDocument.getHashedFingerprint());
- } else if (field.equals("or_addresses")) {
- dd.setOrAddresses(detailsDocument.getOrAddresses());
- } else if (field.equals("exit_addresses")) {
- dd.setExitAddresses(detailsDocument.getExitAddresses());
- } else if (field.equals("dir_address")) {
- dd.setDirAddress(detailsDocument.getDirAddress());
- } else if (field.equals("last_seen")) {
- dd.setLastSeen(detailsDocument.getLastSeen());
- } else if (field.equals("last_changed_address_or_port")) {
- dd.setLastChangedAddressOrPort(
- detailsDocument.getLastChangedAddressOrPort());
- } else if (field.equals("first_seen")) {
- dd.setFirstSeen(detailsDocument.getFirstSeen());
- } else if (field.equals("running")) {
- dd.setRunning(detailsDocument.getRunning());
- } else if (field.equals("flags")) {
- dd.setFlags(detailsDocument.getFlags());
- } else if (field.equals("country")) {
- dd.setCountry(detailsDocument.getCountry());
- } else if (field.equals("country_name")) {
- dd.setCountryName(detailsDocument.getCountryName());
- } else if (field.equals("region_name")) {
- dd.setRegionName(detailsDocument.getRegionName());
- } else if (field.equals("city_name")) {
- dd.setCityName(detailsDocument.getCityName());
- } else if (field.equals("latitude")) {
- dd.setLatitude(detailsDocument.getLatitude());
- } else if (field.equals("longitude")) {
- dd.setLongitude(detailsDocument.getLongitude());
- } else if (field.equals("as_number")) {
- dd.setAsNumber(detailsDocument.getAsNumber());
- } else if (field.equals("as_name")) {
- dd.setAsName(detailsDocument.getAsName());
- } else if (field.equals("consensus_weight")) {
- dd.setConsensusWeight(detailsDocument.getConsensusWeight());
- } else if (field.equals("host_name")) {
- dd.setHostName(detailsDocument.getHostName());
- } else if (field.equals("last_restarted")) {
- dd.setLastRestarted(detailsDocument.getLastRestarted());
- } else if (field.equals("bandwidth_rate")) {
- dd.setBandwidthRate(detailsDocument.getBandwidthRate());
- } else if (field.equals("bandwidth_burst")) {
- dd.setBandwidthBurst(detailsDocument.getBandwidthBurst());
- } else if (field.equals("observed_bandwidth")) {
- dd.setObservedBandwidth(
- detailsDocument.getObservedBandwidth());
- } else if (field.equals("advertised_bandwidth")) {
- dd.setAdvertisedBandwidth(
- detailsDocument.getAdvertisedBandwidth());
- } else if (field.equals("exit_policy")) {
- dd.setExitPolicy(detailsDocument.getExitPolicy());
- } else if (field.equals("exit_policy_summary")) {
- dd.setExitPolicySummary(
- detailsDocument.getExitPolicySummary());
- } else if (field.equals("exit_policy_v6_summary")) {
- dd.setExitPolicyV6Summary(
- detailsDocument.getExitPolicyV6Summary());
- } else if (field.equals("contact")) {
- dd.setContact(detailsDocument.getContact());
- } else if (field.equals("platform")) {
- dd.setPlatform(detailsDocument.getPlatform());
- } else if (field.equals("family")) {
- dd.setFamily(detailsDocument.getFamily());
- } else if (field.equals("advertised_bandwidth_fraction")) {
- dd.setAdvertisedBandwidthFraction(
- detailsDocument.getAdvertisedBandwidthFraction());
- } else if (field.equals("consensus_weight_fraction")) {
- dd.setConsensusWeightFraction(
- detailsDocument.getConsensusWeightFraction());
- } else if (field.equals("guard_probability")) {
- dd.setGuardProbability(detailsDocument.getGuardProbability());
- } else if (field.equals("middle_probability")) {
- dd.setMiddleProbability(
- detailsDocument.getMiddleProbability());
- } else if (field.equals("exit_probability")) {
- dd.setExitProbability(detailsDocument.getExitProbability());
- } else if (field.equals("recommended_version")) {
- dd.setRecommendedVersion(
- detailsDocument.getRecommendedVersion());
- } else if (field.equals("hibernating")) {
- dd.setHibernating(detailsDocument.getHibernating());
- } else if (field.equals("pool_assignment")) {
- dd.setPoolAssignment(detailsDocument.getPoolAssignment());
- }
- }
- /* Don't escape HTML characters, like < and >, contained in
- * strings. */
- Gson gson = new GsonBuilder().disableHtmlEscaping().create();
- /* Whenever we provide Gson with a string containing an escaped
- * non-ASCII character like \u00F2, it escapes the \ to \\, which
- * we need to undo before including the string in a response. */
- return gson.toJson(dd).replaceAll("\\\\\\\\u", "\\\\u");
- } else {
- // TODO We should probably log that we didn't find a details
- // document that we expected to exist.
- return "";
- }
- } else {
- DetailsDocument detailsDocument = documentStore.retrieve(
- DetailsDocument.class, false, fingerprint);
- if (detailsDocument != null) {
- return detailsDocument.getDocumentString();
- } else {
- // TODO We should probably log that we didn't find a details
- // document that we expected to exist.
- return "";
- }
- }
- }
-
- private String writeBandwidthLines(SummaryDocument entry) {
- String fingerprint = entry.getFingerprint();
- BandwidthDocument bandwidthDocument = this.documentStore.retrieve(
- BandwidthDocument.class, false, fingerprint);
- if (bandwidthDocument != null &&
- bandwidthDocument.getDocumentString() != null) {
- return bandwidthDocument.getDocumentString();
- } else {
- return "{\"fingerprint\":\"" + fingerprint.toUpperCase() + "\"}";
- }
- }
-
- private String writeWeightsLines(SummaryDocument entry) {
- String fingerprint = entry.getFingerprint();
- WeightsDocument weightsDocument = this.documentStore.retrieve(
- WeightsDocument.class, false, fingerprint);
- if (weightsDocument != null &&
- weightsDocument.getDocumentString() != null) {
- return weightsDocument.getDocumentString();
- } else {
- return "{\"fingerprint\":\"" + fingerprint.toUpperCase() + "\"}";
- }
- }
-
- private String writeClientsLines(SummaryDocument entry) {
- String fingerprint = entry.getFingerprint();
- ClientsDocument clientsDocument = this.documentStore.retrieve(
- ClientsDocument.class, false, fingerprint);
- if (clientsDocument != null &&
- clientsDocument.getDocumentString() != null) {
- return clientsDocument.getDocumentString();
- } else {
- return "{\"fingerprint\":\"" + fingerprint.toUpperCase() + "\"}";
- }
- }
-
- private String writeUptimeLines(SummaryDocument entry) {
- String fingerprint = entry.getFingerprint();
- UptimeDocument uptimeDocument = this.documentStore.retrieve(
- UptimeDocument.class, false, fingerprint);
- if (uptimeDocument != null &&
- uptimeDocument.getDocumentString() != null) {
- return uptimeDocument.getDocumentString();
- } else {
- return "{\"fingerprint\":\"" + fingerprint.toUpperCase() + "\"}";
- }
- }
-}
diff --git a/src/org/torproject/onionoo/ReverseDomainNameResolver.java b/src/org/torproject/onionoo/ReverseDomainNameResolver.java
deleted file mode 100644
index 3dbf9d1..0000000
--- a/src/org/torproject/onionoo/ReverseDomainNameResolver.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/* Copyright 2013 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-public class ReverseDomainNameResolver {
-
- private class RdnsLookupWorker extends Thread {
- public void run() {
- while (time.currentTimeMillis() - RDNS_LOOKUP_MAX_DURATION_MILLIS
- <= startedRdnsLookups) {
- String rdnsLookupJob = null;
- synchronized (rdnsLookupJobs) {
- for (String job : rdnsLookupJobs) {
- rdnsLookupJob = job;
- rdnsLookupJobs.remove(job);
- break;
- }
- }
- if (rdnsLookupJob == null) {
- break;
- }
- RdnsLookupRequest request = new RdnsLookupRequest(this,
- rdnsLookupJob);
- request.setDaemon(true);
- request.start();
- try {
- Thread.sleep(RDNS_LOOKUP_MAX_REQUEST_MILLIS);
- } catch (InterruptedException e) {
- /* Getting interrupted should be the default case. */
- }
- String hostName = request.getHostName();
- if (hostName != null) {
- synchronized (rdnsLookupResults) {
- rdnsLookupResults.put(rdnsLookupJob, hostName);
- }
- }
- long lookupMillis = request.getLookupMillis();
- if (lookupMillis >= 0L) {
- synchronized (rdnsLookupMillis) {
- rdnsLookupMillis.add(lookupMillis);
- }
- }
- }
- }
- }
-
- private class RdnsLookupRequest extends Thread {
- private RdnsLookupWorker parent;
- private String address, hostName;
- private long lookupStartedMillis = -1L, lookupCompletedMillis = -1L;
- public RdnsLookupRequest(RdnsLookupWorker parent, String address) {
- this.parent = parent;
- this.address = address;
- }
- public void run() {
- this.lookupStartedMillis = time.currentTimeMillis();
- try {
- String result = InetAddress.getByName(this.address).getHostName();
- synchronized (this) {
- this.hostName = result;
- }
- } catch (UnknownHostException e) {
- /* We'll try again the next time. */
- }
- this.lookupCompletedMillis = time.currentTimeMillis();
- this.parent.interrupt();
- }
- public synchronized String getHostName() {
- return hostName;
- }
- public synchronized long getLookupMillis() {
- return this.lookupCompletedMillis - this.lookupStartedMillis;
- }
- }
-
- private Time time;
-
- public ReverseDomainNameResolver() {
- this.time = ApplicationFactory.getTime();
- }
-
- private static final long RDNS_LOOKUP_MAX_REQUEST_MILLIS =
- DateTimeHelper.TEN_SECONDS;
- private static final long RDNS_LOOKUP_MAX_DURATION_MILLIS =
- DateTimeHelper.FIVE_MINUTES;
- private static final long RDNS_LOOKUP_MAX_AGE_MILLIS =
- DateTimeHelper.TWELVE_HOURS;
- private static final int RDNS_LOOKUP_WORKERS_NUM = 5;
-
- private Map<String, Long> addressLastLookupTimes;
-
- private Set<String> rdnsLookupJobs;
-
- private Map<String, String> rdnsLookupResults;
-
- private List<Long> rdnsLookupMillis;
-
- private long startedRdnsLookups;
-
- private List<RdnsLookupWorker> rdnsLookupWorkers;
-
- public void setAddresses(Map<String, Long> addressLastLookupTimes) {
- this.addressLastLookupTimes = addressLastLookupTimes;
- }
-
- public void startReverseDomainNameLookups() {
- this.startedRdnsLookups = this.time.currentTimeMillis();
- this.rdnsLookupJobs = new HashSet<String>();
- for (Map.Entry<String, Long> e :
- this.addressLastLookupTimes.entrySet()) {
- if (e.getValue() < this.startedRdnsLookups
- - RDNS_LOOKUP_MAX_AGE_MILLIS) {
- this.rdnsLookupJobs.add(e.getKey());
- }
- }
- this.rdnsLookupResults = new HashMap<String, String>();
- this.rdnsLookupMillis = new ArrayList<Long>();
- this.rdnsLookupWorkers = new ArrayList<RdnsLookupWorker>();
- for (int i = 0; i < RDNS_LOOKUP_WORKERS_NUM; i++) {
- RdnsLookupWorker rdnsLookupWorker = new RdnsLookupWorker();
- this.rdnsLookupWorkers.add(rdnsLookupWorker);
- rdnsLookupWorker.setDaemon(true);
- rdnsLookupWorker.start();
- }
- }
-
- public void finishReverseDomainNameLookups() {
- for (RdnsLookupWorker rdnsLookupWorker : this.rdnsLookupWorkers) {
- try {
- rdnsLookupWorker.join();
- } catch (InterruptedException e) {
- /* This is not something that we can take care of. Just leave the
- * worker thread alone. */
- }
- }
- }
-
- public Map<String, String> getLookupResults() {
- synchronized (this.rdnsLookupResults) {
- return new HashMap<String, String>(this.rdnsLookupResults);
- }
- }
-
- public long getLookupStartMillis() {
- return this.startedRdnsLookups;
- }
-
- public String getStatsString() {
- StringBuilder sb = new StringBuilder();
- sb.append(" " + Logger.formatDecimalNumber(rdnsLookupMillis.size())
- + " lookups performed\n");
- if (rdnsLookupMillis.size() > 0) {
- Collections.sort(rdnsLookupMillis);
- sb.append(" " + Logger.formatMillis(rdnsLookupMillis.get(0))
- + " minimum lookup time\n");
- sb.append(" " + Logger.formatMillis(rdnsLookupMillis.get(
- rdnsLookupMillis.size() / 2)) + " median lookup time\n");
- sb.append(" " + Logger.formatMillis(rdnsLookupMillis.get(
- rdnsLookupMillis.size() - 1)) + " maximum lookup time\n");
- }
- return sb.toString();
- }
-}
-
diff --git a/src/org/torproject/onionoo/StatusUpdater.java b/src/org/torproject/onionoo/StatusUpdater.java
deleted file mode 100644
index fb82182..0000000
--- a/src/org/torproject/onionoo/StatusUpdater.java
+++ /dev/null
@@ -1,11 +0,0 @@
-/* Copyright 2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-public interface StatusUpdater {
-
- public abstract void updateStatuses();
-
- public abstract String getStatsString();
-}
-
diff --git a/src/org/torproject/onionoo/SummaryDocument.java b/src/org/torproject/onionoo/SummaryDocument.java
deleted file mode 100644
index 61a3ec6..0000000
--- a/src/org/torproject/onionoo/SummaryDocument.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/* Copyright 2013--2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.List;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.regex.Pattern;
-
-import org.apache.commons.codec.DecoderException;
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.codec.digest.DigestUtils;
-
-public class SummaryDocument extends Document {
-
- private boolean t;
- public void setRelay(boolean isRelay) {
- this.t = isRelay;
- }
- public boolean isRelay() {
- return this.t;
- }
-
- private String f;
- public void setFingerprint(String fingerprint) {
- if (fingerprint != null) {
- Pattern fingerprintPattern = Pattern.compile("^[0-9a-fA-F]{40}$");
- if (!fingerprintPattern.matcher(fingerprint).matches()) {
- throw new IllegalArgumentException("Fingerprint '" + fingerprint
- + "' is not a valid fingerprint.");
- }
- }
- this.f = fingerprint;
- }
- public String getFingerprint() {
- return this.f;
- }
-
- public String getHashedFingerprint() {
- String hashedFingerprint = null;
- if (this.f != null) {
- try {
- hashedFingerprint = DigestUtils.shaHex(Hex.decodeHex(
- this.f.toCharArray())).toUpperCase();
- } catch (DecoderException e) {
- /* Format tested in setFingerprint(). */
- }
- }
- return hashedFingerprint;
- }
-
- private String n;
- public void setNickname(String nickname) {
- if (nickname == null || nickname.equals("Unnamed")) {
- this.n = null;
- } else {
- this.n = nickname;
- }
- }
- public String getNickname() {
- return this.n == null ? "Unnamed" : this.n;
- }
-
- private String[] ad;
- public void setAddresses(List<String> addresses) {
- this.ad = this.collectionToStringArray(addresses);
- }
- public List<String> getAddresses() {
- return this.stringArrayToList(this.ad);
- }
-
- private String[] collectionToStringArray(
- Collection<String> collection) {
- String[] stringArray = null;
- if (collection != null && !collection.isEmpty()) {
- stringArray = new String[collection.size()];
- int i = 0;
- for (String string : collection) {
- stringArray[i++] = string;
- }
- }
- return stringArray;
- }
- private List<String> stringArrayToList(String[] stringArray) {
- List<String> list;
- if (stringArray == null) {
- list = new ArrayList<String>();
- } else {
- list = Arrays.asList(stringArray);
- }
- return list;
- }
- private SortedSet<String> stringArrayToSortedSet(String[] stringArray) {
- SortedSet<String> sortedSet = new TreeSet<String>();
- if (stringArray != null) {
- sortedSet.addAll(Arrays.asList(stringArray));
- }
- return sortedSet;
- }
-
- private String cc;
- public void setCountryCode(String countryCode) {
- this.cc = countryCode;
- }
- public String getCountryCode() {
- return this.cc;
- }
-
- private String as;
- public void setASNumber(String aSNumber) {
- this.as = aSNumber;
- }
- public String getASNumber() {
- return this.as;
- }
-
- private String fs;
- public void setFirstSeenMillis(long firstSeenMillis) {
- this.fs = DateTimeHelper.format(firstSeenMillis);
- }
- public long getFirstSeenMillis() {
- return DateTimeHelper.parse(this.fs);
- }
-
- private String ls;
- public void setLastSeenMillis(long lastSeenMillis) {
- this.ls = DateTimeHelper.format(lastSeenMillis);
- }
- public long getLastSeenMillis() {
- return DateTimeHelper.parse(this.ls);
- }
-
- private String[] rf;
- public void setRelayFlags(SortedSet<String> relayFlags) {
- this.rf = this.collectionToStringArray(relayFlags);
- }
- public SortedSet<String> getRelayFlags() {
- return this.stringArrayToSortedSet(this.rf);
- }
-
- private long cw;
- public void setConsensusWeight(long consensusWeight) {
- this.cw = consensusWeight;
- }
- public long getConsensusWeight() {
- return this.cw;
- }
-
- private boolean r;
- public void setRunning(boolean isRunning) {
- this.r = isRunning;
- }
- public boolean isRunning() {
- return this.r;
- }
-
- private String c;
- public void setContact(String contact) {
- if (contact != null && contact.length() == 0) {
- this.c = null;
- } else {
- this.c = contact;
- }
- }
- public String getContact() {
- return this.c;
- }
-
- private String[] ff;
- public void setFamilyFingerprints(
- SortedSet<String> familyFingerprints) {
- this.ff = this.collectionToStringArray(familyFingerprints);
- }
- public SortedSet<String> getFamilyFingerprints() {
- return this.stringArrayToSortedSet(this.ff);
- }
-
- public SummaryDocument(boolean isRelay, String nickname,
- String fingerprint, List<String> addresses, long lastSeenMillis,
- boolean running, SortedSet<String> relayFlags, long consensusWeight,
- String countryCode, long firstSeenMillis, String aSNumber,
- String contact, SortedSet<String> familyFingerprints) {
- this.setRelay(isRelay);
- this.setNickname(nickname);
- this.setFingerprint(fingerprint);
- this.setAddresses(addresses);
- this.setLastSeenMillis(lastSeenMillis);
- this.setRunning(running);
- this.setRelayFlags(relayFlags);
- this.setConsensusWeight(consensusWeight);
- this.setCountryCode(countryCode);
- this.setFirstSeenMillis(firstSeenMillis);
- this.setASNumber(aSNumber);
- this.setContact(contact);
- this.setFamilyFingerprints(familyFingerprints);
- }
-}
-
diff --git a/src/org/torproject/onionoo/SummaryDocumentWriter.java b/src/org/torproject/onionoo/SummaryDocumentWriter.java
deleted file mode 100644
index 7b257f5..0000000
--- a/src/org/torproject/onionoo/SummaryDocumentWriter.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/* Copyright 2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.SortedSet;
-
-public class SummaryDocumentWriter implements DocumentWriter {
-
- private DocumentStore documentStore;
-
- public SummaryDocumentWriter() {
- this.documentStore = ApplicationFactory.getDocumentStore();
- }
-
- private int writtenDocuments = 0, deletedDocuments = 0;
-
- public void writeDocuments() {
- long maxLastSeenMillis = 0L;
- for (String fingerprint : this.documentStore.list(NodeStatus.class)) {
- NodeStatus nodeStatus = this.documentStore.retrieve(
- NodeStatus.class, true, fingerprint);
- if (nodeStatus != null &&
- nodeStatus.getLastSeenMillis() > maxLastSeenMillis) {
- maxLastSeenMillis = nodeStatus.getLastSeenMillis();
- }
- }
- long cutoff = maxLastSeenMillis - DateTimeHelper.ONE_WEEK;
- for (String fingerprint : this.documentStore.list(NodeStatus.class)) {
- NodeStatus nodeStatus = this.documentStore.retrieve(
- NodeStatus.class,
- true, fingerprint);
- if (nodeStatus == null) {
- continue;
- }
- if (nodeStatus.getLastSeenMillis() < cutoff) {
- if (this.documentStore.remove(SummaryDocument.class,
- fingerprint)) {
- this.deletedDocuments++;
- }
- continue;
- }
- boolean isRelay = nodeStatus.isRelay();
- String nickname = nodeStatus.getNickname();
- List<String> addresses = new ArrayList<String>();
- addresses.add(nodeStatus.getAddress());
- for (String orAddress : nodeStatus.getOrAddresses()) {
- if (!addresses.contains(orAddress)) {
- addresses.add(orAddress);
- }
- }
- for (String exitAddress : nodeStatus.getExitAddresses()) {
- if (!addresses.contains(exitAddress)) {
- addresses.add(exitAddress);
- }
- }
- long lastSeenMillis = nodeStatus.getLastSeenMillis();
- boolean running = nodeStatus.getRunning();
- SortedSet<String> relayFlags = nodeStatus.getRelayFlags();
- long consensusWeight = nodeStatus.getConsensusWeight();
- String countryCode = nodeStatus.getCountryCode();
- long firstSeenMillis = nodeStatus.getFirstSeenMillis();
- String aSNumber = nodeStatus.getASNumber();
- String contact = nodeStatus.getContact();
- SortedSet<String> familyFingerprints =
- nodeStatus.getFamilyFingerprints();
- SummaryDocument summaryDocument = new SummaryDocument(isRelay,
- nickname, fingerprint, addresses, lastSeenMillis, running,
- relayFlags, consensusWeight, countryCode, firstSeenMillis,
- aSNumber, contact, familyFingerprints);
- if (this.documentStore.store(summaryDocument, fingerprint)) {
- this.writtenDocuments++;
- };
- }
- Logger.printStatusTime("Wrote summary document files");
- }
-
- public String getStatsString() {
- StringBuilder sb = new StringBuilder();
- sb.append(" " + Logger.formatDecimalNumber(this.writtenDocuments)
- + " summary document files written\n");
- sb.append(" " + Logger.formatDecimalNumber(this.deletedDocuments)
- + " summary document files deleted\n");
- return sb.toString();
- }
-}
diff --git a/src/org/torproject/onionoo/Time.java b/src/org/torproject/onionoo/Time.java
deleted file mode 100644
index c969556..0000000
--- a/src/org/torproject/onionoo/Time.java
+++ /dev/null
@@ -1,14 +0,0 @@
-/* Copyright 2013 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-/*
- * Wrapper for System.currentTimeMillis() that can be replaced with a
- * custom time source for testing.
- */
-public class Time {
- public long currentTimeMillis() {
- return System.currentTimeMillis();
- }
-}
-
diff --git a/src/org/torproject/onionoo/UpdateStatus.java b/src/org/torproject/onionoo/UpdateStatus.java
deleted file mode 100644
index a697cbf..0000000
--- a/src/org/torproject/onionoo/UpdateStatus.java
+++ /dev/null
@@ -1,7 +0,0 @@
-/* Copyright 2013 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-public class UpdateStatus extends Document {
-}
-
diff --git a/src/org/torproject/onionoo/UptimeDocument.java b/src/org/torproject/onionoo/UptimeDocument.java
deleted file mode 100644
index 1946e2c..0000000
--- a/src/org/torproject/onionoo/UptimeDocument.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/* Copyright 2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.Map;
-
-public class UptimeDocument extends Document {
-
- @SuppressWarnings("unused")
- private String fingerprint;
- public void setFingerprint(String fingerprint) {
- this.fingerprint = fingerprint;
- }
-
- private Map<String, GraphHistory> uptime;
- public void setUptime(Map<String, GraphHistory> uptime) {
- this.uptime = uptime;
- }
- public Map<String, GraphHistory> getUptime() {
- return this.uptime;
- }
-}
-
diff --git a/src/org/torproject/onionoo/UptimeDocumentWriter.java b/src/org/torproject/onionoo/UptimeDocumentWriter.java
deleted file mode 100644
index 8293f77..0000000
--- a/src/org/torproject/onionoo/UptimeDocumentWriter.java
+++ /dev/null
@@ -1,291 +0,0 @@
-/* Copyright 2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.ArrayList;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-public class UptimeDocumentWriter implements FingerprintListener,
- DocumentWriter {
-
- private DescriptorSource descriptorSource;
-
- private DocumentStore documentStore;
-
- private long now;
-
- public UptimeDocumentWriter() {
- this.descriptorSource = ApplicationFactory.getDescriptorSource();
- this.documentStore = ApplicationFactory.getDocumentStore();
- this.now = ApplicationFactory.getTime().currentTimeMillis();
- this.registerFingerprintListeners();
- }
-
- private void registerFingerprintListeners() {
- this.descriptorSource.registerFingerprintListener(this,
- DescriptorType.RELAY_CONSENSUSES);
- this.descriptorSource.registerFingerprintListener(this,
- DescriptorType.BRIDGE_STATUSES);
- }
-
- private SortedSet<String> newRelayFingerprints = new TreeSet<String>(),
- newBridgeFingerprints = new TreeSet<String>();
-
- public void processFingerprints(SortedSet<String> fingerprints,
- boolean relay) {
- if (relay) {
- this.newRelayFingerprints.addAll(fingerprints);
- } else {
- this.newBridgeFingerprints.addAll(fingerprints);
- }
- }
-
- public void writeDocuments() {
- UptimeStatus uptimeStatus = this.documentStore.retrieve(
- UptimeStatus.class, true);
- if (uptimeStatus == null) {
- return;
- }
- for (String fingerprint : this.newRelayFingerprints) {
- this.updateDocument(true, fingerprint,
- uptimeStatus.getRelayHistory());
- }
- for (String fingerprint : this.newBridgeFingerprints) {
- this.updateDocument(false, fingerprint,
- uptimeStatus.getBridgeHistory());
- }
- Logger.printStatusTime("Wrote uptime document files");
- }
-
- private int writtenDocuments = 0;
-
- private void updateDocument(boolean relay, String fingerprint,
- SortedSet<UptimeHistory> knownStatuses) {
- UptimeStatus uptimeStatus = this.documentStore.retrieve(
- UptimeStatus.class, true, fingerprint);
- if (uptimeStatus != null) {
- SortedSet<UptimeHistory> history = relay
- ? uptimeStatus.getRelayHistory()
- : uptimeStatus.getBridgeHistory();
- UptimeDocument uptimeDocument = this.compileUptimeDocument(relay,
- fingerprint, history, knownStatuses);
- this.documentStore.store(uptimeDocument, fingerprint);
- this.writtenDocuments++;
- }
- }
-
- private String[] graphNames = new String[] {
- "1_week",
- "1_month",
- "3_months",
- "1_year",
- "5_years" };
-
- private long[] graphIntervals = new long[] {
- DateTimeHelper.ONE_WEEK,
- DateTimeHelper.ROUGHLY_ONE_MONTH,
- DateTimeHelper.ROUGHLY_THREE_MONTHS,
- DateTimeHelper.ROUGHLY_ONE_YEAR,
- DateTimeHelper.ROUGHLY_FIVE_YEARS };
-
- private long[] dataPointIntervals = new long[] {
- DateTimeHelper.ONE_HOUR,
- DateTimeHelper.FOUR_HOURS,
- DateTimeHelper.TWELVE_HOURS,
- DateTimeHelper.TWO_DAYS,
- DateTimeHelper.TEN_DAYS };
-
- private UptimeDocument compileUptimeDocument(boolean relay,
- String fingerprint, SortedSet<UptimeHistory> history,
- SortedSet<UptimeHistory> knownStatuses) {
- UptimeDocument uptimeDocument = new UptimeDocument();
- uptimeDocument.setFingerprint(fingerprint);
- Map<String, GraphHistory> uptime =
- new LinkedHashMap<String, GraphHistory>();
- for (int graphIntervalIndex = 0; graphIntervalIndex <
- this.graphIntervals.length; graphIntervalIndex++) {
- String graphName = this.graphNames[graphIntervalIndex];
- GraphHistory graphHistory = this.compileUptimeHistory(
- graphIntervalIndex, relay, history, knownStatuses);
- if (graphHistory != null) {
- uptime.put(graphName, graphHistory);
- }
- }
- uptimeDocument.setUptime(uptime);
- return uptimeDocument;
- }
-
- private GraphHistory compileUptimeHistory(int graphIntervalIndex,
- boolean relay, SortedSet<UptimeHistory> history,
- SortedSet<UptimeHistory> knownStatuses) {
- long graphInterval = this.graphIntervals[graphIntervalIndex];
- long dataPointInterval =
- this.dataPointIntervals[graphIntervalIndex];
- int dataPointIntervalHours = (int) (dataPointInterval
- / DateTimeHelper.ONE_HOUR);
- List<Integer> uptimeDataPoints = new ArrayList<Integer>();
- long intervalStartMillis = ((this.now - graphInterval)
- / dataPointInterval) * dataPointInterval;
- int uptimeHours = 0;
- long firstStatusStartMillis = -1L;
- for (UptimeHistory hist : history) {
- if (hist.isRelay() != relay) {
- continue;
- }
- if (firstStatusStartMillis < 0L) {
- firstStatusStartMillis = hist.getStartMillis();
- }
- long histEndMillis = hist.getStartMillis() + DateTimeHelper.ONE_HOUR
- * hist.getUptimeHours();
- if (histEndMillis < intervalStartMillis) {
- continue;
- }
- while (hist.getStartMillis() >= intervalStartMillis
- + dataPointInterval) {
- if (firstStatusStartMillis < intervalStartMillis
- + dataPointInterval) {
- uptimeDataPoints.add(uptimeHours);
- } else {
- uptimeDataPoints.add(-1);
- }
- uptimeHours = 0;
- intervalStartMillis += dataPointInterval;
- }
- while (histEndMillis >= intervalStartMillis + dataPointInterval) {
- uptimeHours += (int) ((intervalStartMillis + dataPointInterval
- - Math.max(hist.getStartMillis(), intervalStartMillis))
- / DateTimeHelper.ONE_HOUR);
- uptimeDataPoints.add(uptimeHours);
- uptimeHours = 0;
- intervalStartMillis += dataPointInterval;
- }
- uptimeHours += (int) ((histEndMillis - Math.max(
- hist.getStartMillis(), intervalStartMillis))
- / DateTimeHelper.ONE_HOUR);
- }
- uptimeDataPoints.add(uptimeHours);
- List<Integer> statusDataPoints = new ArrayList<Integer>();
- intervalStartMillis = ((this.now - graphInterval)
- / dataPointInterval) * dataPointInterval;
- int statusHours = -1;
- for (UptimeHistory hist : knownStatuses) {
- if (hist.isRelay() != relay) {
- continue;
- }
- long histEndMillis = hist.getStartMillis() + DateTimeHelper.ONE_HOUR
- * hist.getUptimeHours();
- if (histEndMillis < intervalStartMillis) {
- continue;
- }
- while (hist.getStartMillis() >= intervalStartMillis
- + dataPointInterval) {
- statusDataPoints.add(statusHours * 5 > dataPointIntervalHours
- ? statusHours : -1);
- statusHours = -1;
- intervalStartMillis += dataPointInterval;
- }
- while (histEndMillis >= intervalStartMillis + dataPointInterval) {
- if (statusHours < 0) {
- statusHours = 0;
- }
- statusHours += (int) ((intervalStartMillis + dataPointInterval
- - Math.max(Math.max(hist.getStartMillis(),
- firstStatusStartMillis), intervalStartMillis))
- / DateTimeHelper.ONE_HOUR);
- statusDataPoints.add(statusHours * 5 > dataPointIntervalHours
- ? statusHours : -1);
- statusHours = -1;
- intervalStartMillis += dataPointInterval;
- }
- if (statusHours < 0) {
- statusHours = 0;
- }
- statusHours += (int) ((histEndMillis - Math.max(Math.max(
- hist.getStartMillis(), firstStatusStartMillis),
- intervalStartMillis)) / DateTimeHelper.ONE_HOUR);
- }
- if (statusHours > 0) {
- statusDataPoints.add(statusHours * 5 > dataPointIntervalHours
- ? statusHours : -1);
- }
- List<Double> dataPoints = new ArrayList<Double>();
- for (int dataPointIndex = 0; dataPointIndex < statusDataPoints.size();
- dataPointIndex++) {
- if (dataPointIndex >= uptimeDataPoints.size()) {
- dataPoints.add(0.0);
- } else if (uptimeDataPoints.get(dataPointIndex) >= 0 &&
- statusDataPoints.get(dataPointIndex) > 0) {
- dataPoints.add(((double) uptimeDataPoints.get(dataPointIndex))
- / ((double) statusDataPoints.get(dataPointIndex)));
- } else {
- dataPoints.add(-1.0);
- }
- }
- int firstNonNullIndex = -1, lastNonNullIndex = -1;
- for (int dataPointIndex = 0; dataPointIndex < dataPoints.size();
- dataPointIndex++) {
- double dataPoint = dataPoints.get(dataPointIndex);
- if (dataPoint >= 0.0) {
- if (firstNonNullIndex < 0) {
- firstNonNullIndex = dataPointIndex;
- }
- lastNonNullIndex = dataPointIndex;
- }
- }
- if (firstNonNullIndex < 0) {
- return null;
- }
- long firstDataPointMillis = (((this.now - graphInterval)
- / dataPointInterval) + firstNonNullIndex)
- * dataPointInterval + dataPointInterval / 2L;
- if (graphIntervalIndex > 0 && firstDataPointMillis >=
- this.now - graphIntervals[graphIntervalIndex - 1]) {
- /* Skip uptime history object, because it doesn't contain
- * anything new that wasn't already contained in the last
- * uptime history object(s). */
- return null;
- }
- long lastDataPointMillis = firstDataPointMillis
- + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval;
- int count = lastNonNullIndex - firstNonNullIndex + 1;
- GraphHistory graphHistory = new GraphHistory();
- graphHistory.setFirst(DateTimeHelper.format(firstDataPointMillis));
- graphHistory.setLast(DateTimeHelper.format(lastDataPointMillis));
- graphHistory.setInterval((int) (dataPointInterval
- / DateTimeHelper.ONE_SECOND));
- graphHistory.setFactor(1.0 / 999.0);
- graphHistory.setCount(count);
- int previousNonNullIndex = -2;
- boolean foundTwoAdjacentDataPoints = false;
- List<Integer> values = new ArrayList<Integer>();
- for (int dataPointIndex = firstNonNullIndex; dataPointIndex <=
- lastNonNullIndex; dataPointIndex++) {
- double dataPoint = dataPoints.get(dataPointIndex);
- if (dataPoint >= 0.0) {
- if (dataPointIndex - previousNonNullIndex == 1) {
- foundTwoAdjacentDataPoints = true;
- }
- previousNonNullIndex = dataPointIndex;
- }
- values.add(dataPoint < -0.5 ? null : ((int) (dataPoint * 999.0)));
- }
- graphHistory.setValues(values);
- if (foundTwoAdjacentDataPoints) {
- return graphHistory;
- } else {
- return null;
- }
- }
-
- public String getStatsString() {
- StringBuilder sb = new StringBuilder();
- sb.append(" " + Logger.formatDecimalNumber(this.writtenDocuments)
- + " uptime document files written\n");
- return sb.toString();
- }
-}
-
diff --git a/src/org/torproject/onionoo/UptimeStatus.java b/src/org/torproject/onionoo/UptimeStatus.java
deleted file mode 100644
index 92ca629..0000000
--- a/src/org/torproject/onionoo/UptimeStatus.java
+++ /dev/null
@@ -1,226 +0,0 @@
-/* Copyright 2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.Scanner;
-import java.util.SortedSet;
-import java.util.TreeSet;
-
-class UptimeHistory
- implements Comparable<UptimeHistory> {
-
- private boolean relay;
- public boolean isRelay() {
- return this.relay;
- }
-
- private long startMillis;
- public long getStartMillis() {
- return this.startMillis;
- }
-
- private int uptimeHours;
- public int getUptimeHours() {
- return this.uptimeHours;
- }
-
- UptimeHistory(boolean relay, long startMillis,
- int uptimeHours) {
- this.relay = relay;
- this.startMillis = startMillis;
- this.uptimeHours = uptimeHours;
- }
-
- public static UptimeHistory fromString(String uptimeHistoryString) {
- String[] parts = uptimeHistoryString.split(" ", 3);
- if (parts.length != 3) {
- return null;
- }
- boolean relay = false;
- if (parts[0].equals("r")) {
- relay = true;
- } else if (!parts[0].equals("b")) {
- return null;
- }
- long startMillis = DateTimeHelper.parse(parts[1],
- DateTimeHelper.DATEHOUR_NOSPACE_FORMAT);
- if (startMillis < 0L) {
- return null;
- }
- int uptimeHours = -1;
- try {
- uptimeHours = Integer.parseInt(parts[2]);
- } catch (NumberFormatException e) {
- return null;
- }
- return new UptimeHistory(relay, startMillis, uptimeHours);
- }
-
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append(this.relay ? "r" : "b");
- sb.append(" " + DateTimeHelper.format(this.startMillis,
- DateTimeHelper.DATEHOUR_NOSPACE_FORMAT));
- sb.append(" " + String.format("%d", this.uptimeHours));
- return sb.toString();
- }
-
- public void addUptime(UptimeHistory other) {
- this.uptimeHours += other.uptimeHours;
- if (this.startMillis > other.startMillis) {
- this.startMillis = other.startMillis;
- }
- }
-
- public int compareTo(UptimeHistory other) {
- if (this.relay && !other.relay) {
- return -1;
- } else if (!this.relay && other.relay) {
- return 1;
- }
- return this.startMillis < other.startMillis ? -1 :
- this.startMillis > other.startMillis ? 1 : 0;
- }
-
- public boolean equals(Object other) {
- return other instanceof UptimeHistory &&
- this.relay == ((UptimeHistory) other).relay &&
- this.startMillis == ((UptimeHistory) other).startMillis;
- }
-
- public int hashCode() {
- return (int) this.startMillis + (this.relay ? 1 : 0);
- }
-}
-
-public class UptimeStatus extends Document {
-
- private transient String fingerprint;
-
- private transient boolean isDirty = false;
-
- private SortedSet<UptimeHistory> relayHistory =
- new TreeSet<UptimeHistory>();
- public void setRelayHistory(SortedSet<UptimeHistory> relayHistory) {
- this.relayHistory = relayHistory;
- }
- public SortedSet<UptimeHistory> getRelayHistory() {
- return this.relayHistory;
- }
-
- private SortedSet<UptimeHistory> bridgeHistory =
- new TreeSet<UptimeHistory>();
- public void setBridgeHistory(SortedSet<UptimeHistory> bridgeHistory) {
- this.bridgeHistory = bridgeHistory;
- }
- public SortedSet<UptimeHistory> getBridgeHistory() {
- return this.bridgeHistory;
- }
-
- public static UptimeStatus loadOrCreate(String fingerprint) {
- UptimeStatus uptimeStatus = (fingerprint == null) ?
- ApplicationFactory.getDocumentStore().retrieve(
- UptimeStatus.class, true) :
- ApplicationFactory.getDocumentStore().retrieve(
- UptimeStatus.class, true, fingerprint);
- if (uptimeStatus == null) {
- uptimeStatus = new UptimeStatus();
- }
- uptimeStatus.fingerprint = fingerprint;
- return uptimeStatus;
- }
-
- public void fromDocumentString(String documentString) {
- Scanner s = new Scanner(documentString);
- while (s.hasNextLine()) {
- String line = s.nextLine();
- UptimeHistory parsedLine = UptimeHistory.fromString(line);
- if (parsedLine != null) {
- if (parsedLine.isRelay()) {
- this.relayHistory.add(parsedLine);
- } else {
- this.bridgeHistory.add(parsedLine);
- }
- } else {
- System.err.println("Could not parse uptime history line '"
- + line + "'. Skipping.");
- }
- }
- s.close();
- }
-
- public void addToHistory(boolean relay, SortedSet<Long> newIntervals) {
- for (long startMillis : newIntervals) {
- SortedSet<UptimeHistory> history = relay ? this.relayHistory
- : this.bridgeHistory;
- UptimeHistory interval = new UptimeHistory(relay, startMillis, 1);
- if (!history.headSet(interval).isEmpty()) {
- UptimeHistory prev = history.headSet(interval).last();
- if (prev.isRelay() == interval.isRelay() &&
- prev.getStartMillis() + DateTimeHelper.ONE_HOUR
- * prev.getUptimeHours() > interval.getStartMillis()) {
- continue;
- }
- }
- if (!history.tailSet(interval).isEmpty()) {
- UptimeHistory next = history.tailSet(interval).first();
- if (next.isRelay() == interval.isRelay() &&
- next.getStartMillis() < interval.getStartMillis()
- + DateTimeHelper.ONE_HOUR) {
- continue;
- }
- }
- history.add(interval);
- this.isDirty = true;
- }
- }
-
- public void storeIfChanged() {
- if (this.isDirty) {
- this.compressHistory(this.relayHistory);
- this.compressHistory(this.bridgeHistory);
- if (fingerprint == null) {
- ApplicationFactory.getDocumentStore().store(this);
- } else {
- ApplicationFactory.getDocumentStore().store(this,
- this.fingerprint);
- }
- this.isDirty = false;
- }
- }
-
- private void compressHistory(SortedSet<UptimeHistory> history) {
- SortedSet<UptimeHistory> uncompressedHistory =
- new TreeSet<UptimeHistory>(history);
- history.clear();
- UptimeHistory lastInterval = null;
- for (UptimeHistory interval : uncompressedHistory) {
- if (lastInterval != null &&
- lastInterval.getStartMillis() + DateTimeHelper.ONE_HOUR
- * lastInterval.getUptimeHours() == interval.getStartMillis() &&
- lastInterval.isRelay() == interval.isRelay()) {
- lastInterval.addUptime(interval);
- } else {
- if (lastInterval != null) {
- history.add(lastInterval);
- }
- lastInterval = interval;
- }
- }
- if (lastInterval != null) {
- history.add(lastInterval);
- }
- }
-
- public String toDocumentString() {
- StringBuilder sb = new StringBuilder();
- for (UptimeHistory interval : this.relayHistory) {
- sb.append(interval.toString() + "\n");
- }
- for (UptimeHistory interval : this.bridgeHistory) {
- sb.append(interval.toString() + "\n");
- }
- return sb.toString();
- }
-}
-
diff --git a/src/org/torproject/onionoo/UptimeStatusUpdater.java b/src/org/torproject/onionoo/UptimeStatusUpdater.java
deleted file mode 100644
index eccc2f2..0000000
--- a/src/org/torproject/onionoo/UptimeStatusUpdater.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/* Copyright 2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.torproject.descriptor.BridgeNetworkStatus;
-import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.NetworkStatusEntry;
-import org.torproject.descriptor.RelayNetworkStatusConsensus;
-
-public class UptimeStatusUpdater implements DescriptorListener,
- StatusUpdater {
-
- private DescriptorSource descriptorSource;
-
- public UptimeStatusUpdater() {
- this.descriptorSource = ApplicationFactory.getDescriptorSource();
- this.registerDescriptorListeners();
- }
-
- private void registerDescriptorListeners() {
- this.descriptorSource.registerDescriptorListener(this,
- DescriptorType.RELAY_CONSENSUSES);
- this.descriptorSource.registerDescriptorListener(this,
- DescriptorType.BRIDGE_STATUSES);
- }
-
- public void processDescriptor(Descriptor descriptor, boolean relay) {
- if (descriptor instanceof RelayNetworkStatusConsensus) {
- this.processRelayNetworkStatusConsensus(
- (RelayNetworkStatusConsensus) descriptor);
- } else if (descriptor instanceof BridgeNetworkStatus) {
- this.processBridgeNetworkStatus(
- (BridgeNetworkStatus) descriptor);
- }
- }
-
- private SortedSet<Long> newRelayStatuses = new TreeSet<Long>(),
- newBridgeStatuses = new TreeSet<Long>();
- private SortedMap<String, SortedSet<Long>>
- newRunningRelays = new TreeMap<String, SortedSet<Long>>(),
- newRunningBridges = new TreeMap<String, SortedSet<Long>>();
-
- private void processRelayNetworkStatusConsensus(
- RelayNetworkStatusConsensus consensus) {
- SortedSet<String> fingerprints = new TreeSet<String>();
- for (NetworkStatusEntry entry :
- consensus.getStatusEntries().values()) {
- if (entry.getFlags().contains("Running")) {
- fingerprints.add(entry.getFingerprint());
- }
- }
- if (!fingerprints.isEmpty()) {
- long dateHourMillis = (consensus.getValidAfterMillis()
- / DateTimeHelper.ONE_HOUR) * DateTimeHelper.ONE_HOUR;
- for (String fingerprint : fingerprints) {
- if (!this.newRunningRelays.containsKey(fingerprint)) {
- this.newRunningRelays.put(fingerprint, new TreeSet<Long>());
- }
- this.newRunningRelays.get(fingerprint).add(dateHourMillis);
- }
- this.newRelayStatuses.add(dateHourMillis);
- }
- }
-
- private void processBridgeNetworkStatus(BridgeNetworkStatus status) {
- SortedSet<String> fingerprints = new TreeSet<String>();
- for (NetworkStatusEntry entry :
- status.getStatusEntries().values()) {
- if (entry.getFlags().contains("Running")) {
- fingerprints.add(entry.getFingerprint());
- }
- }
- if (!fingerprints.isEmpty()) {
- long dateHourMillis = (status.getPublishedMillis()
- / DateTimeHelper.ONE_HOUR) * DateTimeHelper.ONE_HOUR;
- for (String fingerprint : fingerprints) {
- if (!this.newRunningBridges.containsKey(fingerprint)) {
- this.newRunningBridges.put(fingerprint, new TreeSet<Long>());
- }
- this.newRunningBridges.get(fingerprint).add(dateHourMillis);
- }
- this.newBridgeStatuses.add(dateHourMillis);
- }
- }
-
- public void updateStatuses() {
- for (Map.Entry<String, SortedSet<Long>> e :
- this.newRunningRelays.entrySet()) {
- this.updateStatus(true, e.getKey(), e.getValue());
- }
- this.updateStatus(true, null, this.newRelayStatuses);
- for (Map.Entry<String, SortedSet<Long>> e :
- this.newRunningBridges.entrySet()) {
- this.updateStatus(false, e.getKey(), e.getValue());
- }
- this.updateStatus(false, null, this.newBridgeStatuses);
- }
-
- private void updateStatus(boolean relay, String fingerprint,
- SortedSet<Long> newUptimeHours) {
- UptimeStatus uptimeStatus = UptimeStatus.loadOrCreate(fingerprint);
- uptimeStatus.addToHistory(relay, newUptimeHours);
- uptimeStatus.storeIfChanged();
- }
-
- public String getStatsString() {
- StringBuilder sb = new StringBuilder();
- sb.append(" " + Logger.formatDecimalNumber(
- this.newRelayStatuses.size()) + " hours of relay uptimes "
- + "processed\n");
- sb.append(" " + Logger.formatDecimalNumber(
- this.newBridgeStatuses.size()) + " hours of bridge uptimes "
- + "processed\n");
- sb.append(" " + Logger.formatDecimalNumber(
- this.newRunningRelays.size() + this.newRunningBridges.size())
- + " uptime status files updated\n");
- return sb.toString();
- }
-}
-
diff --git a/src/org/torproject/onionoo/WeightsDocument.java b/src/org/torproject/onionoo/WeightsDocument.java
deleted file mode 100644
index 4cd0021..0000000
--- a/src/org/torproject/onionoo/WeightsDocument.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/* Copyright 2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.Map;
-
-public class WeightsDocument extends Document {
-
- @SuppressWarnings("unused")
- private String fingerprint;
- public void setFingerprint(String fingerprint) {
- this.fingerprint = fingerprint;
- }
-
- @SuppressWarnings("unused")
- private Map<String, GraphHistory> advertised_bandwidth_fraction;
- public void setAdvertisedBandwidthFraction(
- Map<String, GraphHistory> advertisedBandwidthFraction) {
- this.advertised_bandwidth_fraction = advertisedBandwidthFraction;
- }
-
- @SuppressWarnings("unused")
- private Map<String, GraphHistory> consensus_weight_fraction;
- public void setConsensusWeightFraction(
- Map<String, GraphHistory> consensusWeightFraction) {
- this.consensus_weight_fraction = consensusWeightFraction;
- }
-
- @SuppressWarnings("unused")
- private Map<String, GraphHistory> guard_probability;
- public void setGuardProbability(
- Map<String, GraphHistory> guardProbability) {
- this.guard_probability = guardProbability;
- }
-
- @SuppressWarnings("unused")
- private Map<String, GraphHistory> middle_probability;
- public void setMiddleProbability(
- Map<String, GraphHistory> middleProbability) {
- this.middle_probability = middleProbability;
- }
-
- @SuppressWarnings("unused")
- private Map<String, GraphHistory> exit_probability;
- public void setExitProbability(
- Map<String, GraphHistory> exitProbability) {
- this.exit_probability = exitProbability;
- }
-
- @SuppressWarnings("unused")
- private Map<String, GraphHistory> advertised_bandwidth;
- public void setAdvertisedBandwidth(
- Map<String, GraphHistory> advertisedBandwidth) {
- this.advertised_bandwidth = advertisedBandwidth;
- }
-
- @SuppressWarnings("unused")
- private Map<String, GraphHistory> consensus_weight;
- public void setConsensusWeight(
- Map<String, GraphHistory> consensusWeight) {
- this.consensus_weight = consensusWeight;
- }
-}
-
diff --git a/src/org/torproject/onionoo/WeightsDocumentWriter.java b/src/org/torproject/onionoo/WeightsDocumentWriter.java
deleted file mode 100644
index 2e0d465..0000000
--- a/src/org/torproject/onionoo/WeightsDocumentWriter.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/* Copyright 2012--2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.SortedSet;
-
-public class WeightsDocumentWriter implements FingerprintListener,
- DocumentWriter {
-
- private DescriptorSource descriptorSource;
-
- private DocumentStore documentStore;
-
- private long now;
-
- public WeightsDocumentWriter() {
- this.descriptorSource = ApplicationFactory.getDescriptorSource();
- this.documentStore = ApplicationFactory.getDocumentStore();
- this.now = ApplicationFactory.getTime().currentTimeMillis();
- this.registerFingerprintListeners();
- }
-
- private void registerFingerprintListeners() {
- this.descriptorSource.registerFingerprintListener(this,
- DescriptorType.RELAY_CONSENSUSES);
- this.descriptorSource.registerFingerprintListener(this,
- DescriptorType.RELAY_SERVER_DESCRIPTORS);
- }
-
- private Set<String> updateWeightsDocuments = new HashSet<String>();
-
- public void processFingerprints(SortedSet<String> fingerprints,
- boolean relay) {
- if (relay) {
- this.updateWeightsDocuments.addAll(fingerprints);
- }
- }
-
- public void writeDocuments() {
- this.writeWeightsDataFiles();
- Logger.printStatusTime("Wrote weights document files");
- }
-
- private void writeWeightsDataFiles() {
- for (String fingerprint : this.updateWeightsDocuments) {
- WeightsStatus weightsStatus = this.documentStore.retrieve(
- WeightsStatus.class, true, fingerprint);
- if (weightsStatus == null) {
- continue;
- }
- SortedMap<long[], double[]> history = weightsStatus.getHistory();
- WeightsDocument weightsDocument = this.compileWeightsDocument(
- fingerprint, history);
- this.documentStore.store(weightsDocument, fingerprint);
- }
- Logger.printStatusTime("Wrote weights document files");
- }
-
- private String[] graphNames = new String[] {
- "1_week",
- "1_month",
- "3_months",
- "1_year",
- "5_years" };
-
- private long[] graphIntervals = new long[] {
- DateTimeHelper.ONE_WEEK,
- DateTimeHelper.ROUGHLY_ONE_MONTH,
- DateTimeHelper.ROUGHLY_THREE_MONTHS,
- DateTimeHelper.ROUGHLY_ONE_YEAR,
- DateTimeHelper.ROUGHLY_FIVE_YEARS };
-
- private long[] dataPointIntervals = new long[] {
- DateTimeHelper.ONE_HOUR,
- DateTimeHelper.FOUR_HOURS,
- DateTimeHelper.TWELVE_HOURS,
- DateTimeHelper.TWO_DAYS,
- DateTimeHelper.TEN_DAYS };
-
- private WeightsDocument compileWeightsDocument(String fingerprint,
- SortedMap<long[], double[]> history) {
- WeightsDocument weightsDocument = new WeightsDocument();
- weightsDocument.setFingerprint(fingerprint);
- weightsDocument.setAdvertisedBandwidthFraction(
- this.compileGraphType(history, 0));
- weightsDocument.setConsensusWeightFraction(
- this.compileGraphType(history, 1));
- weightsDocument.setGuardProbability(
- this.compileGraphType(history, 2));
- weightsDocument.setMiddleProbability(
- this.compileGraphType(history, 3));
- weightsDocument.setExitProbability(
- this.compileGraphType(history, 4));
- weightsDocument.setAdvertisedBandwidth(
- this.compileGraphType(history, 5));
- weightsDocument.setConsensusWeight(
- this.compileGraphType(history, 6));
- return weightsDocument;
- }
-
- private Map<String, GraphHistory> compileGraphType(
- SortedMap<long[], double[]> history, int graphTypeIndex) {
- Map<String, GraphHistory> graphs =
- new LinkedHashMap<String, GraphHistory>();
- for (int graphIntervalIndex = 0; graphIntervalIndex <
- this.graphIntervals.length; graphIntervalIndex++) {
- String graphName = this.graphNames[graphIntervalIndex];
- GraphHistory graphHistory = this.compileWeightsHistory(
- graphTypeIndex, graphIntervalIndex, history);
- if (graphHistory != null) {
- graphs.put(graphName, graphHistory);
- }
- }
- return graphs;
- }
-
- private GraphHistory compileWeightsHistory(int graphTypeIndex,
- int graphIntervalIndex, SortedMap<long[], double[]> history) {
- long graphInterval = this.graphIntervals[graphIntervalIndex];
- long dataPointInterval =
- this.dataPointIntervals[graphIntervalIndex];
- List<Double> dataPoints = new ArrayList<Double>();
- long intervalStartMillis = ((this.now - graphInterval)
- / dataPointInterval) * dataPointInterval;
- long totalMillis = 0L;
- double totalWeightTimesMillis = 0.0;
- for (Map.Entry<long[], double[]> e : history.entrySet()) {
- long startMillis = e.getKey()[0], endMillis = e.getKey()[1];
- double weight = e.getValue()[graphTypeIndex];
- if (endMillis < intervalStartMillis) {
- continue;
- }
- while ((intervalStartMillis / dataPointInterval) !=
- (endMillis / dataPointInterval)) {
- dataPoints.add(totalMillis * 5L < dataPointInterval
- ? -1.0 : totalWeightTimesMillis / (double) totalMillis);
- totalWeightTimesMillis = 0.0;
- totalMillis = 0L;
- intervalStartMillis += dataPointInterval;
- }
- if (weight >= 0.0) {
- totalWeightTimesMillis += weight
- * ((double) (endMillis - startMillis));
- totalMillis += (endMillis - startMillis);
- }
- }
- dataPoints.add(totalMillis * 5L < dataPointInterval
- ? -1.0 : totalWeightTimesMillis / (double) totalMillis);
- double maxValue = 0.0;
- int firstNonNullIndex = -1, lastNonNullIndex = -1;
- for (int dataPointIndex = 0; dataPointIndex < dataPoints.size();
- dataPointIndex++) {
- double dataPoint = dataPoints.get(dataPointIndex);
- if (dataPoint >= 0.0) {
- if (firstNonNullIndex < 0) {
- firstNonNullIndex = dataPointIndex;
- }
- lastNonNullIndex = dataPointIndex;
- if (dataPoint > maxValue) {
- maxValue = dataPoint;
- }
- }
- }
- if (firstNonNullIndex < 0) {
- return null;
- }
- long firstDataPointMillis = (((this.now - graphInterval)
- / dataPointInterval) + firstNonNullIndex) * dataPointInterval
- + dataPointInterval / 2L;
- if (graphIntervalIndex > 0 && firstDataPointMillis >=
- this.now - graphIntervals[graphIntervalIndex - 1]) {
- /* Skip weights history object, because it doesn't contain
- * anything new that wasn't already contained in the last
- * weights history object(s). */
- return null;
- }
- long lastDataPointMillis = firstDataPointMillis
- + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval;
- double factor = ((double) maxValue) / 999.0;
- int count = lastNonNullIndex - firstNonNullIndex + 1;
- GraphHistory graphHistory = new GraphHistory();
- graphHistory.setFirst(DateTimeHelper.format(firstDataPointMillis));
- graphHistory.setLast(DateTimeHelper.format(lastDataPointMillis));
- graphHistory.setInterval((int) (dataPointInterval
- / DateTimeHelper.ONE_SECOND));
- graphHistory.setFactor(factor);
- graphHistory.setCount(count);
- int previousNonNullIndex = -2;
- boolean foundTwoAdjacentDataPoints = false;
- List<Integer> values = new ArrayList<Integer>();
- for (int dataPointIndex = firstNonNullIndex; dataPointIndex <=
- lastNonNullIndex; dataPointIndex++) {
- double dataPoint = dataPoints.get(dataPointIndex);
- if (dataPoint >= 0.0) {
- if (dataPointIndex - previousNonNullIndex == 1) {
- foundTwoAdjacentDataPoints = true;
- }
- previousNonNullIndex = dataPointIndex;
- }
- values.add(dataPoint < 0.0 ? null :
- (int) ((dataPoint * 999.0) / maxValue));
- }
- graphHistory.setValues(values);
- if (foundTwoAdjacentDataPoints) {
- return graphHistory;
- } else {
- return null;
- }
- }
-
- public String getStatsString() {
- /* TODO Add statistics string. */
- return null;
- }
-}
diff --git a/src/org/torproject/onionoo/WeightsStatus.java b/src/org/torproject/onionoo/WeightsStatus.java
deleted file mode 100644
index 6d06ca4..0000000
--- a/src/org/torproject/onionoo/WeightsStatus.java
+++ /dev/null
@@ -1,97 +0,0 @@
-package org.torproject.onionoo;
-
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Scanner;
-import java.util.SortedMap;
-import java.util.TreeMap;
-
-public class WeightsStatus extends Document {
-
- private SortedMap<long[], double[]> history =
- new TreeMap<long[], double[]>(new Comparator<long[]>() {
- public int compare(long[] a, long[] b) {
- return a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0;
- }
- });
- public void setHistory(SortedMap<long[], double[]> history) {
- this.history = history;
- }
- public SortedMap<long[], double[]> getHistory() {
- return this.history;
- }
-
- private Map<String, Integer> advertisedBandwidths =
- new HashMap<String, Integer>();
- public Map<String, Integer> getAdvertisedBandwidths() {
- return this.advertisedBandwidths;
- }
-
- public void fromDocumentString(String documentString) {
- Scanner s = new Scanner(documentString);
- while (s.hasNextLine()) {
- String line = s.nextLine();
- String[] parts = line.split(" ");
- if (parts.length == 2) {
- String descriptorDigest = parts[0];
- int advertisedBandwidth = Integer.parseInt(parts[1]);
- this.advertisedBandwidths.put(descriptorDigest,
- advertisedBandwidth);
- continue;
- }
- if (parts.length != 9 && parts.length != 11) {
- System.err.println("Illegal line '" + line + "' in weights "
- + "status file. Skipping this line.");
- continue;
- }
- if (parts[4].equals("NaN")) {
- /* Remove corrupt lines written on 2013-07-07 and the days
- * after. */
- continue;
- }
- long validAfterMillis = DateTimeHelper.parse(parts[0] + " "
- + parts[1]);
- long freshUntilMillis = DateTimeHelper.parse(parts[2] + " "
- + parts[3]);
- if (validAfterMillis < 0L || freshUntilMillis < 0L) {
- System.err.println("Could not parse timestamp while reading "
- + "weights status file. Skipping.");
- break;
- }
- long[] interval = new long[] { validAfterMillis, freshUntilMillis };
- double[] weights = new double[] {
- Double.parseDouble(parts[4]),
- Double.parseDouble(parts[5]),
- Double.parseDouble(parts[6]),
- Double.parseDouble(parts[7]),
- Double.parseDouble(parts[8]), -1.0, -1.0 };
- if (parts.length == 11) {
- weights[5] = Double.parseDouble(parts[9]);
- weights[6] = Double.parseDouble(parts[10]);
- }
- this.history.put(interval, weights);
- }
- s.close();
- }
-
- public String toDocumentString() {
- StringBuilder sb = new StringBuilder();
- for (Map.Entry<String, Integer> e :
- this.advertisedBandwidths.entrySet()) {
- sb.append(e.getKey() + " " + String.valueOf(e.getValue()) + "\n");
- }
- for (Map.Entry<long[], double[]> e : history.entrySet()) {
- long[] fresh = e.getKey();
- double[] weights = e.getValue();
- sb.append(DateTimeHelper.format(fresh[0]) + " "
- + DateTimeHelper.format(fresh[1]));
- for (double weight : weights) {
- sb.append(String.format(" %.12f", weight));
- }
- sb.append("\n");
- }
- return sb.toString();
- }
-}
-
diff --git a/src/org/torproject/onionoo/WeightsStatusUpdater.java b/src/org/torproject/onionoo/WeightsStatusUpdater.java
deleted file mode 100644
index 5932da6..0000000
--- a/src/org/torproject/onionoo/WeightsStatusUpdater.java
+++ /dev/null
@@ -1,328 +0,0 @@
-/* Copyright 2012--2014 The Tor Project
- * See LICENSE for licensing information */
-package org.torproject.onionoo;
-
-import java.util.Arrays;
-import java.util.Map;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.NetworkStatusEntry;
-import org.torproject.descriptor.RelayNetworkStatusConsensus;
-import org.torproject.descriptor.ServerDescriptor;
-
-public class WeightsStatusUpdater implements DescriptorListener,
- StatusUpdater {
-
- private DescriptorSource descriptorSource;
-
- private DocumentStore documentStore;
-
- private long now;
-
- public WeightsStatusUpdater() {
- this.descriptorSource = ApplicationFactory.getDescriptorSource();
- this.documentStore = ApplicationFactory.getDocumentStore();
- this.now = ApplicationFactory.getTime().currentTimeMillis();
- this.registerDescriptorListeners();
- }
-
- private void registerDescriptorListeners() {
- this.descriptorSource.registerDescriptorListener(this,
- DescriptorType.RELAY_CONSENSUSES);
- this.descriptorSource.registerDescriptorListener(this,
- DescriptorType.RELAY_SERVER_DESCRIPTORS);
- }
-
- public void processDescriptor(Descriptor descriptor, boolean relay) {
- if (descriptor instanceof ServerDescriptor) {
- this.processRelayServerDescriptor((ServerDescriptor) descriptor);
- } else if (descriptor instanceof RelayNetworkStatusConsensus) {
- this.processRelayNetworkConsensus(
- (RelayNetworkStatusConsensus) descriptor);
- }
- }
-
- public void updateStatuses() {
- /* Nothing to do. */
- }
-
- private void processRelayNetworkConsensus(
- RelayNetworkStatusConsensus consensus) {
- long validAfterMillis = consensus.getValidAfterMillis(),
- freshUntilMillis = consensus.getFreshUntilMillis();
- SortedMap<String, double[]> pathSelectionWeights =
- this.calculatePathSelectionProbabilities(consensus);
- this.updateWeightsHistory(validAfterMillis, freshUntilMillis,
- pathSelectionWeights);
- }
-
- private void processRelayServerDescriptor(
- ServerDescriptor serverDescriptor) {
- String digest = serverDescriptor.getServerDescriptorDigest().
- toUpperCase();
- int advertisedBandwidth = Math.min(Math.min(
- serverDescriptor.getBandwidthBurst(),
- serverDescriptor.getBandwidthObserved()),
- serverDescriptor.getBandwidthRate());
- String fingerprint = serverDescriptor.getFingerprint();
- WeightsStatus weightsStatus = this.documentStore.retrieve(
- WeightsStatus.class, true, fingerprint);
- if (weightsStatus == null) {
- weightsStatus = new WeightsStatus();
- }
- weightsStatus.getAdvertisedBandwidths().put(digest,
- advertisedBandwidth);
- this.documentStore.store(weightsStatus, fingerprint);
-}
-
- private void updateWeightsHistory(long validAfterMillis,
- long freshUntilMillis,
- SortedMap<String, double[]> pathSelectionWeights) {
- String fingerprint = null;
- double[] weights = null;
- do {
- fingerprint = null;
- synchronized (pathSelectionWeights) {
- if (!pathSelectionWeights.isEmpty()) {
- fingerprint = pathSelectionWeights.firstKey();
- weights = pathSelectionWeights.remove(fingerprint);
- }
- }
- if (fingerprint != null) {
- this.addToHistory(fingerprint, validAfterMillis,
- freshUntilMillis, weights);
- }
- } while (fingerprint != null);
- }
-
- private SortedMap<String, double[]> calculatePathSelectionProbabilities(
- RelayNetworkStatusConsensus consensus) {
- boolean containsBandwidthWeights = false;
- double wgg = 1.0, wgd = 1.0, wmg = 1.0, wmm = 1.0, wme = 1.0,
- wmd = 1.0, wee = 1.0, wed = 1.0;
- SortedMap<String, Integer> bandwidthWeights =
- consensus.getBandwidthWeights();
- if (bandwidthWeights != null) {
- SortedSet<String> missingWeightKeys = new TreeSet<String>(
- Arrays.asList("Wgg,Wgd,Wmg,Wmm,Wme,Wmd,Wee,Wed".split(",")));
- missingWeightKeys.removeAll(bandwidthWeights.keySet());
- if (missingWeightKeys.isEmpty()) {
- wgg = ((double) bandwidthWeights.get("Wgg")) / 10000.0;
- wgd = ((double) bandwidthWeights.get("Wgd")) / 10000.0;
- wmg = ((double) bandwidthWeights.get("Wmg")) / 10000.0;
- wmm = ((double) bandwidthWeights.get("Wmm")) / 10000.0;
- wme = ((double) bandwidthWeights.get("Wme")) / 10000.0;
- wmd = ((double) bandwidthWeights.get("Wmd")) / 10000.0;
- wee = ((double) bandwidthWeights.get("Wee")) / 10000.0;
- wed = ((double) bandwidthWeights.get("Wed")) / 10000.0;
- containsBandwidthWeights = true;
- }
- }
- SortedMap<String, Double>
- advertisedBandwidths = new TreeMap<String, Double>(),
- consensusWeights = new TreeMap<String, Double>(),
- guardWeights = new TreeMap<String, Double>(),
- middleWeights = new TreeMap<String, Double>(),
- exitWeights = new TreeMap<String, Double>();
- double totalAdvertisedBandwidth = 0.0;
- double totalConsensusWeight = 0.0;
- double totalGuardWeight = 0.0;
- double totalMiddleWeight = 0.0;
- double totalExitWeight = 0.0;
- for (NetworkStatusEntry relay :
- consensus.getStatusEntries().values()) {
- String fingerprint = relay.getFingerprint();
- if (!relay.getFlags().contains("Running")) {
- continue;
- }
- String digest = relay.getDescriptor().toUpperCase();
- WeightsStatus weightsStatus = this.documentStore.retrieve(
- WeightsStatus.class, true, fingerprint);
- if (weightsStatus != null &&
- weightsStatus.getAdvertisedBandwidths() != null &&
- weightsStatus.getAdvertisedBandwidths().containsKey(digest)) {
- /* Read advertised bandwidth from weights status file. Server
- * descriptors are parsed before consensuses, so we're sure that
- * if there's a server descriptor for this relay, it'll be
- * contained in the weights status file by now. */
- double advertisedBandwidth =
- (double) weightsStatus.getAdvertisedBandwidths().get(digest);
- advertisedBandwidths.put(fingerprint, advertisedBandwidth);
- totalAdvertisedBandwidth += advertisedBandwidth;
- }
- if (relay.getBandwidth() >= 0L) {
- double consensusWeight = (double) relay.getBandwidth();
- consensusWeights.put(fingerprint, consensusWeight);
- totalConsensusWeight += consensusWeight;
- if (containsBandwidthWeights) {
- double guardWeight = (double) relay.getBandwidth();
- double middleWeight = (double) relay.getBandwidth();
- double exitWeight = (double) relay.getBandwidth();
- boolean isExit = relay.getFlags().contains("Exit") &&
- !relay.getFlags().contains("BadExit");
- boolean isGuard = relay.getFlags().contains("Guard");
- if (isGuard && isExit) {
- guardWeight *= wgd;
- middleWeight *= wmd;
- exitWeight *= wed;
- } else if (isGuard) {
- guardWeight *= wgg;
- middleWeight *= wmg;
- exitWeight = 0.0;
- } else if (isExit) {
- guardWeight = 0.0;
- middleWeight *= wme;
- exitWeight *= wee;
- } else {
- guardWeight = 0.0;
- middleWeight *= wmm;
- exitWeight = 0.0;
- }
- guardWeights.put(fingerprint, guardWeight);
- middleWeights.put(fingerprint, middleWeight);
- exitWeights.put(fingerprint, exitWeight);
- totalGuardWeight += guardWeight;
- totalMiddleWeight += middleWeight;
- totalExitWeight += exitWeight;
- }
- }
- }
- SortedMap<String, double[]> pathSelectionProbabilities =
- new TreeMap<String, double[]>();
- SortedSet<String> fingerprints = new TreeSet<String>();
- fingerprints.addAll(consensusWeights.keySet());
- fingerprints.addAll(advertisedBandwidths.keySet());
- for (String fingerprint : fingerprints) {
- double[] probabilities = new double[] { -1.0, -1.0, -1.0, -1.0,
- -1.0, -1.0, -1.0 };
- if (consensusWeights.containsKey(fingerprint) &&
- totalConsensusWeight > 0.0) {
- probabilities[1] = consensusWeights.get(fingerprint) /
- totalConsensusWeight;
- probabilities[6] = consensusWeights.get(fingerprint);
- }
- if (guardWeights.containsKey(fingerprint) &&
- totalGuardWeight > 0.0) {
- probabilities[2] = guardWeights.get(fingerprint) /
- totalGuardWeight;
- }
- if (middleWeights.containsKey(fingerprint) &&
- totalMiddleWeight > 0.0) {
- probabilities[3] = middleWeights.get(fingerprint) /
- totalMiddleWeight;
- }
- if (exitWeights.containsKey(fingerprint) &&
- totalExitWeight > 0.0) {
- probabilities[4] = exitWeights.get(fingerprint) /
- totalExitWeight;
- }
- if (advertisedBandwidths.containsKey(fingerprint) &&
- totalAdvertisedBandwidth > 0.0) {
- probabilities[0] = advertisedBandwidths.get(fingerprint)
- / totalAdvertisedBandwidth;
- probabilities[5] = advertisedBandwidths.get(fingerprint);
- }
- pathSelectionProbabilities.put(fingerprint, probabilities);
- }
- return pathSelectionProbabilities;
- }
-
- private void addToHistory(String fingerprint, long validAfterMillis,
- long freshUntilMillis, double[] weights) {
- WeightsStatus weightsStatus = this.documentStore.retrieve(
- WeightsStatus.class, true, fingerprint);
- if (weightsStatus == null) {
- weightsStatus = new WeightsStatus();
- }
- SortedMap<long[], double[]> history = weightsStatus.getHistory();
- long[] interval = new long[] { validAfterMillis, freshUntilMillis };
- if ((history.headMap(interval).isEmpty() ||
- history.headMap(interval).lastKey()[1] <= validAfterMillis) &&
- (history.tailMap(interval).isEmpty() ||
- history.tailMap(interval).firstKey()[0] >= freshUntilMillis)) {
- history.put(interval, weights);
- this.compressHistory(weightsStatus);
- this.documentStore.store(weightsStatus, fingerprint);
- }
- }
-
- private void compressHistory(WeightsStatus weightsStatus) {
- SortedMap<long[], double[]> history = weightsStatus.getHistory();
- SortedMap<long[], double[]> compressedHistory =
- new TreeMap<long[], double[]>(history.comparator());
- long lastStartMillis = 0L, lastEndMillis = 0L;
- double[] lastWeights = null;
- String lastMonthString = "1970-01";
- int lastMissingValues = -1;
- for (Map.Entry<long[], double[]> e : history.entrySet()) {
- long startMillis = e.getKey()[0], endMillis = e.getKey()[1];
- double[] weights = e.getValue();
- long intervalLengthMillis;
- if (this.now - endMillis <= DateTimeHelper.ONE_WEEK) {
- intervalLengthMillis = DateTimeHelper.ONE_HOUR;
- } else if (this.now - endMillis <=
- DateTimeHelper.ROUGHLY_ONE_MONTH) {
- intervalLengthMillis = DateTimeHelper.FOUR_HOURS;
- } else if (this.now - endMillis <=
- DateTimeHelper.ROUGHLY_THREE_MONTHS) {
- intervalLengthMillis = DateTimeHelper.TWELVE_HOURS;
- } else if (this.now - endMillis <=
- DateTimeHelper.ROUGHLY_ONE_YEAR) {
- intervalLengthMillis = DateTimeHelper.TWO_DAYS;
- } else {
- intervalLengthMillis = DateTimeHelper.TEN_DAYS;
- }
- String monthString = DateTimeHelper.format(startMillis,
- DateTimeHelper.ISO_YEARMONTH_FORMAT);
- int missingValues = 0;
- for (int i = 0; i < weights.length; i++) {
- if (weights[i] < -0.5) {
- missingValues += 1 << i;
- }
- }
- if (lastEndMillis == startMillis &&
- ((lastEndMillis - 1L) / intervalLengthMillis) ==
- ((endMillis - 1L) / intervalLengthMillis) &&
- lastMonthString.equals(monthString) &&
- lastMissingValues == missingValues) {
- double lastIntervalInHours = (double) ((lastEndMillis
- - lastStartMillis) / DateTimeHelper.ONE_HOUR);
- double currentIntervalInHours = (double) ((endMillis
- - startMillis) / DateTimeHelper.ONE_HOUR);
- double newIntervalInHours = (double) ((endMillis
- - lastStartMillis) / DateTimeHelper.ONE_HOUR);
- for (int i = 0; i < lastWeights.length; i++) {
- lastWeights[i] *= lastIntervalInHours;
- lastWeights[i] += weights[i] * currentIntervalInHours;
- lastWeights[i] /= newIntervalInHours;
- }
- lastEndMillis = endMillis;
- } else {
- if (lastStartMillis > 0L) {
- compressedHistory.put(new long[] { lastStartMillis,
- lastEndMillis }, lastWeights);
- }
- lastStartMillis = startMillis;
- lastEndMillis = endMillis;
- lastWeights = weights;
- }
- lastMonthString = monthString;
- lastMissingValues = missingValues;
- }
- if (lastStartMillis > 0L) {
- compressedHistory.put(new long[] { lastStartMillis, lastEndMillis },
- lastWeights);
- }
- weightsStatus.setHistory(compressedHistory);
- }
-
- public String getStatsString() {
- /* TODO Add statistics string. */
- return null;
- }
-}
-
diff --git a/src/org/torproject/onionoo/cron/Main.java b/src/org/torproject/onionoo/cron/Main.java
new file mode 100644
index 0000000..94dfa00
--- /dev/null
+++ b/src/org/torproject/onionoo/cron/Main.java
@@ -0,0 +1,140 @@
+/* Copyright 2011, 2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.cron;
+
+import java.io.File;
+
+import org.torproject.onionoo.docs.DocumentStore;
+import org.torproject.onionoo.updater.BandwidthStatusUpdater;
+import org.torproject.onionoo.updater.ClientsStatusUpdater;
+import org.torproject.onionoo.updater.DescriptorSource;
+import org.torproject.onionoo.updater.LookupService;
+import org.torproject.onionoo.updater.NodeDetailsStatusUpdater;
+import org.torproject.onionoo.updater.ReverseDomainNameResolver;
+import org.torproject.onionoo.updater.StatusUpdater;
+import org.torproject.onionoo.updater.UptimeStatusUpdater;
+import org.torproject.onionoo.updater.WeightsStatusUpdater;
+import org.torproject.onionoo.util.ApplicationFactory;
+import org.torproject.onionoo.util.LockFile;
+import org.torproject.onionoo.util.Logger;
+import org.torproject.onionoo.writer.BandwidthDocumentWriter;
+import org.torproject.onionoo.writer.ClientsDocumentWriter;
+import org.torproject.onionoo.writer.DetailsDocumentWriter;
+import org.torproject.onionoo.writer.DocumentWriter;
+import org.torproject.onionoo.writer.SummaryDocumentWriter;
+import org.torproject.onionoo.writer.UptimeDocumentWriter;
+import org.torproject.onionoo.writer.WeightsDocumentWriter;
+
+/* Update search data and status data files. */
+public class Main {
+
+ private Main() {
+ }
+
+ public static void main(String[] args) {
+
+ LockFile lf = new LockFile();
+ Logger.setTime();
+ Logger.printStatus("Initializing.");
+ if (lf.acquireLock()) {
+ Logger.printStatusTime("Acquired lock");
+ } else {
+ Logger.printErrorTime("Could not acquire lock. Is Onionoo "
+ + "already running? Terminating");
+ return;
+ }
+
+ DescriptorSource dso = ApplicationFactory.getDescriptorSource();
+ Logger.printStatusTime("Initialized descriptor source");
+ DocumentStore ds = ApplicationFactory.getDocumentStore();
+ Logger.printStatusTime("Initialized document store");
+ LookupService ls = new LookupService(new File("geoip"));
+ Logger.printStatusTime("Initialized Geoip lookup service");
+ ReverseDomainNameResolver rdnr = new ReverseDomainNameResolver();
+ Logger.printStatusTime("Initialized reverse domain name resolver");
+ NodeDetailsStatusUpdater ndsu = new NodeDetailsStatusUpdater(rdnr,
+ ls);
+ Logger.printStatusTime("Initialized node data writer");
+ BandwidthStatusUpdater bsu = new BandwidthStatusUpdater();
+ Logger.printStatusTime("Initialized bandwidth status updater");
+ WeightsStatusUpdater wsu = new WeightsStatusUpdater();
+ Logger.printStatusTime("Initialized weights status updater");
+ ClientsStatusUpdater csu = new ClientsStatusUpdater();
+ Logger.printStatusTime("Initialized clients status updater");
+ UptimeStatusUpdater usu = new UptimeStatusUpdater();
+ Logger.printStatusTime("Initialized uptime status updater");
+ StatusUpdater[] sus = new StatusUpdater[] { ndsu, bsu, wsu, csu,
+ usu };
+
+ SummaryDocumentWriter sdw = new SummaryDocumentWriter();
+ Logger.printStatusTime("Initialized summary document writer");
+ DetailsDocumentWriter ddw = new DetailsDocumentWriter();
+ Logger.printStatusTime("Initialized details document writer");
+ BandwidthDocumentWriter bdw = new BandwidthDocumentWriter();
+ Logger.printStatusTime("Initialized bandwidth document writer");
+ WeightsDocumentWriter wdw = new WeightsDocumentWriter();
+ Logger.printStatusTime("Initialized weights document writer");
+ ClientsDocumentWriter cdw = new ClientsDocumentWriter();
+ Logger.printStatusTime("Initialized clients document writer");
+ UptimeDocumentWriter udw = new UptimeDocumentWriter();
+ Logger.printStatusTime("Initialized uptime document writer");
+ DocumentWriter[] dws = new DocumentWriter[] { sdw, ddw, bdw, wdw, cdw,
+ udw };
+
+ Logger.printStatus("Downloading descriptors.");
+ dso.downloadDescriptors();
+
+ Logger.printStatus("Reading descriptors.");
+ dso.readDescriptors();
+
+ Logger.printStatus("Updating internal status files.");
+ for (StatusUpdater su : sus) {
+ su.updateStatuses();
+ Logger.printStatusTime(su.getClass().getSimpleName()
+ + " updated status files");
+ }
+
+ Logger.printStatus("Updating document files.");
+ for (DocumentWriter dw : dws) {
+ dw.writeDocuments();
+ }
+
+ Logger.printStatus("Shutting down.");
+ dso.writeHistoryFiles();
+ Logger.printStatusTime("Wrote parse histories");
+ ds.flushDocumentCache();
+ Logger.printStatusTime("Flushed document cache");
+
+ Logger.printStatus("Gathering statistics.");
+ for (StatusUpdater su : sus) {
+ String statsString = su.getStatsString();
+ if (statsString != null) {
+ Logger.printStatistics(su.getClass().getSimpleName(),
+ statsString);
+ }
+ }
+ for (DocumentWriter dw : dws) {
+ String statsString = dw.getStatsString();
+ if (statsString != null) {
+ Logger.printStatistics(dw.getClass().getSimpleName(),
+ statsString);
+ }
+ }
+ Logger.printStatistics("Descriptor source", dso.getStatsString());
+ Logger.printStatistics("Document store", ds.getStatsString());
+ Logger.printStatistics("GeoIP lookup service", ls.getStatsString());
+ Logger.printStatistics("Reverse domain name resolver",
+ rdnr.getStatsString());
+
+ Logger.printStatus("Releasing lock.");
+ if (lf.releaseLock()) {
+ Logger.printStatusTime("Released lock");
+ } else {
+ Logger.printErrorTime("Could not release lock. The next "
+ + "execution may not start as expected");
+ }
+
+ Logger.printStatus("Terminating.");
+ }
+}
+
diff --git a/src/org/torproject/onionoo/docs/BandwidthDocument.java b/src/org/torproject/onionoo/docs/BandwidthDocument.java
new file mode 100644
index 0000000..ea20a5e
--- /dev/null
+++ b/src/org/torproject/onionoo/docs/BandwidthDocument.java
@@ -0,0 +1,27 @@
+/* Copyright 2013 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.docs;
+
+import java.util.Map;
+
+public class BandwidthDocument extends Document {
+
+ @SuppressWarnings("unused")
+ private String fingerprint;
+ public void setFingerprint(String fingerprint) {
+ this.fingerprint = fingerprint;
+ }
+
+ @SuppressWarnings("unused")
+ private Map<String, GraphHistory> write_history;
+ public void setWriteHistory(Map<String, GraphHistory> writeHistory) {
+ this.write_history = writeHistory;
+ }
+
+ @SuppressWarnings("unused")
+ private Map<String, GraphHistory> read_history;
+ public void setReadHistory(Map<String, GraphHistory> readHistory) {
+ this.read_history = readHistory;
+ }
+}
+
diff --git a/src/org/torproject/onionoo/docs/BandwidthStatus.java b/src/org/torproject/onionoo/docs/BandwidthStatus.java
new file mode 100644
index 0000000..a2980e5
--- /dev/null
+++ b/src/org/torproject/onionoo/docs/BandwidthStatus.java
@@ -0,0 +1,80 @@
+/* Copyright 2013--2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.docs;
+
+import java.util.Scanner;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.torproject.onionoo.util.DateTimeHelper;
+
+public class BandwidthStatus extends Document {
+
+ private SortedMap<Long, long[]> writeHistory =
+ new TreeMap<Long, long[]>();
+ public void setWriteHistory(SortedMap<Long, long[]> writeHistory) {
+ this.writeHistory = writeHistory;
+ }
+ public SortedMap<Long, long[]> getWriteHistory() {
+ return this.writeHistory;
+ }
+
+ private SortedMap<Long, long[]> readHistory =
+ new TreeMap<Long, long[]>();
+ public void setReadHistory(SortedMap<Long, long[]> readHistory) {
+ this.readHistory = readHistory;
+ }
+ public SortedMap<Long, long[]> getReadHistory() {
+ return this.readHistory;
+ }
+
+ public void fromDocumentString(String documentString) {
+ Scanner s = new Scanner(documentString);
+ while (s.hasNextLine()) {
+ String line = s.nextLine();
+ String[] parts = line.split(" ");
+ if (parts.length != 6) {
+ System.err.println("Illegal line '" + line + "' in bandwidth "
+ + "history. Skipping this line.");
+ continue;
+ }
+ SortedMap<Long, long[]> history = parts[0].equals("r")
+ ? readHistory : writeHistory;
+ long startMillis = DateTimeHelper.parse(parts[1] + " " + parts[2]);
+ long endMillis = DateTimeHelper.parse(parts[3] + " " + parts[4]);
+ if (startMillis < 0L || endMillis < 0L) {
+ System.err.println("Could not parse timestamp while reading "
+ + "bandwidth history. Skipping.");
+ break;
+ }
+ long bandwidth = Long.parseLong(parts[5]);
+ long previousEndMillis = history.headMap(startMillis).isEmpty()
+ ? startMillis
+ : history.get(history.headMap(startMillis).lastKey())[1];
+ long nextStartMillis = history.tailMap(startMillis).isEmpty()
+ ? endMillis : history.tailMap(startMillis).firstKey();
+ if (previousEndMillis <= startMillis &&
+ nextStartMillis >= endMillis) {
+ history.put(startMillis, new long[] { startMillis, endMillis,
+ bandwidth });
+ }
+ }
+ s.close();
+ }
+
+ public String toDocumentString() {
+ StringBuilder sb = new StringBuilder();
+ for (long[] v : writeHistory.values()) {
+ sb.append("w " + DateTimeHelper.format(v[0]) + " "
+ + DateTimeHelper.format(v[1]) + " " + String.valueOf(v[2])
+ + "\n");
+ }
+ for (long[] v : readHistory.values()) {
+ sb.append("r " + DateTimeHelper.format(v[0]) + " "
+ + DateTimeHelper.format(v[1]) + " " + String.valueOf(v[2])
+ + "\n");
+ }
+ return sb.toString();
+ }
+}
+
diff --git a/src/org/torproject/onionoo/docs/ClientsDocument.java b/src/org/torproject/onionoo/docs/ClientsDocument.java
new file mode 100644
index 0000000..27b1588
--- /dev/null
+++ b/src/org/torproject/onionoo/docs/ClientsDocument.java
@@ -0,0 +1,22 @@
+/* Copyright 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.docs;
+
+import java.util.Map;
+
+public class ClientsDocument extends Document {
+
+ @SuppressWarnings("unused")
+ private String fingerprint;
+ public void setFingerprint(String fingerprint) {
+ this.fingerprint = fingerprint;
+ }
+
+ @SuppressWarnings("unused")
+ private Map<String, ClientsGraphHistory> average_clients;
+ public void setAverageClients(
+ Map<String, ClientsGraphHistory> averageClients) {
+ this.average_clients = averageClients;
+ }
+}
+
diff --git a/src/org/torproject/onionoo/docs/ClientsGraphHistory.java b/src/org/torproject/onionoo/docs/ClientsGraphHistory.java
new file mode 100644
index 0000000..e1db663
--- /dev/null
+++ b/src/org/torproject/onionoo/docs/ClientsGraphHistory.java
@@ -0,0 +1,83 @@
+/* Copyright 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.docs;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.SortedMap;
+
+public class ClientsGraphHistory {
+
+ private String first;
+ public void setFirst(String first) {
+ this.first = first;
+ }
+ public String getFirst() {
+ return this.first;
+ }
+
+ private String last;
+ public void setLast(String last) {
+ this.last = last;
+ }
+ public String getLast() {
+ return this.last;
+ }
+
+ private Integer interval;
+ public void setInterval(Integer interval) {
+ this.interval = interval;
+ }
+ public Integer getInterval() {
+ return this.interval;
+ }
+
+ private Double factor;
+ public void setFactor(Double factor) {
+ this.factor = factor;
+ }
+ public Double getFactor() {
+ return this.factor;
+ }
+
+ private Integer count;
+ public void setCount(Integer count) {
+ this.count = count;
+ }
+ public Integer getCount() {
+ return this.count;
+ }
+
+ private List<Integer> values = new ArrayList<Integer>();
+ public void setValues(List<Integer> values) {
+ this.values = values;
+ }
+ public List<Integer> getValues() {
+ return this.values;
+ }
+
+ private SortedMap<String, Float> countries;
+ public void setCountries(SortedMap<String, Float> countries) {
+ this.countries = countries;
+ }
+ public SortedMap<String, Float> getCountries() {
+ return this.countries;
+ }
+
+ private SortedMap<String, Float> transports;
+ public void setTransports(SortedMap<String, Float> transports) {
+ this.transports = transports;
+ }
+ public SortedMap<String, Float> getTransports() {
+ return this.transports;
+ }
+
+ private SortedMap<String, Float> versions;
+ public void setVersions(SortedMap<String, Float> versions) {
+ this.versions = versions;
+ }
+ public SortedMap<String, Float> getVersions() {
+ return this.versions;
+ }
+}
+
diff --git a/src/org/torproject/onionoo/docs/ClientsHistory.java b/src/org/torproject/onionoo/docs/ClientsHistory.java
new file mode 100644
index 0000000..446dd10
--- /dev/null
+++ b/src/org/torproject/onionoo/docs/ClientsHistory.java
@@ -0,0 +1,174 @@
+/* Copyright 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.docs;
+
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.torproject.onionoo.util.DateTimeHelper;
+
+public class ClientsHistory implements Comparable<ClientsHistory> {
+
+ private long startMillis;
+ public long getStartMillis() {
+ return this.startMillis;
+ }
+
+ private long endMillis;
+ public long getEndMillis() {
+ return this.endMillis;
+ }
+
+ private double totalResponses;
+ public double getTotalResponses() {
+ return this.totalResponses;
+ }
+
+ private SortedMap<String, Double> responsesByCountry;
+ public SortedMap<String, Double> getResponsesByCountry() {
+ return this.responsesByCountry;
+ }
+
+ private SortedMap<String, Double> responsesByTransport;
+ public SortedMap<String, Double> getResponsesByTransport() {
+ return this.responsesByTransport;
+ }
+
+ private SortedMap<String, Double> responsesByVersion;
+ public SortedMap<String, Double> getResponsesByVersion() {
+ return this.responsesByVersion;
+ }
+
+ public ClientsHistory(long startMillis, long endMillis,
+ double totalResponses,
+ SortedMap<String, Double> responsesByCountry,
+ SortedMap<String, Double> responsesByTransport,
+ SortedMap<String, Double> responsesByVersion) {
+ this.startMillis = startMillis;
+ this.endMillis = endMillis;
+ this.totalResponses = totalResponses;
+ this.responsesByCountry = responsesByCountry;
+ this.responsesByTransport = responsesByTransport;
+ this.responsesByVersion = responsesByVersion;
+ }
+
+ public static ClientsHistory fromString(
+ String responseHistoryString) {
+ String[] parts = responseHistoryString.split(" ", 8);
+ if (parts.length != 8) {
+ return null;
+ }
+ long startMillis = DateTimeHelper.parse(parts[0] + " " + parts[1]);
+ long endMillis = DateTimeHelper.parse(parts[2] + " " + parts[3]);
+ if (startMillis < 0L || endMillis < 0L) {
+ return null;
+ }
+ if (startMillis >= endMillis) {
+ return null;
+ }
+ double totalResponses = 0.0;
+ try {
+ totalResponses = Double.parseDouble(parts[4]);
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ SortedMap<String, Double> responsesByCountry =
+ parseResponses(parts[5]);
+ SortedMap<String, Double> responsesByTransport =
+ parseResponses(parts[6]);
+ SortedMap<String, Double> responsesByVersion =
+ parseResponses(parts[7]);
+ if (responsesByCountry == null || responsesByTransport == null ||
+ responsesByVersion == null) {
+ return null;
+ }
+ return new ClientsHistory(startMillis, endMillis, totalResponses,
+ responsesByCountry, responsesByTransport, responsesByVersion);
+ }
+
+ private static SortedMap<String, Double> parseResponses(
+ String responsesString) {
+ SortedMap<String, Double> responses = new TreeMap<String, Double>();
+ if (responsesString.length() > 0) {
+ for (String pair : responsesString.split(",")) {
+ String[] keyValue = pair.split("=");
+ if (keyValue.length != 2) {
+ return null;
+ }
+ double value = 0.0;
+ try {
+ value = Double.parseDouble(keyValue[1]);
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ responses.put(keyValue[0], value);
+ }
+ }
+ return responses;
+ }
+
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(DateTimeHelper.format(startMillis));
+ sb.append(" " + DateTimeHelper.format(endMillis));
+ sb.append(" " + String.format("%.3f", this.totalResponses));
+ this.appendResponses(sb, this.responsesByCountry);
+ this.appendResponses(sb, this.responsesByTransport);
+ this.appendResponses(sb, this.responsesByVersion);
+ return sb.toString();
+ }
+
+ private void appendResponses(StringBuilder sb,
+ SortedMap<String, Double> responses) {
+ sb.append(" ");
+ int written = 0;
+ for (Map.Entry<String, Double> e : responses.entrySet()) {
+ sb.append((written++ > 0 ? "," : "") + e.getKey() + "="
+ + String.format("%.3f", e.getValue()));
+ }
+ }
+
+ public void addResponses(ClientsHistory other) {
+ this.totalResponses += other.totalResponses;
+ this.addResponsesByCategory(this.responsesByCountry,
+ other.responsesByCountry);
+ this.addResponsesByCategory(this.responsesByTransport,
+ other.responsesByTransport);
+ this.addResponsesByCategory(this.responsesByVersion,
+ other.responsesByVersion);
+ if (this.startMillis > other.startMillis) {
+ this.startMillis = other.startMillis;
+ }
+ if (this.endMillis < other.endMillis) {
+ this.endMillis = other.endMillis;
+ }
+ }
+
+ private void addResponsesByCategory(
+ SortedMap<String, Double> thisResponses,
+ SortedMap<String, Double> otherResponses) {
+ for (Map.Entry<String, Double> e : otherResponses.entrySet()) {
+ if (thisResponses.containsKey(e.getKey())) {
+ thisResponses.put(e.getKey(), thisResponses.get(e.getKey())
+ + e.getValue());
+ } else {
+ thisResponses.put(e.getKey(), e.getValue());
+ }
+ }
+ }
+
+ public int compareTo(ClientsHistory other) {
+ return this.startMillis < other.startMillis ? -1 :
+ this.startMillis > other.startMillis ? 1 : 0;
+ }
+
+ public boolean equals(Object other) {
+ return other instanceof ClientsHistory &&
+ this.startMillis == ((ClientsHistory) other).startMillis;
+ }
+
+ public int hashCode() {
+ return (int) this.startMillis;
+ }
+}
\ No newline at end of file
diff --git a/src/org/torproject/onionoo/docs/ClientsStatus.java b/src/org/torproject/onionoo/docs/ClientsStatus.java
new file mode 100644
index 0000000..2bd2168
--- /dev/null
+++ b/src/org/torproject/onionoo/docs/ClientsStatus.java
@@ -0,0 +1,43 @@
+/* Copyright 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.docs;
+
+import java.util.Scanner;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+public class ClientsStatus extends Document {
+
+ private SortedSet<ClientsHistory> history =
+ new TreeSet<ClientsHistory>();
+ public void setHistory(SortedSet<ClientsHistory> history) {
+ this.history = history;
+ }
+ public SortedSet<ClientsHistory> getHistory() {
+ return this.history;
+ }
+
+ public void fromDocumentString(String documentString) {
+ Scanner s = new Scanner(documentString);
+ while (s.hasNextLine()) {
+ String line = s.nextLine();
+ ClientsHistory parsedLine = ClientsHistory.fromString(line);
+ if (parsedLine != null) {
+ this.history.add(parsedLine);
+ } else {
+ System.err.println("Could not parse clients history line '"
+ + line + "'. Skipping.");
+ }
+ }
+ s.close();
+ }
+
+ public String toDocumentString() {
+ StringBuilder sb = new StringBuilder();
+ for (ClientsHistory interval : this.history) {
+ sb.append(interval.toString() + "\n");
+ }
+ return sb.toString();
+ }
+}
+
diff --git a/src/org/torproject/onionoo/docs/DetailsDocument.java b/src/org/torproject/onionoo/docs/DetailsDocument.java
new file mode 100644
index 0000000..142b591
--- /dev/null
+++ b/src/org/torproject/onionoo/docs/DetailsDocument.java
@@ -0,0 +1,365 @@
+/* Copyright 2013--2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.docs;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringEscapeUtils;
+
+public class DetailsDocument extends Document {
+
+ /* We must ensure that details files only contain ASCII characters
+ * and no UTF-8 characters. While UTF-8 characters are perfectly
+ * valid in JSON, this would break compatibility with existing files
+ * pretty badly. We do this by escaping non-ASCII characters, e.g.,
+ * \u00F2. Gson won't treat this as UTF-8, but will think that we want
+ * to write six characters '\', 'u', '0', '0', 'F', '2'. The only thing
+ * we'll have to do is to change back the '\\' that Gson writes for the
+ * '\'. */
+ private static String escapeJSON(String s) {
+ return s == null ? null :
+ StringEscapeUtils.escapeJavaScript(s).replaceAll("\\\\'", "'");
+ }
+ private static String unescapeJSON(String s) {
+ return s == null ? null :
+ StringEscapeUtils.unescapeJavaScript(s.replaceAll("'", "\\'"));
+ }
+
+ private String nickname;
+ public void setNickname(String nickname) {
+ this.nickname = nickname;
+ }
+ public String getNickname() {
+ return this.nickname;
+ }
+
+ private String fingerprint;
+ public void setFingerprint(String fingerprint) {
+ this.fingerprint = fingerprint;
+ }
+ public String getFingerprint() {
+ return this.fingerprint;
+ }
+
+ private String hashed_fingerprint;
+ public void setHashedFingerprint(String hashedFingerprint) {
+ this.hashed_fingerprint = hashedFingerprint;
+ }
+ public String getHashedFingerprint() {
+ return this.hashed_fingerprint;
+ }
+
+ private List<String> or_addresses;
+ public void setOrAddresses(List<String> orAddresses) {
+ this.or_addresses = orAddresses;
+ }
+ public List<String> getOrAddresses() {
+ return this.or_addresses;
+ }
+
+ private List<String> exit_addresses;
+ public void setExitAddresses(List<String> exitAddresses) {
+ this.exit_addresses = exitAddresses;
+ }
+ public List<String> getExitAddresses() {
+ return this.exit_addresses;
+ }
+
+ private String dir_address;
+ public void setDirAddress(String dirAddress) {
+ this.dir_address = dirAddress;
+ }
+ public String getDirAddress() {
+ return this.dir_address;
+ }
+
+ private String last_seen;
+ public void setLastSeen(String lastSeen) {
+ this.last_seen = lastSeen;
+ }
+ public String getLastSeen() {
+ return this.last_seen;
+ }
+
+ private String last_changed_address_or_port;
+ public void setLastChangedAddressOrPort(
+ String lastChangedAddressOrPort) {
+ this.last_changed_address_or_port = lastChangedAddressOrPort;
+ }
+ public String getLastChangedAddressOrPort() {
+ return this.last_changed_address_or_port;
+ }
+
+ private String first_seen;
+ public void setFirstSeen(String firstSeen) {
+ this.first_seen = firstSeen;
+ }
+ public String getFirstSeen() {
+ return this.first_seen;
+ }
+
+ private Boolean running;
+ public void setRunning(Boolean running) {
+ this.running = running;
+ }
+ public Boolean getRunning() {
+ return this.running;
+ }
+
+ private List<String> flags;
+ public void setFlags(List<String> flags) {
+ this.flags = flags;
+ }
+ public List<String> getFlags() {
+ return this.flags;
+ }
+
+ private String country;
+ public void setCountry(String country) {
+ this.country = country;
+ }
+ public String getCountry() {
+ return this.country;
+ }
+
+ private String country_name;
+ public void setCountryName(String countryName) {
+ this.country_name = escapeJSON(countryName);
+ }
+ public String getCountryName() {
+ return unescapeJSON(this.country_name);
+ }
+
+ private String region_name;
+ public void setRegionName(String regionName) {
+ this.region_name = escapeJSON(regionName);
+ }
+ public String getRegionName() {
+ return unescapeJSON(this.region_name);
+ }
+
+ private String city_name;
+ public void setCityName(String cityName) {
+ this.city_name = escapeJSON(cityName);
+ }
+ public String getCityName() {
+ return unescapeJSON(this.city_name);
+ }
+
+ private Float latitude;
+ public void setLatitude(Float latitude) {
+ this.latitude = latitude;
+ }
+ public Float getLatitude() {
+ return this.latitude;
+ }
+
+ private Float longitude;
+ public void setLongitude(Float longitude) {
+ this.longitude = longitude;
+ }
+ public Float getLongitude() {
+ return this.longitude;
+ }
+
+ private String as_number;
+ public void setAsNumber(String asNumber) {
+ this.as_number = escapeJSON(asNumber);
+ }
+ public String getAsNumber() {
+ return unescapeJSON(this.as_number);
+ }
+
+ private String as_name;
+ public void setAsName(String asName) {
+ this.as_name = escapeJSON(asName);
+ }
+ public String getAsName() {
+ return unescapeJSON(this.as_name);
+ }
+
+ private Long consensus_weight;
+ public void setConsensusWeight(Long consensusWeight) {
+ this.consensus_weight = consensusWeight;
+ }
+ public Long getConsensusWeight() {
+ return this.consensus_weight;
+ }
+
+ private String host_name;
+ public void setHostName(String hostName) {
+ this.host_name = escapeJSON(hostName);
+ }
+ public String getHostName() {
+ return unescapeJSON(this.host_name);
+ }
+
+ private String last_restarted;
+ public void setLastRestarted(String lastRestarted) {
+ this.last_restarted = lastRestarted;
+ }
+ public String getLastRestarted() {
+ return this.last_restarted;
+ }
+
+ private Integer bandwidth_rate;
+ public void setBandwidthRate(Integer bandwidthRate) {
+ this.bandwidth_rate = bandwidthRate;
+ }
+ public Integer getBandwidthRate() {
+ return this.bandwidth_rate;
+ }
+
+ private Integer bandwidth_burst;
+ public void setBandwidthBurst(Integer bandwidthBurst) {
+ this.bandwidth_burst = bandwidthBurst;
+ }
+ public Integer getBandwidthBurst() {
+ return this.bandwidth_burst;
+ }
+
+ private Integer observed_bandwidth;
+ public void setObservedBandwidth(Integer observedBandwidth) {
+ this.observed_bandwidth = observedBandwidth;
+ }
+ public Integer getObservedBandwidth() {
+ return this.observed_bandwidth;
+ }
+
+ private Integer advertised_bandwidth;
+ public void setAdvertisedBandwidth(Integer advertisedBandwidth) {
+ this.advertised_bandwidth = advertisedBandwidth;
+ }
+ public Integer getAdvertisedBandwidth() {
+ return this.advertised_bandwidth;
+ }
+
+ private List<String> exit_policy;
+ public void setExitPolicy(List<String> exitPolicy) {
+ this.exit_policy = exitPolicy;
+ }
+ public List<String> getExitPolicy() {
+ return this.exit_policy;
+ }
+
+ private Map<String, List<String>> exit_policy_summary;
+ public void setExitPolicySummary(
+ Map<String, List<String>> exitPolicySummary) {
+ this.exit_policy_summary = exitPolicySummary;
+ }
+ public Map<String, List<String>> getExitPolicySummary() {
+ return this.exit_policy_summary;
+ }
+
+ private Map<String, List<String>> exit_policy_v6_summary;
+ public void setExitPolicyV6Summary(
+ Map<String, List<String>> exitPolicyV6Summary) {
+ this.exit_policy_v6_summary = exitPolicyV6Summary;
+ }
+ public Map<String, List<String>> getExitPolicyV6Summary() {
+ return this.exit_policy_v6_summary;
+ }
+
+ private String contact;
+ public void setContact(String contact) {
+ this.contact = escapeJSON(contact);
+ }
+ public String getContact() {
+ return unescapeJSON(this.contact);
+ }
+
+ private String platform;
+ public void setPlatform(String platform) {
+ this.platform = escapeJSON(platform);
+ }
+ public String getPlatform() {
+ return unescapeJSON(this.platform);
+ }
+
+ private List<String> family;
+ public void setFamily(List<String> family) {
+ this.family = family;
+ }
+ public List<String> getFamily() {
+ return this.family;
+ }
+
+ private Float advertised_bandwidth_fraction;
+ public void setAdvertisedBandwidthFraction(
+ Float advertisedBandwidthFraction) {
+ if (advertisedBandwidthFraction == null ||
+ advertisedBandwidthFraction >= 0.0) {
+ this.advertised_bandwidth_fraction = advertisedBandwidthFraction;
+ }
+ }
+ public Float getAdvertisedBandwidthFraction() {
+ return this.advertised_bandwidth_fraction;
+ }
+
+ private Float consensus_weight_fraction;
+ public void setConsensusWeightFraction(Float consensusWeightFraction) {
+ if (consensusWeightFraction == null ||
+ consensusWeightFraction >= 0.0) {
+ this.consensus_weight_fraction = consensusWeightFraction;
+ }
+ }
+ public Float getConsensusWeightFraction() {
+ return this.consensus_weight_fraction;
+ }
+
+ private Float guard_probability;
+ public void setGuardProbability(Float guardProbability) {
+ if (guardProbability == null || guardProbability >= 0.0) {
+ this.guard_probability = guardProbability;
+ }
+ }
+ public Float getGuardProbability() {
+ return this.guard_probability;
+ }
+
+ private Float middle_probability;
+ public void setMiddleProbability(Float middleProbability) {
+ if (middleProbability == null || middleProbability >= 0.0) {
+ this.middle_probability = middleProbability;
+ }
+ }
+ public Float getMiddleProbability() {
+ return this.middle_probability;
+ }
+
+ private Float exit_probability;
+ public void setExitProbability(Float exitProbability) {
+ if (exitProbability == null || exitProbability >= 0.0) {
+ this.exit_probability = exitProbability;
+ }
+ }
+ public Float getExitProbability() {
+ return this.exit_probability;
+ }
+
+ private Boolean recommended_version;
+ public void setRecommendedVersion(Boolean recommendedVersion) {
+ this.recommended_version = recommendedVersion;
+ }
+ public Boolean getRecommendedVersion() {
+ return this.recommended_version;
+ }
+
+ private Boolean hibernating;
+ public void setHibernating(Boolean hibernating) {
+ this.hibernating = hibernating;
+ }
+ public Boolean getHibernating() {
+ return this.hibernating;
+ }
+
+ private String pool_assignment;
+ public void setPoolAssignment(String poolAssignment) {
+ this.pool_assignment = poolAssignment;
+ }
+ public String getPoolAssignment() {
+ return this.pool_assignment;
+ }
+}
+
diff --git a/src/org/torproject/onionoo/docs/DetailsStatus.java b/src/org/torproject/onionoo/docs/DetailsStatus.java
new file mode 100644
index 0000000..a19b4b9
--- /dev/null
+++ b/src/org/torproject/onionoo/docs/DetailsStatus.java
@@ -0,0 +1,141 @@
+/* Copyright 2013--2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.docs;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringEscapeUtils;
+
+public class DetailsStatus extends Document {
+
+ /* We must ensure that details files only contain ASCII characters
+ * and no UTF-8 characters. While UTF-8 characters are perfectly
+ * valid in JSON, this would break compatibility with existing files
+ * pretty badly. We do this by escaping non-ASCII characters, e.g.,
+ * \u00F2. Gson won't treat this as UTF-8, but will think that we want
+ * to write six characters '\', 'u', '0', '0', 'F', '2'. The only thing
+ * we'll have to do is to change back the '\\' that Gson writes for the
+ * '\'. */
+ private static String escapeJSON(String s) {
+ return s == null ? null :
+ StringEscapeUtils.escapeJavaScript(s).replaceAll("\\\\'", "'");
+ }
+ private static String unescapeJSON(String s) {
+ return s == null ? null :
+ StringEscapeUtils.unescapeJavaScript(s.replaceAll("'", "\\'"));
+ }
+
+ private String desc_published;
+ public void setDescPublished(String descPublished) {
+ this.desc_published = descPublished;
+ }
+ public String getDescPublished() {
+ return this.desc_published;
+ }
+
+ private String last_restarted;
+ public void setLastRestarted(String lastRestarted) {
+ this.last_restarted = lastRestarted;
+ }
+ public String getLastRestarted() {
+ return this.last_restarted;
+ }
+
+ private Integer bandwidth_rate;
+ public void setBandwidthRate(Integer bandwidthRate) {
+ this.bandwidth_rate = bandwidthRate;
+ }
+ public Integer getBandwidthRate() {
+ return this.bandwidth_rate;
+ }
+
+ private Integer bandwidth_burst;
+ public void setBandwidthBurst(Integer bandwidthBurst) {
+ this.bandwidth_burst = bandwidthBurst;
+ }
+ public Integer getBandwidthBurst() {
+ return this.bandwidth_burst;
+ }
+
+ private Integer observed_bandwidth;
+ public void setObservedBandwidth(Integer observedBandwidth) {
+ this.observed_bandwidth = observedBandwidth;
+ }
+ public Integer getObservedBandwidth() {
+ return this.observed_bandwidth;
+ }
+
+ private Integer advertised_bandwidth;
+ public void setAdvertisedBandwidth(Integer advertisedBandwidth) {
+ this.advertised_bandwidth = advertisedBandwidth;
+ }
+ public Integer getAdvertisedBandwidth() {
+ return this.advertised_bandwidth;
+ }
+
+ private List<String> exit_policy;
+ public void setExitPolicy(List<String> exitPolicy) {
+ this.exit_policy = exitPolicy;
+ }
+ public List<String> getExitPolicy() {
+ return this.exit_policy;
+ }
+
+ private String contact;
+ public void setContact(String contact) {
+ this.contact = escapeJSON(contact);
+ }
+ public String getContact() {
+ return unescapeJSON(this.contact);
+ }
+
+ private String platform;
+ public void setPlatform(String platform) {
+ this.platform = escapeJSON(platform);
+ }
+ public String getPlatform() {
+ return unescapeJSON(this.platform);
+ }
+
+ private List<String> family;
+ public void setFamily(List<String> family) {
+ this.family = family;
+ }
+ public List<String> getFamily() {
+ return this.family;
+ }
+
+ private Map<String, List<String>> exit_policy_v6_summary;
+ public void setExitPolicyV6Summary(
+ Map<String, List<String>> exitPolicyV6Summary) {
+ this.exit_policy_v6_summary = exitPolicyV6Summary;
+ }
+ public Map<String, List<String>> getExitPolicyV6Summary() {
+ return this.exit_policy_v6_summary;
+ }
+
+ private Boolean hibernating;
+ public void setHibernating(Boolean hibernating) {
+ this.hibernating = hibernating;
+ }
+ public Boolean getHibernating() {
+ return this.hibernating;
+ }
+
+ private String pool_assignment;
+ public void setPoolAssignment(String poolAssignment) {
+ this.pool_assignment = poolAssignment;
+ }
+ public String getPoolAssignment() {
+ return this.pool_assignment;
+ }
+
+ private Map<String, Long> exit_addresses;
+ public void setExitAddresses(Map<String, Long> exitAddresses) {
+ this.exit_addresses = exitAddresses;
+ }
+ public Map<String, Long> getExitAddresses() {
+ return this.exit_addresses;
+ }
+}
diff --git a/src/org/torproject/onionoo/docs/Document.java b/src/org/torproject/onionoo/docs/Document.java
new file mode 100644
index 0000000..a581795
--- /dev/null
+++ b/src/org/torproject/onionoo/docs/Document.java
@@ -0,0 +1,24 @@
+/* Copyright 2013 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.docs;
+
+public abstract class Document {
+
+ private transient String documentString;
+ public void setDocumentString(String documentString) {
+ this.documentString = documentString;
+ }
+ public String getDocumentString() {
+ return this.documentString;
+ }
+
+ public void fromDocumentString(String documentString) {
+ /* Subclasses may override this method to parse documentString. */
+ }
+
+ public String toDocumentString() {
+ /* Subclasses may override this method to write documentString. */
+ return null;
+ }
+}
+
diff --git a/src/org/torproject/onionoo/docs/DocumentStore.java b/src/org/torproject/onionoo/docs/DocumentStore.java
new file mode 100644
index 0000000..c4fe965
--- /dev/null
+++ b/src/org/torproject/onionoo/docs/DocumentStore.java
@@ -0,0 +1,748 @@
+/* Copyright 2013 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.docs;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.Stack;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.torproject.onionoo.util.ApplicationFactory;
+import org.torproject.onionoo.util.Logger;
+import org.torproject.onionoo.util.Time;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonParseException;
+
+// TODO For later migration from disk to database, do the following:
+// - read from database and then from disk if not found
+// - write only to database, delete from disk once in database
+// - move entirely to database once disk is "empty"
+// TODO Also look into simple key-value stores instead of real databases.
+public class DocumentStore {
+
+ private final File statusDir = new File("status");
+
+ private File outDir = new File("out");
+ public void setOutDir(File outDir) {
+ this.outDir = outDir;
+ }
+
+ private Time time;
+
+ public DocumentStore() {
+ this.time = ApplicationFactory.getTime();
+ }
+
+ private long listOperations = 0L, listedFiles = 0L, storedFiles = 0L,
+ storedBytes = 0L, retrievedFiles = 0L, retrievedBytes = 0L,
+ removedFiles = 0L;
+
+ /* Node statuses and summary documents are cached in memory, as opposed
+ * to all other document types. These caches are initialized when first
+ * accessing or modifying a NodeStatus or SummaryDocument document,
+ * respectively. */
+ private SortedMap<String, NodeStatus> cachedNodeStatuses;
+ private SortedMap<String, SummaryDocument> cachedSummaryDocuments;
+
+ public <T extends Document> SortedSet<String> list(
+ Class<T> documentType) {
+ if (documentType.equals(NodeStatus.class)) {
+ return this.listNodeStatuses();
+ } else if (documentType.equals(SummaryDocument.class)) {
+ return this.listSummaryDocuments();
+ } else {
+ return this.listDocumentFiles(documentType);
+ }
+ }
+
+ private SortedSet<String> listNodeStatuses() {
+ if (this.cachedNodeStatuses == null) {
+ this.cacheNodeStatuses();
+ }
+ return new TreeSet<String>(this.cachedNodeStatuses.keySet());
+ }
+
+ private void cacheNodeStatuses() {
+ SortedMap<String, NodeStatus> parsedNodeStatuses =
+ new TreeMap<String, NodeStatus>();
+ File directory = this.statusDir;
+ if (directory != null) {
+ File summaryFile = new File(directory, "summary");
+ if (summaryFile.exists()) {
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ summaryFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (line.length() == 0) {
+ continue;
+ }
+ NodeStatus node = NodeStatus.fromString(line);
+ if (node != null) {
+ parsedNodeStatuses.put(node.getFingerprint(), node);
+ }
+ }
+ br.close();
+ this.listedFiles += parsedNodeStatuses.size();
+ this.listOperations++;
+ } catch (IOException e) {
+ System.err.println("Could not read file '"
+ + summaryFile.getAbsolutePath() + "'.");
+ e.printStackTrace();
+ }
+ }
+ }
+ this.cachedNodeStatuses = parsedNodeStatuses;
+ }
+
+ private SortedSet<String> listSummaryDocuments() {
+ if (this.cachedSummaryDocuments == null) {
+ this.cacheSummaryDocuments();
+ }
+ return new TreeSet<String>(this.cachedSummaryDocuments.keySet());
+ }
+
+ private void cacheSummaryDocuments() {
+ SortedMap<String, SummaryDocument> parsedSummaryDocuments =
+ new TreeMap<String, SummaryDocument>();
+ File directory = this.outDir;
+ if (directory != null) {
+ File summaryFile = new File(directory, "summary");
+ if (summaryFile.exists()) {
+ String line = null;
+ try {
+ Gson gson = new Gson();
+ BufferedReader br = new BufferedReader(new FileReader(
+ summaryFile));
+ while ((line = br.readLine()) != null) {
+ if (line.length() == 0) {
+ continue;
+ }
+ SummaryDocument summaryDocument = gson.fromJson(line,
+ SummaryDocument.class);
+ if (summaryDocument != null) {
+ parsedSummaryDocuments.put(summaryDocument.getFingerprint(),
+ summaryDocument);
+ }
+ }
+ br.close();
+ this.listedFiles += parsedSummaryDocuments.size();
+ this.listOperations++;
+ } catch (IOException e) {
+ System.err.println("Could not read file '"
+ + summaryFile.getAbsolutePath() + "'.");
+ e.printStackTrace();
+ } catch (JsonParseException e) {
+ System.err.println("Could not parse summary document '" + line
+ + "' in file '" + summaryFile.getAbsolutePath() + "'.");
+ e.printStackTrace();
+ }
+ }
+ }
+ this.cachedSummaryDocuments = parsedSummaryDocuments;
+ }
+
+ private <T extends Document> SortedSet<String> listDocumentFiles(
+ Class<T> documentType) {
+ SortedSet<String> fingerprints = new TreeSet<String>();
+ File directory = null;
+ String subdirectory = null;
+ if (documentType.equals(DetailsStatus.class)) {
+ directory = this.statusDir;
+ subdirectory = "details";
+ } else if (documentType.equals(BandwidthStatus.class)) {
+ directory = this.statusDir;
+ subdirectory = "bandwidth";
+ } else if (documentType.equals(WeightsStatus.class)) {
+ directory = this.statusDir;
+ subdirectory = "weights";
+ } else if (documentType.equals(ClientsStatus.class)) {
+ directory = this.statusDir;
+ subdirectory = "clients";
+ } else if (documentType.equals(UptimeStatus.class)) {
+ directory = this.statusDir;
+ subdirectory = "uptimes";
+ } else if (documentType.equals(DetailsDocument.class)) {
+ directory = this.outDir;
+ subdirectory = "details";
+ } else if (documentType.equals(BandwidthDocument.class)) {
+ directory = this.outDir;
+ subdirectory = "bandwidth";
+ } else if (documentType.equals(WeightsDocument.class)) {
+ directory = this.outDir;
+ subdirectory = "weights";
+ } else if (documentType.equals(ClientsDocument.class)) {
+ directory = this.outDir;
+ subdirectory = "clients";
+ } else if (documentType.equals(UptimeDocument.class)) {
+ directory = this.outDir;
+ subdirectory = "uptimes";
+ }
+ if (directory != null && subdirectory != null) {
+ Stack<File> files = new Stack<File>();
+ files.add(new File(directory, subdirectory));
+ while (!files.isEmpty()) {
+ File file = files.pop();
+ if (file.isDirectory()) {
+ files.addAll(Arrays.asList(file.listFiles()));
+ } else if (file.getName().length() == 40) {
+ fingerprints.add(file.getName());
+ }
+ }
+ }
+ this.listOperations++;
+ this.listedFiles += fingerprints.size();
+ return fingerprints;
+ }
+
+ public <T extends Document> boolean store(T document) {
+ return this.store(document, null);
+ }
+
+ public <T extends Document> boolean store(T document,
+ String fingerprint) {
+ if (document instanceof NodeStatus) {
+ return this.storeNodeStatus((NodeStatus) document, fingerprint);
+ } else if (document instanceof SummaryDocument) {
+ return this.storeSummaryDocument((SummaryDocument) document,
+ fingerprint);
+ } else {
+ return this.storeDocumentFile(document, fingerprint);
+ }
+ }
+
+ private <T extends Document> boolean storeNodeStatus(
+ NodeStatus nodeStatus, String fingerprint) {
+ if (this.cachedNodeStatuses == null) {
+ this.cacheNodeStatuses();
+ }
+ this.cachedNodeStatuses.put(fingerprint, nodeStatus);
+ return true;
+ }
+
+ private <T extends Document> boolean storeSummaryDocument(
+ SummaryDocument summaryDocument, String fingerprint) {
+ if (this.cachedSummaryDocuments == null) {
+ this.cacheSummaryDocuments();
+ }
+ this.cachedSummaryDocuments.put(fingerprint, summaryDocument);
+ return true;
+ }
+
+ private <T extends Document> boolean storeDocumentFile(T document,
+ String fingerprint) {
+ File documentFile = this.getDocumentFile(document.getClass(),
+ fingerprint);
+ if (documentFile == null) {
+ return false;
+ }
+ String documentString;
+ if (document.getDocumentString() != null) {
+ documentString = document.getDocumentString();
+ } else if (document instanceof BandwidthDocument ||
+ document instanceof WeightsDocument ||
+ document instanceof ClientsDocument ||
+ document instanceof UptimeDocument) {
+ Gson gson = new Gson();
+ documentString = gson.toJson(document);
+ } else if (document instanceof DetailsStatus ||
+ document instanceof DetailsDocument) {
+ /* Don't escape HTML characters, like < and >, contained in
+ * strings. */
+ Gson gson = new GsonBuilder().disableHtmlEscaping().create();
+ /* We must ensure that details files only contain ASCII characters
+ * and no UTF-8 characters. While UTF-8 characters are perfectly
+ * valid in JSON, this would break compatibility with existing files
+ * pretty badly. We already make sure that all strings in details
+ * objects are escaped JSON, e.g., \u00F2. When Gson serlializes
+ * this string, it escapes the \ to \\, hence writes \\u00F2. We
+ * need to undo this and change \\u00F2 back to \u00F2. */
+ documentString = gson.toJson(document).replaceAll("\\\\\\\\u",
+ "\\\\u");
+ /* Existing details statuses don't contain opening and closing curly
+ * brackets, so we should remove them from new details statuses,
+ * too. */
+ if (document instanceof DetailsStatus) {
+ documentString = documentString.substring(
+ documentString.indexOf("{") + 1,
+ documentString.lastIndexOf("}"));
+ }
+ } else if (document instanceof BandwidthStatus ||
+ document instanceof WeightsStatus ||
+ document instanceof ClientsStatus ||
+ document instanceof UptimeStatus) {
+ documentString = document.toDocumentString();
+ } else {
+ System.err.println("Serializing is not supported for type "
+ + document.getClass().getName() + ".");
+ return false;
+ }
+ try {
+ documentFile.getParentFile().mkdirs();
+ File documentTempFile = new File(
+ documentFile.getAbsolutePath() + ".tmp");
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ documentTempFile));
+ bw.write(documentString);
+ bw.close();
+ documentFile.delete();
+ documentTempFile.renameTo(documentFile);
+ this.storedFiles++;
+ this.storedBytes += documentString.length();
+ } catch (IOException e) {
+ System.err.println("Could not write file '"
+ + documentFile.getAbsolutePath() + "'.");
+ e.printStackTrace();
+ return false;
+ }
+ return true;
+ }
+
+ public <T extends Document> T retrieve(Class<T> documentType,
+ boolean parse) {
+ return this.retrieve(documentType, parse, null);
+ }
+
+ public <T extends Document> T retrieve(Class<T> documentType,
+ boolean parse, String fingerprint) {
+ if (documentType.equals(NodeStatus.class)) {
+ return documentType.cast(this.retrieveNodeStatus(fingerprint));
+ } else if (documentType.equals(SummaryDocument.class)) {
+ return documentType.cast(this.retrieveSummaryDocument(fingerprint));
+ } else {
+ return this.retrieveDocumentFile(documentType, parse, fingerprint);
+ }
+ }
+
+ private NodeStatus retrieveNodeStatus(String fingerprint) {
+ if (this.cachedNodeStatuses == null) {
+ this.cacheNodeStatuses();
+ }
+ return this.cachedNodeStatuses.get(fingerprint);
+ }
+
+ private SummaryDocument retrieveSummaryDocument(String fingerprint) {
+ if (this.cachedSummaryDocuments == null) {
+ this.cacheSummaryDocuments();
+ }
+ if (this.cachedSummaryDocuments.containsKey(fingerprint)) {
+ return this.cachedSummaryDocuments.get(fingerprint);
+ }
+ /* TODO This is an evil hack to support looking up relays or bridges
+ * that haven't been running for a week without having to load
+ * 500,000 NodeStatus instances into memory. Maybe there's a better
+ * way? Or do we need to switch to a real database for this? */
+ DetailsDocument detailsDocument = this.retrieveDocumentFile(
+ DetailsDocument.class, true, fingerprint);
+ if (detailsDocument == null) {
+ return null;
+ }
+ boolean isRelay = detailsDocument.getHashedFingerprint() == null;
+ boolean running = false;
+ String nickname = detailsDocument.getNickname();
+ List<String> addresses = new ArrayList<String>();
+ String countryCode = null, aSNumber = null, contact = null;
+ for (String orAddressAndPort : detailsDocument.getOrAddresses()) {
+ if (!orAddressAndPort.contains(":")) {
+ return null;
+ }
+ String orAddress = orAddressAndPort.substring(0,
+ orAddressAndPort.lastIndexOf(":"));
+ if (!addresses.contains(orAddress)) {
+ addresses.add(orAddress);
+ }
+ }
+ if (detailsDocument.getExitAddresses() != null) {
+ for (String exitAddress : detailsDocument.getExitAddresses()) {
+ if (!addresses.contains(exitAddress)) {
+ addresses.add(exitAddress);
+ }
+ }
+ }
+ SortedSet<String> relayFlags = new TreeSet<String>(), family = null;
+ long lastSeenMillis = -1L, consensusWeight = -1L,
+ firstSeenMillis = -1L;
+ SummaryDocument summaryDocument = new SummaryDocument(isRelay,
+ nickname, fingerprint, addresses, lastSeenMillis, running,
+ relayFlags, consensusWeight, countryCode, firstSeenMillis,
+ aSNumber, contact, family);
+ return summaryDocument;
+ }
+
+ private <T extends Document> T retrieveDocumentFile(
+ Class<T> documentType, boolean parse, String fingerprint) {
+ File documentFile = this.getDocumentFile(documentType, fingerprint);
+ if (documentFile == null || !documentFile.exists()) {
+ return null;
+ } else if (documentFile.isDirectory()) {
+ System.err.println("Could not read file '"
+ + documentFile.getAbsolutePath() + "', because it is a "
+ + "directory.");
+ return null;
+ }
+ String documentString = null;
+ try {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ BufferedInputStream bis = new BufferedInputStream(
+ new FileInputStream(documentFile));
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = bis.read(data, 0, 1024)) >= 0) {
+ baos.write(data, 0, len);
+ }
+ bis.close();
+ byte[] allData = baos.toByteArray();
+ if (allData.length == 0) {
+ return null;
+ }
+ documentString = new String(allData, "US-ASCII");
+ this.retrievedFiles++;
+ this.retrievedBytes += documentString.length();
+ } catch (IOException e) {
+ System.err.println("Could not read file '"
+ + documentFile.getAbsolutePath() + "'.");
+ e.printStackTrace();
+ return null;
+ }
+ T result = null;
+ if (!parse) {
+ return this.retrieveUnparsedDocumentFile(documentType,
+ documentString);
+ } else if (documentType.equals(DetailsDocument.class) ||
+ documentType.equals(BandwidthDocument.class) ||
+ documentType.equals(WeightsDocument.class) ||
+ documentType.equals(ClientsDocument.class) ||
+ documentType.equals(UptimeDocument.class)) {
+ return this.retrieveParsedDocumentFile(documentType,
+ documentString);
+ } else if (documentType.equals(BandwidthStatus.class) ||
+ documentType.equals(WeightsStatus.class) ||
+ documentType.equals(ClientsStatus.class) ||
+ documentType.equals(UptimeStatus.class)) {
+ return this.retrieveParsedStatusFile(documentType, documentString);
+ } else if (documentType.equals(DetailsStatus.class)) {
+ return this.retrieveParsedDocumentFile(documentType, "{"
+ + documentString + "}");
+ } else {
+ System.err.println("Parsing is not supported for type "
+ + documentType.getName() + ".");
+ }
+ return result;
+ }
+
+ private <T extends Document> T retrieveParsedStatusFile(
+ Class<T> documentType, String documentString) {
+ T result = null;
+ try {
+ result = documentType.newInstance();
+ result.fromDocumentString(documentString);
+ } catch (InstantiationException e) {
+ /* Handle below. */
+ e.printStackTrace();
+ } catch (IllegalAccessException e) {
+ /* Handle below. */
+ e.printStackTrace();
+ }
+ if (result == null) {
+ System.err.println("Could not initialize parsed status file of "
+ + "type " + documentType.getName() + ".");
+ }
+ return result;
+ }
+
+ private <T extends Document> T retrieveParsedDocumentFile(
+ Class<T> documentType, String documentString) {
+ T result = null;
+ Gson gson = new Gson();
+ try {
+ result = gson.fromJson(documentString, documentType);
+ } catch (JsonParseException e) {
+ /* Handle below. */
+ e.printStackTrace();
+ }
+ if (result == null) {
+ System.err.println("Could not initialize parsed document of type "
+ + documentType.getName() + ".");
+ }
+ return result;
+ }
+
+ private <T extends Document> T retrieveUnparsedDocumentFile(
+ Class<T> documentType, String documentString) {
+ T result = null;
+ try {
+ result = documentType.newInstance();
+ result.setDocumentString(documentString);
+ } catch (InstantiationException e) {
+ /* Handle below. */
+ e.printStackTrace();
+ } catch (IllegalAccessException e) {
+ /* Handle below. */
+ e.printStackTrace();
+ }
+ if (result == null) {
+ System.err.println("Could not initialize unparsed document of type "
+ + documentType.getName() + ".");
+ }
+ return result;
+ }
+
+ public <T extends Document> boolean remove(Class<T> documentType) {
+ return this.remove(documentType, null);
+ }
+
+ public <T extends Document> boolean remove(Class<T> documentType,
+ String fingerprint) {
+ if (documentType.equals(NodeStatus.class)) {
+ return this.removeNodeStatus(fingerprint);
+ } else if (documentType.equals(SummaryDocument.class)) {
+ return this.removeSummaryDocument(fingerprint);
+ } else {
+ return this.removeDocumentFile(documentType, fingerprint);
+ }
+ }
+
+ private boolean removeNodeStatus(String fingerprint) {
+ if (this.cachedNodeStatuses == null) {
+ this.cacheNodeStatuses();
+ }
+ return this.cachedNodeStatuses.remove(fingerprint) != null;
+ }
+
+ private boolean removeSummaryDocument(String fingerprint) {
+ if (this.cachedSummaryDocuments == null) {
+ this.cacheSummaryDocuments();
+ }
+ return this.cachedSummaryDocuments.remove(fingerprint) != null;
+ }
+
+ private <T extends Document> boolean removeDocumentFile(
+ Class<T> documentType, String fingerprint) {
+ File documentFile = this.getDocumentFile(documentType, fingerprint);
+ if (documentFile == null || !documentFile.delete()) {
+ System.err.println("Could not delete file '"
+ + documentFile.getAbsolutePath() + "'.");
+ return false;
+ }
+ this.removedFiles++;
+ return true;
+ }
+
+ private <T extends Document> File getDocumentFile(Class<T> documentType,
+ String fingerprint) {
+ File documentFile = null;
+ if (fingerprint == null && !documentType.equals(UpdateStatus.class) &&
+ !documentType.equals(UptimeStatus.class)) {
+ // TODO Instead of using the update file workaround, add new method
+ // lastModified(Class<T> documentType) that serves a similar
+ // purpose.
+ return null;
+ }
+ File directory = null;
+ String fileName = null;
+ if (documentType.equals(DetailsStatus.class)) {
+ directory = this.statusDir;
+ fileName = String.format("details/%s/%s/%s",
+ fingerprint.substring(0, 1), fingerprint.substring(1, 2),
+ fingerprint);
+ } else if (documentType.equals(BandwidthStatus.class)) {
+ directory = this.statusDir;
+ fileName = String.format("bandwidth/%s/%s/%s",
+ fingerprint.substring(0, 1), fingerprint.substring(1, 2),
+ fingerprint);
+ } else if (documentType.equals(WeightsStatus.class)) {
+ directory = this.statusDir;
+ fileName = String.format("weights/%s/%s/%s",
+ fingerprint.substring(0, 1), fingerprint.substring(1, 2),
+ fingerprint);
+ } else if (documentType.equals(ClientsStatus.class)) {
+ directory = this.statusDir;
+ fileName = String.format("clients/%s/%s/%s",
+ fingerprint.substring(0, 1), fingerprint.substring(1, 2),
+ fingerprint);
+ } else if (documentType.equals(UptimeStatus.class)) {
+ directory = this.statusDir;
+ if (fingerprint == null) {
+ fileName = "uptime";
+ } else {
+ fileName = String.format("uptimes/%s/%s/%s",
+ fingerprint.substring(0, 1), fingerprint.substring(1, 2),
+ fingerprint);
+ }
+ } else if (documentType.equals(UpdateStatus.class)) {
+ directory = this.outDir;
+ fileName = "update";
+ } else if (documentType.equals(DetailsDocument.class)) {
+ directory = this.outDir;
+ fileName = String.format("details/%s", fingerprint);
+ } else if (documentType.equals(BandwidthDocument.class)) {
+ directory = this.outDir;
+ fileName = String.format("bandwidth/%s", fingerprint);
+ } else if (documentType.equals(WeightsDocument.class)) {
+ directory = this.outDir;
+ fileName = String.format("weights/%s", fingerprint);
+ } else if (documentType.equals(ClientsDocument.class)) {
+ directory = this.outDir;
+ fileName = String.format("clients/%s", fingerprint);
+ } else if (documentType.equals(UptimeDocument.class)) {
+ directory = this.outDir;
+ fileName = String.format("uptimes/%s", fingerprint);
+ }
+ if (directory != null && fileName != null) {
+ documentFile = new File(directory, fileName);
+ }
+ return documentFile;
+ }
+
+ public void flushDocumentCache() {
+ /* Write cached node statuses to disk, and write update file
+ * containing current time. It's important to write the update file
+ * now, not earlier, because the front-end should not read new node
+ * statuses until all details, bandwidths, and weights are ready. */
+ if (this.cachedNodeStatuses != null ||
+ this.cachedSummaryDocuments != null) {
+ if (this.cachedNodeStatuses != null) {
+ this.writeNodeStatuses();
+ }
+ if (this.cachedSummaryDocuments != null) {
+ this.writeSummaryDocuments();
+ }
+ this.writeUpdateStatus();
+ }
+ }
+
+ private void writeNodeStatuses() {
+ File directory = this.statusDir;
+ if (directory == null) {
+ return;
+ }
+ File summaryFile = new File(directory, "summary");
+ SortedMap<String, NodeStatus>
+ cachedRelays = new TreeMap<String, NodeStatus>(),
+ cachedBridges = new TreeMap<String, NodeStatus>();
+ for (Map.Entry<String, NodeStatus> e :
+ this.cachedNodeStatuses.entrySet()) {
+ if (e.getValue().isRelay()) {
+ cachedRelays.put(e.getKey(), e.getValue());
+ } else {
+ cachedBridges.put(e.getKey(), e.getValue());
+ }
+ }
+ StringBuilder sb = new StringBuilder();
+ for (NodeStatus relay : cachedRelays.values()) {
+ String line = relay.toString();
+ if (line != null) {
+ sb.append(line + "\n");
+ } else {
+ System.err.println("Could not serialize relay node status '"
+ + relay.getFingerprint() + "'");
+ }
+ }
+ for (NodeStatus bridge : cachedBridges.values()) {
+ String line = bridge.toString();
+ if (line != null) {
+ sb.append(line + "\n");
+ } else {
+ System.err.println("Could not serialize bridge node status '"
+ + bridge.getFingerprint() + "'");
+ }
+ }
+ String documentString = sb.toString();
+ try {
+ summaryFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(summaryFile));
+ bw.write(documentString);
+ bw.close();
+ this.storedFiles++;
+ this.storedBytes += documentString.length();
+ } catch (IOException e) {
+ System.err.println("Could not write file '"
+ + summaryFile.getAbsolutePath() + "'.");
+ e.printStackTrace();
+ }
+ }
+
+ private void writeSummaryDocuments() {
+ StringBuilder sb = new StringBuilder();
+ Gson gson = new Gson();
+ for (SummaryDocument summaryDocument :
+ this.cachedSummaryDocuments.values()) {
+ String line = gson.toJson(summaryDocument);
+ if (line != null) {
+ sb.append(line + "\n");
+ } else {
+ System.err.println("Could not serialize relay summary document '"
+ + summaryDocument.getFingerprint() + "'");
+ }
+ }
+ String documentString = sb.toString();
+ File summaryFile = new File(this.outDir, "summary");
+ try {
+ summaryFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(summaryFile));
+ bw.write(documentString);
+ bw.close();
+ this.storedFiles++;
+ this.storedBytes += documentString.length();
+ } catch (IOException e) {
+ System.err.println("Could not write file '"
+ + summaryFile.getAbsolutePath() + "'.");
+ e.printStackTrace();
+ }
+ }
+
+ private void writeUpdateStatus() {
+ if (this.outDir == null) {
+ return;
+ }
+ File updateFile = new File(this.outDir, "update");
+ String documentString = String.valueOf(this.time.currentTimeMillis());
+ try {
+ updateFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(updateFile));
+ bw.write(documentString);
+ bw.close();
+ this.storedFiles++;
+ this.storedBytes += documentString.length();
+ } catch (IOException e) {
+ System.err.println("Could not write file '"
+ + updateFile.getAbsolutePath() + "'.");
+ e.printStackTrace();
+ }
+ }
+
+ public String getStatsString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(" " + Logger.formatDecimalNumber(listOperations)
+ + " list operations performed\n");
+ sb.append(" " + Logger.formatDecimalNumber(listedFiles)
+ + " files listed\n");
+ sb.append(" " + Logger.formatDecimalNumber(storedFiles)
+ + " files stored\n");
+ sb.append(" " + Logger.formatBytes(storedBytes) + " stored\n");
+ sb.append(" " + Logger.formatDecimalNumber(retrievedFiles)
+ + " files retrieved\n");
+ sb.append(" " + Logger.formatBytes(retrievedBytes)
+ + " retrieved\n");
+ sb.append(" " + Logger.formatDecimalNumber(removedFiles)
+ + " files removed\n");
+ return sb.toString();
+ }
+}
+
diff --git a/src/org/torproject/onionoo/docs/GraphHistory.java b/src/org/torproject/onionoo/docs/GraphHistory.java
new file mode 100644
index 0000000..19ace31
--- /dev/null
+++ b/src/org/torproject/onionoo/docs/GraphHistory.java
@@ -0,0 +1,56 @@
+/* Copyright 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.docs;
+
+import java.util.List;
+
+public class GraphHistory {
+
+ private String first;
+ public void setFirst(String first) {
+ this.first = first;
+ }
+ public String getFirst() {
+ return this.first;
+ }
+
+ private String last;
+ public void setLast(String last) {
+ this.last = last;
+ }
+ public String getLast() {
+ return this.last;
+ }
+
+ private Integer interval;
+ public void setInterval(Integer interval) {
+ this.interval = interval;
+ }
+ public Integer getInterval() {
+ return this.interval;
+ }
+
+ private Double factor;
+ public void setFactor(Double factor) {
+ this.factor = factor;
+ }
+ public Double getFactor() {
+ return this.factor;
+ }
+
+ private Integer count;
+ public void setCount(Integer count) {
+ this.count = count;
+ }
+ public Integer getCount() {
+ return this.count;
+ }
+
+ private List<Integer> values;
+ public void setValues(List<Integer> values) {
+ this.values = values;
+ }
+ public List<Integer> getValues() {
+ return this.values;
+ }
+}
diff --git a/src/org/torproject/onionoo/docs/NodeStatus.java b/src/org/torproject/onionoo/docs/NodeStatus.java
new file mode 100644
index 0000000..41292fd
--- /dev/null
+++ b/src/org/torproject/onionoo/docs/NodeStatus.java
@@ -0,0 +1,582 @@
+/* Copyright 2011, 2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.docs;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.torproject.onionoo.util.DateTimeHelper;
+
+/* Store search data of a single relay that was running in the past seven
+ * days. */
+public class NodeStatus extends Document {
+
+ private boolean isRelay;
+ public boolean isRelay() {
+ return this.isRelay;
+ }
+
+ private String fingerprint;
+ public String getFingerprint() {
+ return this.fingerprint;
+ }
+
+ private String hashedFingerprint;
+ public String getHashedFingerprint() {
+ return this.hashedFingerprint;
+ }
+
+ private String nickname;
+ public String getNickname() {
+ return this.nickname;
+ }
+
+ private String address;
+ public String getAddress() {
+ return this.address;
+ }
+
+ private SortedSet<String> orAddresses;
+ private SortedSet<String> orAddressesAndPorts;
+ public SortedSet<String> getOrAddresses() {
+ return new TreeSet<String>(this.orAddresses);
+ }
+ public void addOrAddressAndPort(String orAddressAndPort) {
+ if (orAddressAndPort.contains(":") && orAddressAndPort.length() > 0) {
+ String orAddress = orAddressAndPort.substring(0,
+ orAddressAndPort.lastIndexOf(':'));
+ if (this.exitAddresses.contains(orAddress)) {
+ this.exitAddresses.remove(orAddress);
+ }
+ this.orAddresses.add(orAddress);
+ this.orAddressesAndPorts.add(orAddressAndPort);
+ }
+ }
+ public SortedSet<String> getOrAddressesAndPorts() {
+ return new TreeSet<String>(this.orAddressesAndPorts);
+ }
+
+ private SortedSet<String> exitAddresses;
+ public void addExitAddress(String exitAddress) {
+ if (exitAddress.length() > 0 && !this.address.equals(exitAddress) &&
+ !this.orAddresses.contains(exitAddress)) {
+ this.exitAddresses.add(exitAddress);
+ }
+ }
+ public SortedSet<String> getExitAddresses() {
+ return new TreeSet<String>(this.exitAddresses);
+ }
+
+ private Float latitude;
+ public void setLatitude(Float latitude) {
+ this.latitude = latitude;
+ }
+ public Float getLatitude() {
+ return this.latitude;
+ }
+
+ private Float longitude;
+ public void setLongitude(Float longitude) {
+ this.longitude = longitude;
+ }
+ public Float getLongitude() {
+ return this.longitude;
+ }
+
+ private String countryCode;
+ public void setCountryCode(String countryCode) {
+ this.countryCode = countryCode;
+ }
+ public String getCountryCode() {
+ return this.countryCode;
+ }
+
+ private String countryName;
+ public void setCountryName(String countryName) {
+ this.countryName = countryName;
+ }
+ public String getCountryName() {
+ return this.countryName;
+ }
+
+ private String regionName;
+ public void setRegionName(String regionName) {
+ this.regionName = regionName;
+ }
+ public String getRegionName() {
+ return this.regionName;
+ }
+
+ private String cityName;
+ public void setCityName(String cityName) {
+ this.cityName = cityName;
+ }
+ public String getCityName() {
+ return this.cityName;
+ }
+
+ private String aSName;
+ public void setASName(String aSName) {
+ this.aSName = aSName;
+ }
+ public String getASName() {
+ return this.aSName;
+ }
+
+ private String aSNumber;
+ public void setASNumber(String aSNumber) {
+ this.aSNumber = aSNumber;
+ }
+ public String getASNumber() {
+ return this.aSNumber;
+ }
+
+ private long firstSeenMillis;
+ public long getFirstSeenMillis() {
+ return this.firstSeenMillis;
+ }
+
+ private long lastSeenMillis;
+ public long getLastSeenMillis() {
+ return this.lastSeenMillis;
+ }
+
+ private int orPort;
+ public int getOrPort() {
+ return this.orPort;
+ }
+
+ private int dirPort;
+ public int getDirPort() {
+ return this.dirPort;
+ }
+
+ private SortedSet<String> relayFlags;
+ public SortedSet<String> getRelayFlags() {
+ return this.relayFlags;
+ }
+
+ private long consensusWeight;
+ public long getConsensusWeight() {
+ return this.consensusWeight;
+ }
+
+ private boolean running;
+ public void setRunning(boolean running) {
+ this.running = running;
+ }
+ public boolean getRunning() {
+ return this.running;
+ }
+
+ private String hostName;
+ public void setHostName(String hostName) {
+ this.hostName = hostName;
+ }
+ public String getHostName() {
+ return this.hostName;
+ }
+
+ private long lastRdnsLookup = -1L;
+ public void setLastRdnsLookup(long lastRdnsLookup) {
+ this.lastRdnsLookup = lastRdnsLookup;
+ }
+ public long getLastRdnsLookup() {
+ return this.lastRdnsLookup;
+ }
+
+ private double advertisedBandwidthFraction = -1.0;
+ public void setAdvertisedBandwidthFraction(
+ double advertisedBandwidthFraction) {
+ this.advertisedBandwidthFraction = advertisedBandwidthFraction;
+ }
+ public double getAdvertisedBandwidthFraction() {
+ return this.advertisedBandwidthFraction;
+ }
+
+ private double consensusWeightFraction = -1.0;
+ public void setConsensusWeightFraction(double consensusWeightFraction) {
+ this.consensusWeightFraction = consensusWeightFraction;
+ }
+ public double getConsensusWeightFraction() {
+ return this.consensusWeightFraction;
+ }
+
+ private double guardProbability = -1.0;
+ public void setGuardProbability(double guardProbability) {
+ this.guardProbability = guardProbability;
+ }
+ public double getGuardProbability() {
+ return this.guardProbability;
+ }
+
+ private double middleProbability = -1.0;
+ public void setMiddleProbability(double middleProbability) {
+ this.middleProbability = middleProbability;
+ }
+ public double getMiddleProbability() {
+ return this.middleProbability;
+ }
+
+ private double exitProbability = -1.0;
+ public void setExitProbability(double exitProbability) {
+ this.exitProbability = exitProbability;
+ }
+ public double getExitProbability() {
+ return this.exitProbability;
+ }
+
+ private String defaultPolicy;
+ public String getDefaultPolicy() {
+ return this.defaultPolicy;
+ }
+
+ private String portList;
+ public String getPortList() {
+ return this.portList;
+ }
+
+ private SortedMap<Long, Set<String>> lastAddresses;
+ public SortedMap<Long, Set<String>> getLastAddresses() {
+ return this.lastAddresses == null ? null :
+ new TreeMap<Long, Set<String>>(this.lastAddresses);
+ }
+ public long getLastChangedOrAddress() {
+ long lastChangedAddressesMillis = -1L;
+ if (this.lastAddresses != null) {
+ Set<String> lastAddresses = null;
+ for (Map.Entry<Long, Set<String>> e : this.lastAddresses.entrySet()) {
+ if (lastAddresses != null) {
+ for (String address : e.getValue()) {
+ if (!lastAddresses.contains(address)) {
+ return lastChangedAddressesMillis;
+ }
+ }
+ }
+ lastChangedAddressesMillis = e.getKey();
+ lastAddresses = e.getValue();
+ }
+ }
+ return lastChangedAddressesMillis;
+ }
+
+ private String contact;
+ public void setContact(String contact) {
+ if (contact == null) {
+ this.contact = null;
+ } else {
+ StringBuilder sb = new StringBuilder();
+ for (char c : contact.toLowerCase().toCharArray()) {
+ if (c >= 32 && c < 127) {
+ sb.append(c);
+ } else {
+ sb.append(" ");
+ }
+ }
+ this.contact = sb.toString();
+ }
+ }
+ public String getContact() {
+ return this.contact;
+ }
+
+ private Boolean recommendedVersion;
+ public Boolean getRecommendedVersion() {
+ return this.recommendedVersion;
+ }
+
+ private SortedSet<String> familyFingerprints;
+ public void setFamilyFingerprints(
+ SortedSet<String> familyFingerprints) {
+ this.familyFingerprints = familyFingerprints;
+ }
+ public SortedSet<String> getFamilyFingerprints() {
+ return this.familyFingerprints;
+ }
+
+ public NodeStatus(boolean isRelay, String nickname, String fingerprint,
+ String address, SortedSet<String> orAddressesAndPorts,
+ SortedSet<String> exitAddresses, long lastSeenMillis, int orPort,
+ int dirPort, SortedSet<String> relayFlags, long consensusWeight,
+ String countryCode, String hostName, long lastRdnsLookup,
+ String defaultPolicy, String portList, long firstSeenMillis,
+ long lastChangedAddresses, String aSNumber, String contact,
+ Boolean recommendedVersion, SortedSet<String> familyFingerprints) {
+ this.isRelay = isRelay;
+ this.nickname = nickname;
+ this.fingerprint = fingerprint;
+ try {
+ this.hashedFingerprint = DigestUtils.shaHex(Hex.decodeHex(
+ this.fingerprint.toCharArray())).toUpperCase();
+ } catch (DecoderException e) {
+ throw new IllegalArgumentException("Fingerprint '" + fingerprint
+ + "' is not a valid fingerprint.", e);
+ }
+ this.address = address;
+ this.exitAddresses = new TreeSet<String>();
+ if (exitAddresses != null) {
+ this.exitAddresses.addAll(exitAddresses);
+ }
+ this.exitAddresses.remove(this.address);
+ this.orAddresses = new TreeSet<String>();
+ this.orAddressesAndPorts = new TreeSet<String>();
+ if (orAddressesAndPorts != null) {
+ for (String orAddressAndPort : orAddressesAndPorts) {
+ this.addOrAddressAndPort(orAddressAndPort);
+ }
+ }
+ this.lastSeenMillis = lastSeenMillis;
+ this.orPort = orPort;
+ this.dirPort = dirPort;
+ this.relayFlags = relayFlags;
+ this.consensusWeight = consensusWeight;
+ this.countryCode = countryCode;
+ this.hostName = hostName;
+ this.lastRdnsLookup = lastRdnsLookup;
+ this.defaultPolicy = defaultPolicy;
+ this.portList = portList;
+ this.firstSeenMillis = firstSeenMillis;
+ this.lastAddresses =
+ new TreeMap<Long, Set<String>>(Collections.reverseOrder());
+ Set<String> addresses = new HashSet<String>();
+ addresses.add(address + ":" + orPort);
+ if (dirPort > 0) {
+ addresses.add(address + ":" + dirPort);
+ }
+ addresses.addAll(orAddressesAndPorts);
+ this.lastAddresses.put(lastChangedAddresses, addresses);
+ this.aSNumber = aSNumber;
+ this.contact = contact;
+ this.recommendedVersion = recommendedVersion;
+ this.familyFingerprints = familyFingerprints;
+ }
+
+ public static NodeStatus fromString(String documentString) {
+ boolean isRelay = false;
+ String nickname = null, fingerprint = null, address = null,
+ countryCode = null, hostName = null, defaultPolicy = null,
+ portList = null, aSNumber = null, contact = null;
+ SortedSet<String> orAddressesAndPorts = null, exitAddresses = null,
+ relayFlags = null, familyFingerprints = null;
+ long lastSeenMillis = -1L, consensusWeight = -1L,
+ lastRdnsLookup = -1L, firstSeenMillis = -1L,
+ lastChangedAddresses = -1L;
+ int orPort = -1, dirPort = -1;
+ Boolean recommendedVersion = null;
+ try {
+ String separator = documentString.contains("\t") ? "\t" : " ";
+ String[] parts = documentString.trim().split(separator);
+ isRelay = parts[0].equals("r");
+ if (parts.length < 9) {
+ System.err.println("Too few space-separated values in line '"
+ + documentString.trim() + "'. Skipping.");
+ return null;
+ }
+ nickname = parts[1];
+ fingerprint = parts[2];
+ orAddressesAndPorts = new TreeSet<String>();
+ exitAddresses = new TreeSet<String>();
+ String addresses = parts[3];
+ if (addresses.contains(";")) {
+ String[] addressParts = addresses.split(";", -1);
+ if (addressParts.length != 3) {
+ System.err.println("Invalid addresses entry in line '"
+ + documentString.trim() + "'. Skipping.");
+ return null;
+ }
+ address = addressParts[0];
+ if (addressParts[1].length() > 0) {
+ orAddressesAndPorts.addAll(Arrays.asList(
+ addressParts[1].split("\\+")));
+ }
+ if (addressParts[2].length() > 0) {
+ exitAddresses.addAll(Arrays.asList(
+ addressParts[2].split("\\+")));
+ }
+ } else {
+ address = addresses;
+ }
+ lastSeenMillis = DateTimeHelper.parse(parts[4] + " " + parts[5]);
+ if (lastSeenMillis < 0L) {
+ System.err.println("Parse exception while parsing node status "
+ + "line '" + documentString + "'. Skipping.");
+ return null;
+ }
+ orPort = Integer.parseInt(parts[6]);
+ dirPort = Integer.parseInt(parts[7]);
+ relayFlags = new TreeSet<String>();
+ if (parts[8].length() > 0) {
+ relayFlags.addAll(Arrays.asList(parts[8].split(",")));
+ }
+ if (parts.length > 9) {
+ consensusWeight = Long.parseLong(parts[9]);
+ }
+ if (parts.length > 10) {
+ countryCode = parts[10];
+ }
+ if (parts.length > 12) {
+ hostName = parts[11].equals("null") ? null : parts[11];
+ lastRdnsLookup = Long.parseLong(parts[12]);
+ }
+ if (parts.length > 14) {
+ if (!parts[13].equals("null")) {
+ defaultPolicy = parts[13];
+ }
+ if (!parts[14].equals("null")) {
+ portList = parts[14];
+ }
+ }
+ firstSeenMillis = lastSeenMillis;
+ if (parts.length > 16) {
+ firstSeenMillis = DateTimeHelper.parse(parts[15] + " "
+ + parts[16]);
+ if (firstSeenMillis < 0L) {
+ System.err.println("Parse exception while parsing node status "
+ + "line '" + documentString + "'. Skipping.");
+ return null;
+ }
+ }
+ lastChangedAddresses = lastSeenMillis;
+ if (parts.length > 18 && !parts[17].equals("null")) {
+ lastChangedAddresses = DateTimeHelper.parse(parts[17] + " "
+ + parts[18]);
+ if (lastChangedAddresses < 0L) {
+ System.err.println("Parse exception while parsing node status "
+ + "line '" + documentString + "'. Skipping.");
+ return null;
+ }
+ }
+ if (parts.length > 19) {
+ aSNumber = parts[19];
+ }
+ if (parts.length > 20) {
+ contact = parts[20];
+ }
+ if (parts.length > 21) {
+ recommendedVersion = parts[21].equals("null") ? null :
+ parts[21].equals("true");
+ }
+ if (parts.length > 22 && !parts[22].equals("null")) {
+ familyFingerprints = new TreeSet<String>(Arrays.asList(
+ parts[22].split(";")));
+ }
+ } catch (NumberFormatException e) {
+ System.err.println("Number format exception while parsing node "
+ + "status line '" + documentString + "': " + e.getMessage()
+ + ". Skipping.");
+ return null;
+ } catch (Exception e) {
+ /* This catch block is only here to handle yet unknown errors. It
+ * should go away once we're sure what kind of errors can occur. */
+ System.err.println("Unknown exception while parsing node status "
+ + "line '" + documentString + "': " + e.getMessage() + ". "
+ + "Skipping.");
+ return null;
+ }
+ NodeStatus newNodeStatus = new NodeStatus(isRelay, nickname,
+ fingerprint, address, orAddressesAndPorts, exitAddresses,
+ lastSeenMillis, orPort, dirPort, relayFlags, consensusWeight,
+ countryCode, hostName, lastRdnsLookup, defaultPolicy, portList,
+ firstSeenMillis, lastChangedAddresses, aSNumber, contact,
+ recommendedVersion, familyFingerprints);
+ return newNodeStatus;
+ }
+
+ public void update(NodeStatus newNodeStatus) {
+ if (newNodeStatus.lastSeenMillis > this.lastSeenMillis) {
+ this.nickname = newNodeStatus.nickname;
+ this.address = newNodeStatus.address;
+ this.orAddressesAndPorts = newNodeStatus.orAddressesAndPorts;
+ this.lastSeenMillis = newNodeStatus.lastSeenMillis;
+ this.orPort = newNodeStatus.orPort;
+ this.dirPort = newNodeStatus.dirPort;
+ this.relayFlags = newNodeStatus.relayFlags;
+ this.consensusWeight = newNodeStatus.consensusWeight;
+ this.countryCode = newNodeStatus.countryCode;
+ this.defaultPolicy = newNodeStatus.defaultPolicy;
+ this.portList = newNodeStatus.portList;
+ this.aSNumber = newNodeStatus.aSNumber;
+ this.recommendedVersion = newNodeStatus.recommendedVersion;
+ }
+ if (this.isRelay && newNodeStatus.isRelay) {
+ this.lastAddresses.putAll(newNodeStatus.lastAddresses);
+ }
+ this.firstSeenMillis = Math.min(newNodeStatus.firstSeenMillis,
+ this.firstSeenMillis);
+ }
+
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(this.isRelay ? "r" : "b");
+ sb.append("\t" + this.nickname);
+ sb.append("\t" + this.fingerprint);
+ sb.append("\t" + this.address + ";");
+ int written = 0;
+ for (String orAddressAndPort : this.orAddressesAndPorts) {
+ sb.append((written++ > 0 ? "+" : "") + orAddressAndPort);
+ }
+ sb.append(";");
+ if (this.isRelay) {
+ written = 0;
+ for (String exitAddress : this.exitAddresses) {
+ sb.append((written++ > 0 ? "+" : "")
+ + exitAddress);
+ }
+ }
+ sb.append("\t" + DateTimeHelper.format(this.lastSeenMillis,
+ DateTimeHelper.ISO_DATETIME_TAB_FORMAT));
+ sb.append("\t" + this.orPort);
+ sb.append("\t" + this.dirPort + "\t");
+ written = 0;
+ for (String relayFlag : this.relayFlags) {
+ sb.append((written++ > 0 ? "," : "") + relayFlag);
+ }
+ if (this.isRelay) {
+ sb.append("\t" + String.valueOf(this.consensusWeight));
+ sb.append("\t"
+ + (this.countryCode != null ? this.countryCode : "??"));
+ sb.append("\t" + (this.hostName != null ? this.hostName : "null"));
+ sb.append("\t" + String.valueOf(this.lastRdnsLookup));
+ sb.append("\t" + (this.defaultPolicy != null ? this.defaultPolicy
+ : "null"));
+ sb.append("\t" + (this.portList != null ? this.portList : "null"));
+ } else {
+ sb.append("\t-1\t??\tnull\t-1\tnull\tnull");
+ }
+ sb.append("\t" + DateTimeHelper.format(this.firstSeenMillis,
+ DateTimeHelper.ISO_DATETIME_TAB_FORMAT));
+ if (this.isRelay) {
+ sb.append("\t" + DateTimeHelper.format(
+ this.getLastChangedOrAddress(),
+ DateTimeHelper.ISO_DATETIME_TAB_FORMAT));
+ sb.append("\t" + (this.aSNumber != null ? this.aSNumber : "null"));
+ } else {
+ sb.append("\tnull\tnull\tnull");
+ }
+ sb.append("\t" + (this.contact != null ? this.contact : ""));
+ sb.append("\t" + (this.recommendedVersion == null ? "null" :
+ this.recommendedVersion ? "true" : "false"));
+ if (this.familyFingerprints == null ||
+ this.familyFingerprints.isEmpty()) {
+ sb.append("\tnull");
+ } else {
+ sb.append("\t");
+ written = 0;
+ for (String familyFingerprint : this.familyFingerprints) {
+ sb.append((written++ > 0 ? ";" : "") + familyFingerprint);
+ }
+ }
+ return sb.toString();
+ }
+}
+
diff --git a/src/org/torproject/onionoo/docs/SummaryDocument.java b/src/org/torproject/onionoo/docs/SummaryDocument.java
new file mode 100644
index 0000000..0c71ae2
--- /dev/null
+++ b/src/org/torproject/onionoo/docs/SummaryDocument.java
@@ -0,0 +1,202 @@
+/* Copyright 2013--2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.docs;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.regex.Pattern;
+
+import org.apache.commons.codec.DecoderException;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.torproject.onionoo.util.DateTimeHelper;
+
+public class SummaryDocument extends Document {
+
+ private boolean t;
+ public void setRelay(boolean isRelay) {
+ this.t = isRelay;
+ }
+ public boolean isRelay() {
+ return this.t;
+ }
+
+ private String f;
+ public void setFingerprint(String fingerprint) {
+ if (fingerprint != null) {
+ Pattern fingerprintPattern = Pattern.compile("^[0-9a-fA-F]{40}$");
+ if (!fingerprintPattern.matcher(fingerprint).matches()) {
+ throw new IllegalArgumentException("Fingerprint '" + fingerprint
+ + "' is not a valid fingerprint.");
+ }
+ }
+ this.f = fingerprint;
+ }
+ public String getFingerprint() {
+ return this.f;
+ }
+
+ public String getHashedFingerprint() {
+ String hashedFingerprint = null;
+ if (this.f != null) {
+ try {
+ hashedFingerprint = DigestUtils.shaHex(Hex.decodeHex(
+ this.f.toCharArray())).toUpperCase();
+ } catch (DecoderException e) {
+ /* Format tested in setFingerprint(). */
+ }
+ }
+ return hashedFingerprint;
+ }
+
+ private String n;
+ public void setNickname(String nickname) {
+ if (nickname == null || nickname.equals("Unnamed")) {
+ this.n = null;
+ } else {
+ this.n = nickname;
+ }
+ }
+ public String getNickname() {
+ return this.n == null ? "Unnamed" : this.n;
+ }
+
+ private String[] ad;
+ public void setAddresses(List<String> addresses) {
+ this.ad = this.collectionToStringArray(addresses);
+ }
+ public List<String> getAddresses() {
+ return this.stringArrayToList(this.ad);
+ }
+
+ private String[] collectionToStringArray(
+ Collection<String> collection) {
+ String[] stringArray = null;
+ if (collection != null && !collection.isEmpty()) {
+ stringArray = new String[collection.size()];
+ int i = 0;
+ for (String string : collection) {
+ stringArray[i++] = string;
+ }
+ }
+ return stringArray;
+ }
+ private List<String> stringArrayToList(String[] stringArray) {
+ List<String> list;
+ if (stringArray == null) {
+ list = new ArrayList<String>();
+ } else {
+ list = Arrays.asList(stringArray);
+ }
+ return list;
+ }
+ private SortedSet<String> stringArrayToSortedSet(String[] stringArray) {
+ SortedSet<String> sortedSet = new TreeSet<String>();
+ if (stringArray != null) {
+ sortedSet.addAll(Arrays.asList(stringArray));
+ }
+ return sortedSet;
+ }
+
+ private String cc;
+ public void setCountryCode(String countryCode) {
+ this.cc = countryCode;
+ }
+ public String getCountryCode() {
+ return this.cc;
+ }
+
+ private String as;
+ public void setASNumber(String aSNumber) {
+ this.as = aSNumber;
+ }
+ public String getASNumber() {
+ return this.as;
+ }
+
+ private String fs;
+ public void setFirstSeenMillis(long firstSeenMillis) {
+ this.fs = DateTimeHelper.format(firstSeenMillis);
+ }
+ public long getFirstSeenMillis() {
+ return DateTimeHelper.parse(this.fs);
+ }
+
+ private String ls;
+ public void setLastSeenMillis(long lastSeenMillis) {
+ this.ls = DateTimeHelper.format(lastSeenMillis);
+ }
+ public long getLastSeenMillis() {
+ return DateTimeHelper.parse(this.ls);
+ }
+
+ private String[] rf;
+ public void setRelayFlags(SortedSet<String> relayFlags) {
+ this.rf = this.collectionToStringArray(relayFlags);
+ }
+ public SortedSet<String> getRelayFlags() {
+ return this.stringArrayToSortedSet(this.rf);
+ }
+
+ private long cw;
+ public void setConsensusWeight(long consensusWeight) {
+ this.cw = consensusWeight;
+ }
+ public long getConsensusWeight() {
+ return this.cw;
+ }
+
+ private boolean r;
+ public void setRunning(boolean isRunning) {
+ this.r = isRunning;
+ }
+ public boolean isRunning() {
+ return this.r;
+ }
+
+ private String c;
+ public void setContact(String contact) {
+ if (contact != null && contact.length() == 0) {
+ this.c = null;
+ } else {
+ this.c = contact;
+ }
+ }
+ public String getContact() {
+ return this.c;
+ }
+
+ private String[] ff;
+ public void setFamilyFingerprints(
+ SortedSet<String> familyFingerprints) {
+ this.ff = this.collectionToStringArray(familyFingerprints);
+ }
+ public SortedSet<String> getFamilyFingerprints() {
+ return this.stringArrayToSortedSet(this.ff);
+ }
+
+ public SummaryDocument(boolean isRelay, String nickname,
+ String fingerprint, List<String> addresses, long lastSeenMillis,
+ boolean running, SortedSet<String> relayFlags, long consensusWeight,
+ String countryCode, long firstSeenMillis, String aSNumber,
+ String contact, SortedSet<String> familyFingerprints) {
+ this.setRelay(isRelay);
+ this.setNickname(nickname);
+ this.setFingerprint(fingerprint);
+ this.setAddresses(addresses);
+ this.setLastSeenMillis(lastSeenMillis);
+ this.setRunning(running);
+ this.setRelayFlags(relayFlags);
+ this.setConsensusWeight(consensusWeight);
+ this.setCountryCode(countryCode);
+ this.setFirstSeenMillis(firstSeenMillis);
+ this.setASNumber(aSNumber);
+ this.setContact(contact);
+ this.setFamilyFingerprints(familyFingerprints);
+ }
+}
+
diff --git a/src/org/torproject/onionoo/docs/UpdateStatus.java b/src/org/torproject/onionoo/docs/UpdateStatus.java
new file mode 100644
index 0000000..7bd710b
--- /dev/null
+++ b/src/org/torproject/onionoo/docs/UpdateStatus.java
@@ -0,0 +1,7 @@
+/* Copyright 2013 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.docs;
+
+public class UpdateStatus extends Document {
+}
+
diff --git a/src/org/torproject/onionoo/docs/UptimeDocument.java b/src/org/torproject/onionoo/docs/UptimeDocument.java
new file mode 100644
index 0000000..7f0bacc
--- /dev/null
+++ b/src/org/torproject/onionoo/docs/UptimeDocument.java
@@ -0,0 +1,23 @@
+/* Copyright 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.docs;
+
+import java.util.Map;
+
+public class UptimeDocument extends Document {
+
+ @SuppressWarnings("unused")
+ private String fingerprint;
+ public void setFingerprint(String fingerprint) {
+ this.fingerprint = fingerprint;
+ }
+
+ private Map<String, GraphHistory> uptime;
+ public void setUptime(Map<String, GraphHistory> uptime) {
+ this.uptime = uptime;
+ }
+ public Map<String, GraphHistory> getUptime() {
+ return this.uptime;
+ }
+}
+
diff --git a/src/org/torproject/onionoo/docs/UptimeHistory.java b/src/org/torproject/onionoo/docs/UptimeHistory.java
new file mode 100644
index 0000000..f0a966b
--- /dev/null
+++ b/src/org/torproject/onionoo/docs/UptimeHistory.java
@@ -0,0 +1,90 @@
+package org.torproject.onionoo.docs;
+
+import org.torproject.onionoo.util.DateTimeHelper;
+
+public class UptimeHistory
+ implements Comparable<UptimeHistory> {
+
+ private boolean relay;
+ public boolean isRelay() {
+ return this.relay;
+ }
+
+ private long startMillis;
+ public long getStartMillis() {
+ return this.startMillis;
+ }
+
+ private int uptimeHours;
+ public int getUptimeHours() {
+ return this.uptimeHours;
+ }
+
+ UptimeHistory(boolean relay, long startMillis,
+ int uptimeHours) {
+ this.relay = relay;
+ this.startMillis = startMillis;
+ this.uptimeHours = uptimeHours;
+ }
+
+ public static UptimeHistory fromString(String uptimeHistoryString) {
+ String[] parts = uptimeHistoryString.split(" ", 3);
+ if (parts.length != 3) {
+ return null;
+ }
+ boolean relay = false;
+ if (parts[0].equals("r")) {
+ relay = true;
+ } else if (!parts[0].equals("b")) {
+ return null;
+ }
+ long startMillis = DateTimeHelper.parse(parts[1],
+ DateTimeHelper.DATEHOUR_NOSPACE_FORMAT);
+ if (startMillis < 0L) {
+ return null;
+ }
+ int uptimeHours = -1;
+ try {
+ uptimeHours = Integer.parseInt(parts[2]);
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ return new UptimeHistory(relay, startMillis, uptimeHours);
+ }
+
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(this.relay ? "r" : "b");
+ sb.append(" " + DateTimeHelper.format(this.startMillis,
+ DateTimeHelper.DATEHOUR_NOSPACE_FORMAT));
+ sb.append(" " + String.format("%d", this.uptimeHours));
+ return sb.toString();
+ }
+
+ public void addUptime(UptimeHistory other) {
+ this.uptimeHours += other.uptimeHours;
+ if (this.startMillis > other.startMillis) {
+ this.startMillis = other.startMillis;
+ }
+ }
+
+ public int compareTo(UptimeHistory other) {
+ if (this.relay && !other.relay) {
+ return -1;
+ } else if (!this.relay && other.relay) {
+ return 1;
+ }
+ return this.startMillis < other.startMillis ? -1 :
+ this.startMillis > other.startMillis ? 1 : 0;
+ }
+
+ public boolean equals(Object other) {
+ return other instanceof UptimeHistory &&
+ this.relay == ((UptimeHistory) other).relay &&
+ this.startMillis == ((UptimeHistory) other).startMillis;
+ }
+
+ public int hashCode() {
+ return (int) this.startMillis + (this.relay ? 1 : 0);
+ }
+}
\ No newline at end of file
diff --git a/src/org/torproject/onionoo/docs/UptimeStatus.java b/src/org/torproject/onionoo/docs/UptimeStatus.java
new file mode 100644
index 0000000..1da11f0
--- /dev/null
+++ b/src/org/torproject/onionoo/docs/UptimeStatus.java
@@ -0,0 +1,142 @@
+/* Copyright 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.docs;
+
+import java.util.Scanner;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.torproject.onionoo.util.ApplicationFactory;
+import org.torproject.onionoo.util.DateTimeHelper;
+
+public class UptimeStatus extends Document {
+
+ private transient String fingerprint;
+
+ private transient boolean isDirty = false;
+
+ private SortedSet<UptimeHistory> relayHistory =
+ new TreeSet<UptimeHistory>();
+ public void setRelayHistory(SortedSet<UptimeHistory> relayHistory) {
+ this.relayHistory = relayHistory;
+ }
+ public SortedSet<UptimeHistory> getRelayHistory() {
+ return this.relayHistory;
+ }
+
+ private SortedSet<UptimeHistory> bridgeHistory =
+ new TreeSet<UptimeHistory>();
+ public void setBridgeHistory(SortedSet<UptimeHistory> bridgeHistory) {
+ this.bridgeHistory = bridgeHistory;
+ }
+ public SortedSet<UptimeHistory> getBridgeHistory() {
+ return this.bridgeHistory;
+ }
+
+ public static UptimeStatus loadOrCreate(String fingerprint) {
+ UptimeStatus uptimeStatus = (fingerprint == null) ?
+ ApplicationFactory.getDocumentStore().retrieve(
+ UptimeStatus.class, true) :
+ ApplicationFactory.getDocumentStore().retrieve(
+ UptimeStatus.class, true, fingerprint);
+ if (uptimeStatus == null) {
+ uptimeStatus = new UptimeStatus();
+ }
+ uptimeStatus.fingerprint = fingerprint;
+ return uptimeStatus;
+ }
+
+ public void fromDocumentString(String documentString) {
+ Scanner s = new Scanner(documentString);
+ while (s.hasNextLine()) {
+ String line = s.nextLine();
+ UptimeHistory parsedLine = UptimeHistory.fromString(line);
+ if (parsedLine != null) {
+ if (parsedLine.isRelay()) {
+ this.relayHistory.add(parsedLine);
+ } else {
+ this.bridgeHistory.add(parsedLine);
+ }
+ } else {
+ System.err.println("Could not parse uptime history line '"
+ + line + "'. Skipping.");
+ }
+ }
+ s.close();
+ }
+
+ public void addToHistory(boolean relay, SortedSet<Long> newIntervals) {
+ for (long startMillis : newIntervals) {
+ SortedSet<UptimeHistory> history = relay ? this.relayHistory
+ : this.bridgeHistory;
+ UptimeHistory interval = new UptimeHistory(relay, startMillis, 1);
+ if (!history.headSet(interval).isEmpty()) {
+ UptimeHistory prev = history.headSet(interval).last();
+ if (prev.isRelay() == interval.isRelay() &&
+ prev.getStartMillis() + DateTimeHelper.ONE_HOUR
+ * prev.getUptimeHours() > interval.getStartMillis()) {
+ continue;
+ }
+ }
+ if (!history.tailSet(interval).isEmpty()) {
+ UptimeHistory next = history.tailSet(interval).first();
+ if (next.isRelay() == interval.isRelay() &&
+ next.getStartMillis() < interval.getStartMillis()
+ + DateTimeHelper.ONE_HOUR) {
+ continue;
+ }
+ }
+ history.add(interval);
+ this.isDirty = true;
+ }
+ }
+
+ public void storeIfChanged() {
+ if (this.isDirty) {
+ this.compressHistory(this.relayHistory);
+ this.compressHistory(this.bridgeHistory);
+ if (fingerprint == null) {
+ ApplicationFactory.getDocumentStore().store(this);
+ } else {
+ ApplicationFactory.getDocumentStore().store(this,
+ this.fingerprint);
+ }
+ this.isDirty = false;
+ }
+ }
+
+ private void compressHistory(SortedSet<UptimeHistory> history) {
+ SortedSet<UptimeHistory> uncompressedHistory =
+ new TreeSet<UptimeHistory>(history);
+ history.clear();
+ UptimeHistory lastInterval = null;
+ for (UptimeHistory interval : uncompressedHistory) {
+ if (lastInterval != null &&
+ lastInterval.getStartMillis() + DateTimeHelper.ONE_HOUR
+ * lastInterval.getUptimeHours() == interval.getStartMillis() &&
+ lastInterval.isRelay() == interval.isRelay()) {
+ lastInterval.addUptime(interval);
+ } else {
+ if (lastInterval != null) {
+ history.add(lastInterval);
+ }
+ lastInterval = interval;
+ }
+ }
+ if (lastInterval != null) {
+ history.add(lastInterval);
+ }
+ }
+
+ public String toDocumentString() {
+ StringBuilder sb = new StringBuilder();
+ for (UptimeHistory interval : this.relayHistory) {
+ sb.append(interval.toString() + "\n");
+ }
+ for (UptimeHistory interval : this.bridgeHistory) {
+ sb.append(interval.toString() + "\n");
+ }
+ return sb.toString();
+ }
+}
+
diff --git a/src/org/torproject/onionoo/docs/WeightsDocument.java b/src/org/torproject/onionoo/docs/WeightsDocument.java
new file mode 100644
index 0000000..104b661
--- /dev/null
+++ b/src/org/torproject/onionoo/docs/WeightsDocument.java
@@ -0,0 +1,64 @@
+/* Copyright 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.docs;
+
+import java.util.Map;
+
+public class WeightsDocument extends Document {
+
+ @SuppressWarnings("unused")
+ private String fingerprint;
+ public void setFingerprint(String fingerprint) {
+ this.fingerprint = fingerprint;
+ }
+
+ @SuppressWarnings("unused")
+ private Map<String, GraphHistory> advertised_bandwidth_fraction;
+ public void setAdvertisedBandwidthFraction(
+ Map<String, GraphHistory> advertisedBandwidthFraction) {
+ this.advertised_bandwidth_fraction = advertisedBandwidthFraction;
+ }
+
+ @SuppressWarnings("unused")
+ private Map<String, GraphHistory> consensus_weight_fraction;
+ public void setConsensusWeightFraction(
+ Map<String, GraphHistory> consensusWeightFraction) {
+ this.consensus_weight_fraction = consensusWeightFraction;
+ }
+
+ @SuppressWarnings("unused")
+ private Map<String, GraphHistory> guard_probability;
+ public void setGuardProbability(
+ Map<String, GraphHistory> guardProbability) {
+ this.guard_probability = guardProbability;
+ }
+
+ @SuppressWarnings("unused")
+ private Map<String, GraphHistory> middle_probability;
+ public void setMiddleProbability(
+ Map<String, GraphHistory> middleProbability) {
+ this.middle_probability = middleProbability;
+ }
+
+ @SuppressWarnings("unused")
+ private Map<String, GraphHistory> exit_probability;
+ public void setExitProbability(
+ Map<String, GraphHistory> exitProbability) {
+ this.exit_probability = exitProbability;
+ }
+
+ @SuppressWarnings("unused")
+ private Map<String, GraphHistory> advertised_bandwidth;
+ public void setAdvertisedBandwidth(
+ Map<String, GraphHistory> advertisedBandwidth) {
+ this.advertised_bandwidth = advertisedBandwidth;
+ }
+
+ @SuppressWarnings("unused")
+ private Map<String, GraphHistory> consensus_weight;
+ public void setConsensusWeight(
+ Map<String, GraphHistory> consensusWeight) {
+ this.consensus_weight = consensusWeight;
+ }
+}
+
diff --git a/src/org/torproject/onionoo/docs/WeightsStatus.java b/src/org/torproject/onionoo/docs/WeightsStatus.java
new file mode 100644
index 0000000..678789b
--- /dev/null
+++ b/src/org/torproject/onionoo/docs/WeightsStatus.java
@@ -0,0 +1,99 @@
+package org.torproject.onionoo.docs;
+
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Scanner;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.torproject.onionoo.util.DateTimeHelper;
+
+public class WeightsStatus extends Document {
+
+ private SortedMap<long[], double[]> history =
+ new TreeMap<long[], double[]>(new Comparator<long[]>() {
+ public int compare(long[] a, long[] b) {
+ return a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0;
+ }
+ });
+ public void setHistory(SortedMap<long[], double[]> history) {
+ this.history = history;
+ }
+ public SortedMap<long[], double[]> getHistory() {
+ return this.history;
+ }
+
+ private Map<String, Integer> advertisedBandwidths =
+ new HashMap<String, Integer>();
+ public Map<String, Integer> getAdvertisedBandwidths() {
+ return this.advertisedBandwidths;
+ }
+
+ public void fromDocumentString(String documentString) {
+ Scanner s = new Scanner(documentString);
+ while (s.hasNextLine()) {
+ String line = s.nextLine();
+ String[] parts = line.split(" ");
+ if (parts.length == 2) {
+ String descriptorDigest = parts[0];
+ int advertisedBandwidth = Integer.parseInt(parts[1]);
+ this.advertisedBandwidths.put(descriptorDigest,
+ advertisedBandwidth);
+ continue;
+ }
+ if (parts.length != 9 && parts.length != 11) {
+ System.err.println("Illegal line '" + line + "' in weights "
+ + "status file. Skipping this line.");
+ continue;
+ }
+ if (parts[4].equals("NaN")) {
+ /* Remove corrupt lines written on 2013-07-07 and the days
+ * after. */
+ continue;
+ }
+ long validAfterMillis = DateTimeHelper.parse(parts[0] + " "
+ + parts[1]);
+ long freshUntilMillis = DateTimeHelper.parse(parts[2] + " "
+ + parts[3]);
+ if (validAfterMillis < 0L || freshUntilMillis < 0L) {
+ System.err.println("Could not parse timestamp while reading "
+ + "weights status file. Skipping.");
+ break;
+ }
+ long[] interval = new long[] { validAfterMillis, freshUntilMillis };
+ double[] weights = new double[] {
+ Double.parseDouble(parts[4]),
+ Double.parseDouble(parts[5]),
+ Double.parseDouble(parts[6]),
+ Double.parseDouble(parts[7]),
+ Double.parseDouble(parts[8]), -1.0, -1.0 };
+ if (parts.length == 11) {
+ weights[5] = Double.parseDouble(parts[9]);
+ weights[6] = Double.parseDouble(parts[10]);
+ }
+ this.history.put(interval, weights);
+ }
+ s.close();
+ }
+
+ public String toDocumentString() {
+ StringBuilder sb = new StringBuilder();
+ for (Map.Entry<String, Integer> e :
+ this.advertisedBandwidths.entrySet()) {
+ sb.append(e.getKey() + " " + String.valueOf(e.getValue()) + "\n");
+ }
+ for (Map.Entry<long[], double[]> e : history.entrySet()) {
+ long[] fresh = e.getKey();
+ double[] weights = e.getValue();
+ sb.append(DateTimeHelper.format(fresh[0]) + " "
+ + DateTimeHelper.format(fresh[1]));
+ for (double weight : weights) {
+ sb.append(String.format(" %.12f", weight));
+ }
+ sb.append("\n");
+ }
+ return sb.toString();
+ }
+}
+
diff --git a/src/org/torproject/onionoo/server/NodeIndexer.java b/src/org/torproject/onionoo/server/NodeIndexer.java
new file mode 100644
index 0000000..b76f4c1
--- /dev/null
+++ b/src/org/torproject/onionoo/server/NodeIndexer.java
@@ -0,0 +1,432 @@
+package org.torproject.onionoo.server;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+
+import javax.servlet.ServletContext;
+import javax.servlet.ServletContextEvent;
+import javax.servlet.ServletContextListener;
+
+import org.torproject.onionoo.docs.DocumentStore;
+import org.torproject.onionoo.docs.SummaryDocument;
+import org.torproject.onionoo.docs.UpdateStatus;
+import org.torproject.onionoo.util.ApplicationFactory;
+import org.torproject.onionoo.util.DateTimeHelper;
+import org.torproject.onionoo.util.Time;
+
+class NodeIndex {
+
+ private String relaysPublishedString;
+ public void setRelaysPublishedString(String relaysPublishedString) {
+ this.relaysPublishedString = relaysPublishedString;
+ }
+ public String getRelaysPublishedString() {
+ return relaysPublishedString;
+ }
+
+ private String bridgesPublishedString;
+ public void setBridgesPublishedString(String bridgesPublishedString) {
+ this.bridgesPublishedString = bridgesPublishedString;
+ }
+ public String getBridgesPublishedString() {
+ return bridgesPublishedString;
+ }
+
+ private List<String> relaysByConsensusWeight;
+ public void setRelaysByConsensusWeight(
+ List<String> relaysByConsensusWeight) {
+ this.relaysByConsensusWeight = relaysByConsensusWeight;
+ }
+ public List<String> getRelaysByConsensusWeight() {
+ return relaysByConsensusWeight;
+ }
+
+
+ private Map<String, SummaryDocument> relayFingerprintSummaryLines;
+ public void setRelayFingerprintSummaryLines(
+ Map<String, SummaryDocument> relayFingerprintSummaryLines) {
+ this.relayFingerprintSummaryLines = relayFingerprintSummaryLines;
+ }
+ public Map<String, SummaryDocument> getRelayFingerprintSummaryLines() {
+ return this.relayFingerprintSummaryLines;
+ }
+
+ private Map<String, SummaryDocument> bridgeFingerprintSummaryLines;
+ public void setBridgeFingerprintSummaryLines(
+ Map<String, SummaryDocument> bridgeFingerprintSummaryLines) {
+ this.bridgeFingerprintSummaryLines = bridgeFingerprintSummaryLines;
+ }
+ public Map<String, SummaryDocument> getBridgeFingerprintSummaryLines() {
+ return this.bridgeFingerprintSummaryLines;
+ }
+
+ private Map<String, Set<String>> relaysByCountryCode = null;
+ public void setRelaysByCountryCode(
+ Map<String, Set<String>> relaysByCountryCode) {
+ this.relaysByCountryCode = relaysByCountryCode;
+ }
+ public Map<String, Set<String>> getRelaysByCountryCode() {
+ return relaysByCountryCode;
+ }
+
+ private Map<String, Set<String>> relaysByASNumber = null;
+ public void setRelaysByASNumber(
+ Map<String, Set<String>> relaysByASNumber) {
+ this.relaysByASNumber = relaysByASNumber;
+ }
+ public Map<String, Set<String>> getRelaysByASNumber() {
+ return relaysByASNumber;
+ }
+
+ private Map<String, Set<String>> relaysByFlag = null;
+ public void setRelaysByFlag(Map<String, Set<String>> relaysByFlag) {
+ this.relaysByFlag = relaysByFlag;
+ }
+ public Map<String, Set<String>> getRelaysByFlag() {
+ return relaysByFlag;
+ }
+
+ private Map<String, Set<String>> bridgesByFlag = null;
+ public void setBridgesByFlag(Map<String, Set<String>> bridgesByFlag) {
+ this.bridgesByFlag = bridgesByFlag;
+ }
+ public Map<String, Set<String>> getBridgesByFlag() {
+ return bridgesByFlag;
+ }
+
+ private Map<String, Set<String>> relaysByContact = null;
+ public void setRelaysByContact(
+ Map<String, Set<String>> relaysByContact) {
+ this.relaysByContact = relaysByContact;
+ }
+ public Map<String, Set<String>> getRelaysByContact() {
+ return relaysByContact;
+ }
+
+ private Map<String, Set<String>> relaysByFamily = null;
+ public void setRelaysByFamily(Map<String, Set<String>> relaysByFamily) {
+ this.relaysByFamily = relaysByFamily;
+ }
+ public Map<String, Set<String>> getRelaysByFamily() {
+ return this.relaysByFamily;
+ }
+
+ private SortedMap<Integer, Set<String>> relaysByFirstSeenDays;
+ public void setRelaysByFirstSeenDays(
+ SortedMap<Integer, Set<String>> relaysByFirstSeenDays) {
+ this.relaysByFirstSeenDays = relaysByFirstSeenDays;
+ }
+ public SortedMap<Integer, Set<String>> getRelaysByFirstSeenDays() {
+ return relaysByFirstSeenDays;
+ }
+
+ private SortedMap<Integer, Set<String>> bridgesByFirstSeenDays;
+ public void setBridgesByFirstSeenDays(
+ SortedMap<Integer, Set<String>> bridgesByFirstSeenDays) {
+ this.bridgesByFirstSeenDays = bridgesByFirstSeenDays;
+ }
+ public SortedMap<Integer, Set<String>> getBridgesByFirstSeenDays() {
+ return bridgesByFirstSeenDays;
+ }
+
+ private SortedMap<Integer, Set<String>> relaysByLastSeenDays;
+ public void setRelaysByLastSeenDays(
+ SortedMap<Integer, Set<String>> relaysByLastSeenDays) {
+ this.relaysByLastSeenDays = relaysByLastSeenDays;
+ }
+ public SortedMap<Integer, Set<String>> getRelaysByLastSeenDays() {
+ return relaysByLastSeenDays;
+ }
+
+ private SortedMap<Integer, Set<String>> bridgesByLastSeenDays;
+ public void setBridgesByLastSeenDays(
+ SortedMap<Integer, Set<String>> bridgesByLastSeenDays) {
+ this.bridgesByLastSeenDays = bridgesByLastSeenDays;
+ }
+ public SortedMap<Integer, Set<String>> getBridgesByLastSeenDays() {
+ return bridgesByLastSeenDays;
+ }
+}
+
+public class NodeIndexer implements ServletContextListener, Runnable {
+
+ public void contextInitialized(ServletContextEvent contextEvent) {
+ ServletContext servletContext = contextEvent.getServletContext();
+ File outDir = new File(servletContext.getInitParameter("outDir"));
+ DocumentStore documentStore = ApplicationFactory.getDocumentStore();
+ documentStore.setOutDir(outDir);
+ /* The servlet container created us, and we need to avoid that
+ * ApplicationFactory creates another instance of us. */
+ ApplicationFactory.setNodeIndexer(this);
+ this.startIndexing();
+ }
+
+ public void contextDestroyed(ServletContextEvent contextEvent) {
+ this.stopIndexing();
+ }
+
+ private long lastIndexed = -1L;
+
+ private NodeIndex latestNodeIndex = null;
+
+ private Thread nodeIndexerThread = null;
+
+ public synchronized long getLastIndexed(long timeoutMillis) {
+ if (this.lastIndexed == -1L && this.nodeIndexerThread != null &&
+ timeoutMillis > 0L) {
+ try {
+ this.wait(timeoutMillis);
+ } catch (InterruptedException e) {
+ }
+ }
+ return this.lastIndexed;
+ }
+
+ public synchronized NodeIndex getLatestNodeIndex(long timeoutMillis) {
+ if (this.latestNodeIndex == null && this.nodeIndexerThread != null &&
+ timeoutMillis > 0L) {
+ try {
+ this.wait(timeoutMillis);
+ } catch (InterruptedException e) {
+ }
+ }
+ return this.latestNodeIndex;
+ }
+
+ public synchronized void startIndexing() {
+ if (this.nodeIndexerThread == null) {
+ this.nodeIndexerThread = new Thread(this);
+ this.nodeIndexerThread.setDaemon(true);
+ this.nodeIndexerThread.start();
+ }
+ }
+
+ public void run() {
+ while (this.nodeIndexerThread != null) {
+ this.indexNodeStatuses();
+ try {
+ Thread.sleep(DateTimeHelper.ONE_MINUTE);
+ } catch (InterruptedException e) {
+ }
+ }
+ }
+
+ public synchronized void stopIndexing() {
+ Thread indexerThread = this.nodeIndexerThread;
+ this.nodeIndexerThread = null;
+ indexerThread.interrupt();
+ }
+
+ private void indexNodeStatuses() {
+ long updateStatusMillis = -1L;
+ DocumentStore documentStore = ApplicationFactory.getDocumentStore();
+ UpdateStatus updateStatus = documentStore.retrieve(UpdateStatus.class,
+ false);
+ if (updateStatus != null &&
+ updateStatus.getDocumentString() != null) {
+ String updateString = updateStatus.getDocumentString();
+ try {
+ updateStatusMillis = Long.parseLong(updateString.trim());
+ } catch (NumberFormatException e) {
+ /* Handle below. */
+ }
+ }
+ synchronized (this) {
+ if (updateStatusMillis <= this.lastIndexed) {
+ return;
+ }
+ }
+ List<String> newRelaysByConsensusWeight = new ArrayList<String>();
+ Map<String, SummaryDocument>
+ newRelayFingerprintSummaryLines =
+ new HashMap<String, SummaryDocument>(),
+ newBridgeFingerprintSummaryLines =
+ new HashMap<String, SummaryDocument>();
+ Map<String, Set<String>>
+ newRelaysByCountryCode = new HashMap<String, Set<String>>(),
+ newRelaysByASNumber = new HashMap<String, Set<String>>(),
+ newRelaysByFlag = new HashMap<String, Set<String>>(),
+ newBridgesByFlag = new HashMap<String, Set<String>>(),
+ newRelaysByContact = new HashMap<String, Set<String>>(),
+ newRelaysByFamily = new HashMap<String, Set<String>>();
+ SortedMap<Integer, Set<String>>
+ newRelaysByFirstSeenDays = new TreeMap<Integer, Set<String>>(),
+ newBridgesByFirstSeenDays = new TreeMap<Integer, Set<String>>(),
+ newRelaysByLastSeenDays = new TreeMap<Integer, Set<String>>(),
+ newBridgesByLastSeenDays = new TreeMap<Integer, Set<String>>();
+ Set<SummaryDocument> currentRelays = new HashSet<SummaryDocument>(),
+ currentBridges = new HashSet<SummaryDocument>();
+ SortedSet<String> fingerprints = documentStore.list(
+ SummaryDocument.class);
+ long relaysLastValidAfterMillis = 0L, bridgesLastPublishedMillis = 0L;
+ for (String fingerprint : fingerprints) {
+ SummaryDocument node = documentStore.retrieve(SummaryDocument.class,
+ true, fingerprint);
+ if (node.isRelay()) {
+ relaysLastValidAfterMillis = Math.max(
+ relaysLastValidAfterMillis, node.getLastSeenMillis());
+ currentRelays.add(node);
+ } else {
+ bridgesLastPublishedMillis = Math.max(
+ bridgesLastPublishedMillis, node.getLastSeenMillis());
+ currentBridges.add(node);
+ }
+ }
+ Time time = ApplicationFactory.getTime();
+ List<String> orderRelaysByConsensusWeight = new ArrayList<String>();
+ for (SummaryDocument entry : currentRelays) {
+ String fingerprint = entry.getFingerprint().toUpperCase();
+ String hashedFingerprint = entry.getHashedFingerprint().
+ toUpperCase();
+ newRelayFingerprintSummaryLines.put(fingerprint, entry);
+ newRelayFingerprintSummaryLines.put(hashedFingerprint, entry);
+ long consensusWeight = entry.getConsensusWeight();
+ orderRelaysByConsensusWeight.add(String.format("%020d %s",
+ consensusWeight, fingerprint));
+ orderRelaysByConsensusWeight.add(String.format("%020d %s",
+ consensusWeight, hashedFingerprint));
+ if (entry.getCountryCode() != null) {
+ String countryCode = entry.getCountryCode();
+ if (!newRelaysByCountryCode.containsKey(countryCode)) {
+ newRelaysByCountryCode.put(countryCode,
+ new HashSet<String>());
+ }
+ newRelaysByCountryCode.get(countryCode).add(fingerprint);
+ newRelaysByCountryCode.get(countryCode).add(hashedFingerprint);
+ }
+ if (entry.getASNumber() != null) {
+ String aSNumber = entry.getASNumber();
+ if (!newRelaysByASNumber.containsKey(aSNumber)) {
+ newRelaysByASNumber.put(aSNumber, new HashSet<String>());
+ }
+ newRelaysByASNumber.get(aSNumber).add(fingerprint);
+ newRelaysByASNumber.get(aSNumber).add(hashedFingerprint);
+ }
+ for (String flag : entry.getRelayFlags()) {
+ String flagLowerCase = flag.toLowerCase();
+ if (!newRelaysByFlag.containsKey(flagLowerCase)) {
+ newRelaysByFlag.put(flagLowerCase, new HashSet<String>());
+ }
+ newRelaysByFlag.get(flagLowerCase).add(fingerprint);
+ newRelaysByFlag.get(flagLowerCase).add(hashedFingerprint);
+ }
+ if (entry.getFamilyFingerprints() != null) {
+ newRelaysByFamily.put(fingerprint, entry.getFamilyFingerprints());
+ }
+ int daysSinceFirstSeen = (int) ((time.currentTimeMillis()
+ - entry.getFirstSeenMillis()) / DateTimeHelper.ONE_DAY);
+ if (!newRelaysByFirstSeenDays.containsKey(daysSinceFirstSeen)) {
+ newRelaysByFirstSeenDays.put(daysSinceFirstSeen,
+ new HashSet<String>());
+ }
+ newRelaysByFirstSeenDays.get(daysSinceFirstSeen).add(fingerprint);
+ newRelaysByFirstSeenDays.get(daysSinceFirstSeen).add(
+ hashedFingerprint);
+ int daysSinceLastSeen = (int) ((time.currentTimeMillis()
+ - entry.getLastSeenMillis()) / DateTimeHelper.ONE_DAY);
+ if (!newRelaysByLastSeenDays.containsKey(daysSinceLastSeen)) {
+ newRelaysByLastSeenDays.put(daysSinceLastSeen,
+ new HashSet<String>());
+ }
+ newRelaysByLastSeenDays.get(daysSinceLastSeen).add(fingerprint);
+ newRelaysByLastSeenDays.get(daysSinceLastSeen).add(
+ hashedFingerprint);
+ String contact = entry.getContact();
+ if (!newRelaysByContact.containsKey(contact)) {
+ newRelaysByContact.put(contact, new HashSet<String>());
+ }
+ newRelaysByContact.get(contact).add(fingerprint);
+ newRelaysByContact.get(contact).add(hashedFingerprint);
+ }
+ Collections.sort(orderRelaysByConsensusWeight);
+ newRelaysByConsensusWeight = new ArrayList<String>();
+ for (String relay : orderRelaysByConsensusWeight) {
+ newRelaysByConsensusWeight.add(relay.split(" ")[1]);
+ }
+ for (Map.Entry<String, Set<String>> e :
+ newRelaysByFamily.entrySet()) {
+ String fingerprint = e.getKey();
+ Set<String> inMutualFamilyRelation = new HashSet<String>();
+ for (String otherFingerprint : e.getValue()) {
+ if (newRelaysByFamily.containsKey(otherFingerprint) &&
+ newRelaysByFamily.get(otherFingerprint).contains(
+ fingerprint)) {
+ inMutualFamilyRelation.add(otherFingerprint);
+ }
+ }
+ e.getValue().retainAll(inMutualFamilyRelation);
+ }
+ for (SummaryDocument entry : currentBridges) {
+ String hashedFingerprint = entry.getFingerprint().toUpperCase();
+ String hashedHashedFingerprint = entry.getHashedFingerprint().
+ toUpperCase();
+ newBridgeFingerprintSummaryLines.put(hashedFingerprint, entry);
+ newBridgeFingerprintSummaryLines.put(hashedHashedFingerprint,
+ entry);
+ for (String flag : entry.getRelayFlags()) {
+ String flagLowerCase = flag.toLowerCase();
+ if (!newBridgesByFlag.containsKey(flagLowerCase)) {
+ newBridgesByFlag.put(flagLowerCase, new HashSet<String>());
+ }
+ newBridgesByFlag.get(flagLowerCase).add(hashedFingerprint);
+ newBridgesByFlag.get(flagLowerCase).add(
+ hashedHashedFingerprint);
+ }
+ int daysSinceFirstSeen = (int) ((time.currentTimeMillis()
+ - entry.getFirstSeenMillis()) / DateTimeHelper.ONE_DAY);
+ if (!newBridgesByFirstSeenDays.containsKey(daysSinceFirstSeen)) {
+ newBridgesByFirstSeenDays.put(daysSinceFirstSeen,
+ new HashSet<String>());
+ }
+ newBridgesByFirstSeenDays.get(daysSinceFirstSeen).add(
+ hashedFingerprint);
+ newBridgesByFirstSeenDays.get(daysSinceFirstSeen).add(
+ hashedHashedFingerprint);
+ int daysSinceLastSeen = (int) ((time.currentTimeMillis()
+ - entry.getLastSeenMillis()) / DateTimeHelper.ONE_DAY);
+ if (!newBridgesByLastSeenDays.containsKey(daysSinceLastSeen)) {
+ newBridgesByLastSeenDays.put(daysSinceLastSeen,
+ new HashSet<String>());
+ }
+ newBridgesByLastSeenDays.get(daysSinceLastSeen).add(
+ hashedFingerprint);
+ newBridgesByLastSeenDays.get(daysSinceLastSeen).add(
+ hashedHashedFingerprint);
+ }
+ NodeIndex newNodeIndex = new NodeIndex();
+ newNodeIndex.setRelaysByConsensusWeight(newRelaysByConsensusWeight);
+ newNodeIndex.setRelayFingerprintSummaryLines(
+ newRelayFingerprintSummaryLines);
+ newNodeIndex.setBridgeFingerprintSummaryLines(
+ newBridgeFingerprintSummaryLines);
+ newNodeIndex.setRelaysByCountryCode(newRelaysByCountryCode);
+ newNodeIndex.setRelaysByASNumber(newRelaysByASNumber);
+ newNodeIndex.setRelaysByFlag(newRelaysByFlag);
+ newNodeIndex.setBridgesByFlag(newBridgesByFlag);
+ newNodeIndex.setRelaysByContact(newRelaysByContact);
+ newNodeIndex.setRelaysByFamily(newRelaysByFamily);
+ newNodeIndex.setRelaysByFirstSeenDays(newRelaysByFirstSeenDays);
+ newNodeIndex.setRelaysByLastSeenDays(newRelaysByLastSeenDays);
+ newNodeIndex.setBridgesByFirstSeenDays(newBridgesByFirstSeenDays);
+ newNodeIndex.setBridgesByLastSeenDays(newBridgesByLastSeenDays);
+ newNodeIndex.setRelaysPublishedString(DateTimeHelper.format(
+ relaysLastValidAfterMillis));
+ newNodeIndex.setBridgesPublishedString(DateTimeHelper.format(
+ bridgesLastPublishedMillis));
+ synchronized (this) {
+ this.lastIndexed = updateStatusMillis;
+ this.latestNodeIndex = newNodeIndex;
+ this.notifyAll();
+ }
+ }
+}
+
diff --git a/src/org/torproject/onionoo/server/RequestHandler.java b/src/org/torproject/onionoo/server/RequestHandler.java
new file mode 100644
index 0000000..22e82fb
--- /dev/null
+++ b/src/org/torproject/onionoo/server/RequestHandler.java
@@ -0,0 +1,552 @@
+/* Copyright 2011--2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.server;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+
+import org.torproject.onionoo.docs.DocumentStore;
+import org.torproject.onionoo.docs.SummaryDocument;
+import org.torproject.onionoo.util.ApplicationFactory;
+
+public class RequestHandler {
+
+ private NodeIndex nodeIndex;
+
+ private DocumentStore documentStore;
+
+ public RequestHandler(NodeIndex nodeIndex) {
+ this.nodeIndex = nodeIndex;
+ this.documentStore = ApplicationFactory.getDocumentStore();
+ }
+
+ private String resourceType;
+ public void setResourceType(String resourceType) {
+ this.resourceType = resourceType;
+ }
+
+ private String type;
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ private String running;
+ public void setRunning(String running) {
+ this.running = running;
+ }
+
+ private String[] search;
+ public void setSearch(String[] search) {
+ this.search = new String[search.length];
+ System.arraycopy(search, 0, this.search, 0, search.length);
+ }
+
+ private String lookup;
+ public void setLookup(String lookup) {
+ this.lookup = lookup;
+ }
+
+ private String fingerprint;
+ public void setFingerprint(String fingerprint) {
+ this.fingerprint = fingerprint;
+ }
+
+ private String country;
+ public void setCountry(String country) {
+ this.country = country;
+ }
+
+ private String as;
+ public void setAs(String as) {
+ this.as = as;
+ }
+
+ private String flag;
+ public void setFlag(String flag) {
+ this.flag = flag;
+ }
+
+ private String[] contact;
+ public void setContact(String[] contact) {
+ this.contact = new String[contact.length];
+ System.arraycopy(contact, 0, this.contact, 0, contact.length);
+ }
+
+ private String[] order;
+ public void setOrder(String[] order) {
+ this.order = new String[order.length];
+ System.arraycopy(order, 0, this.order, 0, order.length);
+ }
+
+ private String offset;
+ public void setOffset(String offset) {
+ this.offset = offset;
+ }
+
+ private String limit;
+ public void setLimit(String limit) {
+ this.limit = limit;
+ }
+
+ private int[] firstSeenDays;
+ public void setFirstSeenDays(int[] firstSeenDays) {
+ this.firstSeenDays = new int[firstSeenDays.length];
+ System.arraycopy(firstSeenDays, 0, this.firstSeenDays, 0,
+ firstSeenDays.length);
+ }
+
+ private int[] lastSeenDays;
+ public void setLastSeenDays(int[] lastSeenDays) {
+ this.lastSeenDays = new int[lastSeenDays.length];
+ System.arraycopy(lastSeenDays, 0, this.lastSeenDays, 0,
+ lastSeenDays.length);
+ }
+
+ private String family;
+ public void setFamily(String family) {
+ this.family = family;
+ }
+
+ private Map<String, SummaryDocument> filteredRelays =
+ new HashMap<String, SummaryDocument>();
+
+ private Map<String, SummaryDocument> filteredBridges =
+ new HashMap<String, SummaryDocument>();
+
+ public void handleRequest() {
+ this.filteredRelays.putAll(
+ this.nodeIndex.getRelayFingerprintSummaryLines());
+ this.filteredBridges.putAll(
+ this.nodeIndex.getBridgeFingerprintSummaryLines());
+ this.filterByResourceType();
+ this.filterByType();
+ this.filterByRunning();
+ this.filterBySearchTerms();
+ this.filterByLookup();
+ this.filterByFingerprint();
+ this.filterByCountryCode();
+ this.filterByASNumber();
+ this.filterByFlag();
+ this.filterNodesByFirstSeenDays();
+ this.filterNodesByLastSeenDays();
+ this.filterByContact();
+ this.filterByFamily();
+ this.order();
+ this.offset();
+ this.limit();
+ }
+
+
+ private void filterByResourceType() {
+ if (this.resourceType.equals("clients")) {
+ this.filteredRelays.clear();
+ }
+ if (this.resourceType.equals("weights")) {
+ this.filteredBridges.clear();
+ }
+ }
+
+ private void filterByType() {
+ if (this.type == null) {
+ return;
+ } else if (this.type.equals("relay")) {
+ this.filteredBridges.clear();
+ } else {
+ this.filteredRelays.clear();
+ }
+ }
+
+ private void filterByRunning() {
+ if (this.running == null) {
+ return;
+ }
+ boolean runningRequested = this.running.equals("true");
+ Set<String> removeRelays = new HashSet<String>();
+ for (Map.Entry<String, SummaryDocument> e :
+ filteredRelays.entrySet()) {
+ if (e.getValue().isRunning() != runningRequested) {
+ removeRelays.add(e.getKey());
+ }
+ }
+ for (String fingerprint : removeRelays) {
+ this.filteredRelays.remove(fingerprint);
+ }
+ Set<String> removeBridges = new HashSet<String>();
+ for (Map.Entry<String, SummaryDocument> e :
+ filteredBridges.entrySet()) {
+ if (e.getValue().isRunning() != runningRequested) {
+ removeBridges.add(e.getKey());
+ }
+ }
+ for (String fingerprint : removeBridges) {
+ this.filteredBridges.remove(fingerprint);
+ }
+ }
+
+ private void filterBySearchTerms() {
+ if (this.search == null) {
+ return;
+ }
+ for (String searchTerm : this.search) {
+ filterBySearchTerm(searchTerm);
+ }
+ }
+
+ private void filterBySearchTerm(String searchTerm) {
+ Set<String> removeRelays = new HashSet<String>();
+ for (Map.Entry<String, SummaryDocument> e :
+ filteredRelays.entrySet()) {
+ String fingerprint = e.getKey();
+ SummaryDocument entry = e.getValue();
+ boolean lineMatches = false;
+ String nickname = entry.getNickname() != null ?
+ entry.getNickname().toLowerCase() : "unnamed";
+ if (searchTerm.startsWith("$")) {
+ /* Search is for $-prefixed fingerprint. */
+ if (fingerprint.startsWith(
+ searchTerm.substring(1).toUpperCase())) {
+ /* $-prefixed fingerprint matches. */
+ lineMatches = true;
+ }
+ } else if (nickname.contains(searchTerm.toLowerCase())) {
+ /* Nickname matches. */
+ lineMatches = true;
+ } else if (fingerprint.startsWith(searchTerm.toUpperCase())) {
+ /* Non-$-prefixed fingerprint matches. */
+ lineMatches = true;
+ } else {
+ List<String> addresses = entry.getAddresses();
+ for (String address : addresses) {
+ if (address.startsWith(searchTerm.toLowerCase())) {
+ /* Address matches. */
+ lineMatches = true;
+ break;
+ }
+ }
+ }
+ if (!lineMatches) {
+ removeRelays.add(e.getKey());
+ }
+ }
+ for (String fingerprint : removeRelays) {
+ this.filteredRelays.remove(fingerprint);
+ }
+ Set<String> removeBridges = new HashSet<String>();
+ for (Map.Entry<String, SummaryDocument> e :
+ filteredBridges.entrySet()) {
+ String hashedFingerprint = e.getKey();
+ SummaryDocument entry = e.getValue();
+ boolean lineMatches = false;
+ String nickname = entry.getNickname() != null ?
+ entry.getNickname().toLowerCase() : "unnamed";
+ if (searchTerm.startsWith("$")) {
+ /* Search is for $-prefixed hashed fingerprint. */
+ if (hashedFingerprint.startsWith(
+ searchTerm.substring(1).toUpperCase())) {
+ /* $-prefixed hashed fingerprint matches. */
+ lineMatches = true;
+ }
+ } else if (nickname.contains(searchTerm.toLowerCase())) {
+ /* Nickname matches. */
+ lineMatches = true;
+ } else if (hashedFingerprint.startsWith(searchTerm.toUpperCase())) {
+ /* Non-$-prefixed hashed fingerprint matches. */
+ lineMatches = true;
+ }
+ if (!lineMatches) {
+ removeBridges.add(e.getKey());
+ }
+ }
+ for (String fingerprint : removeBridges) {
+ this.filteredBridges.remove(fingerprint);
+ }
+ }
+
+ private void filterByLookup() {
+ if (this.lookup == null) {
+ return;
+ }
+ String fingerprint = this.lookup;
+ SummaryDocument relayLine = this.filteredRelays.get(fingerprint);
+ this.filteredRelays.clear();
+ if (relayLine != null) {
+ this.filteredRelays.put(fingerprint, relayLine);
+ }
+ SummaryDocument bridgeLine = this.filteredBridges.get(fingerprint);
+ this.filteredBridges.clear();
+ if (bridgeLine != null) {
+ this.filteredBridges.put(fingerprint, bridgeLine);
+ }
+ }
+
+ private void filterByFingerprint() {
+ if (this.fingerprint == null) {
+ return;
+ }
+ this.filteredRelays.clear();
+ this.filteredBridges.clear();
+ String fingerprint = this.fingerprint;
+ SummaryDocument entry = this.documentStore.retrieve(
+ SummaryDocument.class, true, fingerprint);
+ if (entry != null) {
+ if (entry.isRelay()) {
+ this.filteredRelays.put(fingerprint, entry);
+ } else {
+ this.filteredBridges.put(fingerprint, entry);
+ }
+ }
+ }
+
+ private void filterByCountryCode() {
+ if (this.country == null) {
+ return;
+ }
+ String countryCode = this.country.toLowerCase();
+ if (!this.nodeIndex.getRelaysByCountryCode().containsKey(
+ countryCode)) {
+ this.filteredRelays.clear();
+ } else {
+ Set<String> relaysWithCountryCode =
+ this.nodeIndex.getRelaysByCountryCode().get(countryCode);
+ Set<String> removeRelays = new HashSet<String>();
+ for (String fingerprint : this.filteredRelays.keySet()) {
+ if (!relaysWithCountryCode.contains(fingerprint)) {
+ removeRelays.add(fingerprint);
+ }
+ }
+ for (String fingerprint : removeRelays) {
+ this.filteredRelays.remove(fingerprint);
+ }
+ }
+ this.filteredBridges.clear();
+ }
+
+ private void filterByASNumber() {
+ if (this.as == null) {
+ return;
+ }
+ String aSNumber = this.as.toUpperCase();
+ if (!aSNumber.startsWith("AS")) {
+ aSNumber = "AS" + aSNumber;
+ }
+ if (!this.nodeIndex.getRelaysByASNumber().containsKey(aSNumber)) {
+ this.filteredRelays.clear();
+ } else {
+ Set<String> relaysWithASNumber =
+ this.nodeIndex.getRelaysByASNumber().get(aSNumber);
+ Set<String> removeRelays = new HashSet<String>();
+ for (String fingerprint : this.filteredRelays.keySet()) {
+ if (!relaysWithASNumber.contains(fingerprint)) {
+ removeRelays.add(fingerprint);
+ }
+ }
+ for (String fingerprint : removeRelays) {
+ this.filteredRelays.remove(fingerprint);
+ }
+ }
+ this.filteredBridges.clear();
+ }
+
+ private void filterByFlag() {
+ if (this.flag == null) {
+ return;
+ }
+ String flag = this.flag.toLowerCase();
+ if (!this.nodeIndex.getRelaysByFlag().containsKey(flag)) {
+ this.filteredRelays.clear();
+ } else {
+ Set<String> relaysWithFlag = this.nodeIndex.getRelaysByFlag().get(
+ flag);
+ Set<String> removeRelays = new HashSet<String>();
+ for (String fingerprint : this.filteredRelays.keySet()) {
+ if (!relaysWithFlag.contains(fingerprint)) {
+ removeRelays.add(fingerprint);
+ }
+ }
+ for (String fingerprint : removeRelays) {
+ this.filteredRelays.remove(fingerprint);
+ }
+ }
+ if (!this.nodeIndex.getBridgesByFlag().containsKey(flag)) {
+ this.filteredBridges.clear();
+ } else {
+ Set<String> bridgesWithFlag = this.nodeIndex.getBridgesByFlag().get(
+ flag);
+ Set<String> removeBridges = new HashSet<String>();
+ for (String fingerprint : this.filteredBridges.keySet()) {
+ if (!bridgesWithFlag.contains(fingerprint)) {
+ removeBridges.add(fingerprint);
+ }
+ }
+ for (String fingerprint : removeBridges) {
+ this.filteredBridges.remove(fingerprint);
+ }
+ }
+ }
+
+ private void filterNodesByFirstSeenDays() {
+ if (this.firstSeenDays == null) {
+ return;
+ }
+ filterNodesByDays(this.filteredRelays,
+ this.nodeIndex.getRelaysByFirstSeenDays(), this.firstSeenDays);
+ filterNodesByDays(this.filteredBridges,
+ this.nodeIndex.getBridgesByFirstSeenDays(), this.firstSeenDays);
+ }
+
+ private void filterNodesByLastSeenDays() {
+ if (this.lastSeenDays == null) {
+ return;
+ }
+ filterNodesByDays(this.filteredRelays,
+ this.nodeIndex.getRelaysByLastSeenDays(), this.lastSeenDays);
+ filterNodesByDays(this.filteredBridges,
+ this.nodeIndex.getBridgesByLastSeenDays(), this.lastSeenDays);
+ }
+
+ private void filterNodesByDays(
+ Map<String, SummaryDocument> filteredNodes,
+ SortedMap<Integer, Set<String>> nodesByDays, int[] days) {
+ Set<String> removeNodes = new HashSet<String>();
+ for (Set<String> nodes : nodesByDays.headMap(days[0]).values()) {
+ removeNodes.addAll(nodes);
+ }
+ if (days[1] < Integer.MAX_VALUE) {
+ for (Set<String> nodes :
+ nodesByDays.tailMap(days[1] + 1).values()) {
+ removeNodes.addAll(nodes);
+ }
+ }
+ for (String fingerprint : removeNodes) {
+ filteredNodes.remove(fingerprint);
+ }
+ }
+
+ private void filterByContact() {
+ if (this.contact == null) {
+ return;
+ }
+ Set<String> removeRelays = new HashSet<String>();
+ for (Map.Entry<String, Set<String>> e :
+ this.nodeIndex.getRelaysByContact().entrySet()) {
+ String contact = e.getKey();
+ for (String contactPart : this.contact) {
+ if (contact == null ||
+ !contact.contains(contactPart.toLowerCase())) {
+ removeRelays.addAll(e.getValue());
+ break;
+ }
+ }
+ }
+ for (String fingerprint : removeRelays) {
+ this.filteredRelays.remove(fingerprint);
+ }
+ this.filteredBridges.clear();
+ }
+
+ private void filterByFamily() {
+ if (this.family == null) {
+ return;
+ }
+ Set<String> removeRelays = new HashSet<String>(
+ this.filteredRelays.keySet());
+ removeRelays.remove(this.family);
+ if (this.nodeIndex.getRelaysByFamily().containsKey(this.family)) {
+ removeRelays.removeAll(this.nodeIndex.getRelaysByFamily().
+ get(this.family));
+ }
+ for (String fingerprint : removeRelays) {
+ this.filteredRelays.remove(fingerprint);
+ }
+ this.filteredBridges.clear();
+ }
+
+ private void order() {
+ if (this.order != null && this.order.length == 1) {
+ List<String> orderBy = new ArrayList<String>(
+ this.nodeIndex.getRelaysByConsensusWeight());
+ if (this.order[0].startsWith("-")) {
+ Collections.reverse(orderBy);
+ }
+ for (String relay : orderBy) {
+ if (this.filteredRelays.containsKey(relay) &&
+ !this.orderedRelays.contains(filteredRelays.get(relay))) {
+ this.orderedRelays.add(this.filteredRelays.remove(relay));
+ }
+ }
+ for (String relay : this.filteredRelays.keySet()) {
+ if (!this.orderedRelays.contains(this.filteredRelays.get(relay))) {
+ this.orderedRelays.add(this.filteredRelays.remove(relay));
+ }
+ }
+ Set<SummaryDocument> uniqueBridges = new HashSet<SummaryDocument>(
+ this.filteredBridges.values());
+ this.orderedBridges.addAll(uniqueBridges);
+ } else {
+ Set<SummaryDocument> uniqueRelays = new HashSet<SummaryDocument>(
+ this.filteredRelays.values());
+ this.orderedRelays.addAll(uniqueRelays);
+ Set<SummaryDocument> uniqueBridges = new HashSet<SummaryDocument>(
+ this.filteredBridges.values());
+ this.orderedBridges.addAll(uniqueBridges);
+ }
+ }
+
+ private void offset() {
+ if (this.offset == null) {
+ return;
+ }
+ int offsetValue = Integer.parseInt(this.offset);
+ while (offsetValue-- > 0 &&
+ (!this.orderedRelays.isEmpty() ||
+ !this.orderedBridges.isEmpty())) {
+ if (!this.orderedRelays.isEmpty()) {
+ this.orderedRelays.remove(0);
+ } else {
+ this.orderedBridges.remove(0);
+ }
+ }
+ }
+
+ private void limit() {
+ if (this.limit == null) {
+ return;
+ }
+ int limitValue = Integer.parseInt(this.limit);
+ while (!this.orderedRelays.isEmpty() &&
+ limitValue < this.orderedRelays.size()) {
+ this.orderedRelays.remove(this.orderedRelays.size() - 1);
+ }
+ limitValue -= this.orderedRelays.size();
+ while (!this.orderedBridges.isEmpty() &&
+ limitValue < this.orderedBridges.size()) {
+ this.orderedBridges.remove(this.orderedBridges.size() - 1);
+ }
+ }
+
+ private List<SummaryDocument> orderedRelays =
+ new ArrayList<SummaryDocument>();
+ public List<SummaryDocument> getOrderedRelays() {
+ return this.orderedRelays;
+ }
+
+ private List<SummaryDocument> orderedBridges =
+ new ArrayList<SummaryDocument>();
+ public List<SummaryDocument> getOrderedBridges() {
+ return this.orderedBridges;
+ }
+
+ public String getRelaysPublishedString() {
+ return this.nodeIndex.getRelaysPublishedString();
+ }
+
+ public String getBridgesPublishedString() {
+ return this.nodeIndex.getBridgesPublishedString();
+ }
+}
diff --git a/src/org/torproject/onionoo/server/ResourceServlet.java b/src/org/torproject/onionoo/server/ResourceServlet.java
new file mode 100644
index 0000000..1e05d12
--- /dev/null
+++ b/src/org/torproject/onionoo/server/ResourceServlet.java
@@ -0,0 +1,451 @@
+/* Copyright 2011, 2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.server;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.torproject.onionoo.util.ApplicationFactory;
+import org.torproject.onionoo.util.DateTimeHelper;
+
+public class ResourceServlet extends HttpServlet {
+
+ private static final long serialVersionUID = 7236658979947465319L;
+
+ private boolean maintenanceMode = false;
+
+ /* Called by servlet container, not by test class. */
+ public void init(ServletConfig config) throws ServletException {
+ super.init(config);
+ this.maintenanceMode =
+ config.getInitParameter("maintenance") != null &&
+ config.getInitParameter("maintenance").equals("1");
+ }
+
+ public long getLastModified(HttpServletRequest request) {
+ if (this.maintenanceMode) {
+ return super.getLastModified(request);
+ } else {
+ return ApplicationFactory.getNodeIndexer().getLastIndexed(
+ DateTimeHelper.TEN_SECONDS);
+ }
+ }
+
+ public static class HttpServletRequestWrapper {
+ private HttpServletRequest request;
+ protected HttpServletRequestWrapper(HttpServletRequest request) {
+ this.request = request;
+ }
+ protected String getRequestURI() {
+ return this.request.getRequestURI();
+ }
+ @SuppressWarnings("rawtypes")
+ protected Map getParameterMap() {
+ return this.request.getParameterMap();
+ }
+ protected String[] getParameterValues(String parameterKey) {
+ return this.request.getParameterValues(parameterKey);
+ }
+ }
+
+ public static class HttpServletResponseWrapper {
+ private HttpServletResponse response = null;
+ protected HttpServletResponseWrapper(HttpServletResponse response) {
+ this.response = response;
+ }
+ protected void sendError(int errorStatusCode) throws IOException {
+ this.response.sendError(errorStatusCode);
+ }
+ protected void setHeader(String headerName, String headerValue) {
+ this.response.setHeader(headerName, headerValue);
+ }
+ protected void setContentType(String contentType) {
+ this.response.setContentType(contentType);
+ }
+ protected void setCharacterEncoding(String characterEncoding) {
+ this.response.setCharacterEncoding(characterEncoding);
+ }
+ protected PrintWriter getWriter() throws IOException {
+ return this.response.getWriter();
+ }
+ }
+
+ public void doGet(HttpServletRequest request,
+ HttpServletResponse response) throws IOException, ServletException {
+ HttpServletRequestWrapper requestWrapper =
+ new HttpServletRequestWrapper(request);
+ HttpServletResponseWrapper responseWrapper =
+ new HttpServletResponseWrapper(response);
+ this.doGet(requestWrapper, responseWrapper);
+ }
+
+ public void doGet(HttpServletRequestWrapper request,
+ HttpServletResponseWrapper response) throws IOException {
+
+ if (this.maintenanceMode) {
+ response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
+ return;
+ }
+
+ if (ApplicationFactory.getNodeIndexer().getLastIndexed(
+ DateTimeHelper.TEN_SECONDS) + DateTimeHelper.SIX_HOURS
+ < ApplicationFactory.getTime().currentTimeMillis()) {
+ response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ return;
+ }
+
+ NodeIndex nodeIndex = ApplicationFactory.getNodeIndexer().
+ getLatestNodeIndex(DateTimeHelper.TEN_SECONDS);
+ if (nodeIndex == null) {
+ response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ return;
+ }
+
+ String uri = request.getRequestURI();
+ if (uri.startsWith("/onionoo/")) {
+ uri = uri.substring("/onionoo".length());
+ }
+ String resourceType = null;
+ if (uri.startsWith("/summary")) {
+ resourceType = "summary";
+ } else if (uri.startsWith("/details")) {
+ resourceType = "details";
+ } else if (uri.startsWith("/bandwidth")) {
+ resourceType = "bandwidth";
+ } else if (uri.startsWith("/weights")) {
+ resourceType = "weights";
+ } else if (uri.startsWith("/clients")) {
+ resourceType = "clients";
+ } else if (uri.startsWith("/uptime")) {
+ resourceType = "uptime";
+ } else {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+
+ RequestHandler rh = new RequestHandler(nodeIndex);
+ rh.setResourceType(resourceType);
+
+ /* Extract parameters either from the old-style URI or from request
+ * parameters. */
+ Map<String, String> parameterMap = new HashMap<String, String>();
+ for (Object parameterKey : request.getParameterMap().keySet()) {
+ String[] parameterValues =
+ request.getParameterValues((String) parameterKey);
+ parameterMap.put((String) parameterKey, parameterValues[0]);
+ }
+
+ /* Make sure that the request doesn't contain any unknown
+ * parameters. */
+ Set<String> knownParameters = new HashSet<String>(Arrays.asList((
+ "type,running,search,lookup,fingerprint,country,as,flag,"
+ + "first_seen_days,last_seen_days,contact,order,limit,offset,"
+ + "fields,family").split(",")));
+ for (String parameterKey : parameterMap.keySet()) {
+ if (!knownParameters.contains(parameterKey)) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ }
+
+ /* Filter relays and bridges matching the request. */
+ if (parameterMap.containsKey("type")) {
+ String typeParameterValue = parameterMap.get("type").toLowerCase();
+ boolean relaysRequested = true;
+ if (typeParameterValue.equals("bridge")) {
+ relaysRequested = false;
+ } else if (!typeParameterValue.equals("relay")) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ rh.setType(relaysRequested ? "relay" : "bridge");
+ }
+ if (parameterMap.containsKey("running")) {
+ String runningParameterValue =
+ parameterMap.get("running").toLowerCase();
+ boolean runningRequested = true;
+ if (runningParameterValue.equals("false")) {
+ runningRequested = false;
+ } else if (!runningParameterValue.equals("true")) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ rh.setRunning(runningRequested ? "true" : "false");
+ }
+ if (parameterMap.containsKey("search")) {
+ String[] searchTerms = this.parseSearchParameters(
+ parameterMap.get("search"));
+ if (searchTerms == null) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ rh.setSearch(searchTerms);
+ }
+ if (parameterMap.containsKey("lookup")) {
+ String lookupParameter = this.parseFingerprintParameter(
+ parameterMap.get("lookup"));
+ if (lookupParameter == null) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ String fingerprint = lookupParameter.toUpperCase();
+ rh.setLookup(fingerprint);
+ }
+ if (parameterMap.containsKey("fingerprint")) {
+ String fingerprintParameter = this.parseFingerprintParameter(
+ parameterMap.get("fingerprint"));
+ if (fingerprintParameter == null) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ String fingerprint = fingerprintParameter.toUpperCase();
+ rh.setFingerprint(fingerprint);
+ }
+ if (parameterMap.containsKey("country")) {
+ String countryCodeParameter = this.parseCountryCodeParameter(
+ parameterMap.get("country"));
+ if (countryCodeParameter == null) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ rh.setCountry(countryCodeParameter);
+ }
+ if (parameterMap.containsKey("as")) {
+ String aSNumberParameter = this.parseASNumberParameter(
+ parameterMap.get("as"));
+ if (aSNumberParameter == null) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ rh.setAs(aSNumberParameter);
+ }
+ if (parameterMap.containsKey("flag")) {
+ String flagParameter = this.parseFlagParameter(
+ parameterMap.get("flag"));
+ if (flagParameter == null) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ rh.setFlag(flagParameter);
+ }
+ if (parameterMap.containsKey("first_seen_days")) {
+ int[] days = this.parseDaysParameter(
+ parameterMap.get("first_seen_days"));
+ if (days == null) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ rh.setFirstSeenDays(days);
+ }
+ if (parameterMap.containsKey("last_seen_days")) {
+ int[] days = this.parseDaysParameter(
+ parameterMap.get("last_seen_days"));
+ if (days == null) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ rh.setLastSeenDays(days);
+ }
+ if (parameterMap.containsKey("contact")) {
+ String[] contactParts = this.parseContactParameter(
+ parameterMap.get("contact"));
+ if (contactParts == null) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ rh.setContact(contactParts);
+ }
+ if (parameterMap.containsKey("order")) {
+ String orderParameter = parameterMap.get("order").toLowerCase();
+ String orderByField = orderParameter;
+ if (orderByField.startsWith("-")) {
+ orderByField = orderByField.substring(1);
+ }
+ if (!orderByField.equals("consensus_weight")) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ rh.setOrder(new String[] { orderParameter });
+ }
+ if (parameterMap.containsKey("offset")) {
+ String offsetParameter = parameterMap.get("offset");
+ if (offsetParameter.length() > 6) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ try {
+ Integer.parseInt(offsetParameter);
+ } catch (NumberFormatException e) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ rh.setOffset(offsetParameter);
+ }
+ if (parameterMap.containsKey("limit")) {
+ String limitParameter = parameterMap.get("limit");
+ if (limitParameter.length() > 6) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ try {
+ Integer.parseInt(limitParameter);
+ } catch (NumberFormatException e) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ rh.setLimit(limitParameter);
+ }
+ if (parameterMap.containsKey("family")) {
+ String familyParameter = this.parseFingerprintParameter(
+ parameterMap.get("family"));
+ if (familyParameter == null) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ String family = familyParameter.toUpperCase();
+ rh.setFamily(family);
+ }
+ rh.handleRequest();
+
+ ResponseBuilder rb = new ResponseBuilder();
+ rb.setResourceType(resourceType);
+ rb.setRelaysPublishedString(rh.getRelaysPublishedString());
+ rb.setBridgesPublishedString(rh.getBridgesPublishedString());
+ rb.setOrderedRelays(rh.getOrderedRelays());
+ rb.setOrderedBridges(rh.getOrderedBridges());
+ String[] fields = null;
+ if (parameterMap.containsKey("fields")) {
+ fields = this.parseFieldsParameter(parameterMap.get("fields"));
+ if (fields == null) {
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
+ return;
+ }
+ rb.setFields(fields);
+ }
+
+ response.setHeader("Access-Control-Allow-Origin", "*");
+ response.setContentType("application/json");
+ response.setCharacterEncoding("utf-8");
+ PrintWriter pw = response.getWriter();
+ rb.buildResponse(pw);
+ pw.flush();
+ pw.close();
+ }
+
+ private static Pattern searchParameterPattern =
+ Pattern.compile("^\\$?[0-9a-fA-F]{1,40}$|" /* Fingerprint. */
+ + "^[0-9a-zA-Z\\.]{1,19}$|" /* Nickname or IPv4 address. */
+ + "^\\[[0-9a-fA-F:\\.]{1,39}\\]?$"); /* IPv6 address. */
+ private String[] parseSearchParameters(String parameter) {
+ String[] searchParameters;
+ if (parameter.contains(" ")) {
+ searchParameters = parameter.split(" ");
+ } else {
+ searchParameters = new String[] { parameter };
+ }
+ for (String searchParameter : searchParameters) {
+ if (!searchParameterPattern.matcher(searchParameter).matches()) {
+ return null;
+ }
+ }
+ return searchParameters;
+ }
+
+ private static Pattern fingerprintParameterPattern =
+ Pattern.compile("^[0-9a-zA-Z]{1,40}$");
+ private String parseFingerprintParameter(String parameter) {
+ if (!fingerprintParameterPattern.matcher(parameter).matches()) {
+ return null;
+ }
+ if (parameter.length() != 40) {
+ return null;
+ }
+ return parameter;
+ }
+
+ private static Pattern countryCodeParameterPattern =
+ Pattern.compile("^[0-9a-zA-Z]{2}$");
+ private String parseCountryCodeParameter(String parameter) {
+ if (!countryCodeParameterPattern.matcher(parameter).matches()) {
+ return null;
+ }
+ return parameter;
+ }
+
+ private static Pattern aSNumberParameterPattern =
+ Pattern.compile("^[asAS]{0,2}[0-9]{1,10}$");
+ private String parseASNumberParameter(String parameter) {
+ if (!aSNumberParameterPattern.matcher(parameter).matches()) {
+ return null;
+ }
+ return parameter;
+ }
+
+ private static Pattern flagPattern =
+ Pattern.compile("^[a-zA-Z0-9]{1,20}$");
+ private String parseFlagParameter(String parameter) {
+ if (!flagPattern.matcher(parameter).matches()) {
+ return null;
+ }
+ return parameter;
+ }
+
+ private static Pattern daysPattern = Pattern.compile("^[0-9-]{1,10}$");
+ private int[] parseDaysParameter(String parameter) {
+ if (!daysPattern.matcher(parameter).matches()) {
+ return null;
+ }
+ int x = 0, y = Integer.MAX_VALUE;
+ try {
+ if (!parameter.contains("-")) {
+ x = Integer.parseInt(parameter);
+ y = x;
+ } else {
+ String[] parts = parameter.split("-", 2);
+ if (parts[0].length() > 0) {
+ x = Integer.parseInt(parts[0]);
+ }
+ if (parts.length > 1 && parts[1].length() > 0) {
+ y = Integer.parseInt(parts[1]);
+ }
+ }
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ if (x > y) {
+ return null;
+ }
+ return new int[] { x, y };
+ }
+
+ private String[] parseContactParameter(String parameter) {
+ for (char c : parameter.toCharArray()) {
+ if (c < 32 || c >= 127) {
+ return null;
+ }
+ }
+ return parameter.split(" ");
+ }
+
+ private static Pattern fieldsParameterPattern =
+ Pattern.compile("^[0-9a-zA-Z_,]*$");
+ private String[] parseFieldsParameter(String parameter) {
+ if (!fieldsParameterPattern.matcher(parameter).matches()) {
+ return null;
+ }
+ return parameter.split(",");
+ }
+}
+
diff --git a/src/org/torproject/onionoo/server/ResponseBuilder.java b/src/org/torproject/onionoo/server/ResponseBuilder.java
new file mode 100644
index 0000000..161692c
--- /dev/null
+++ b/src/org/torproject/onionoo/server/ResponseBuilder.java
@@ -0,0 +1,320 @@
+/* Copyright 2011--2013 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.server;
+
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.torproject.onionoo.docs.BandwidthDocument;
+import org.torproject.onionoo.docs.ClientsDocument;
+import org.torproject.onionoo.docs.DetailsDocument;
+import org.torproject.onionoo.docs.DocumentStore;
+import org.torproject.onionoo.docs.SummaryDocument;
+import org.torproject.onionoo.docs.UptimeDocument;
+import org.torproject.onionoo.docs.WeightsDocument;
+import org.torproject.onionoo.util.ApplicationFactory;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+public class ResponseBuilder {
+
+ private DocumentStore documentStore;
+
+ public ResponseBuilder() {
+ this.documentStore = ApplicationFactory.getDocumentStore();
+ }
+
+ private String resourceType;
+ public void setResourceType(String resourceType) {
+ this.resourceType = resourceType;
+ }
+
+ private String relaysPublishedString;
+ public void setRelaysPublishedString(String relaysPublishedString) {
+ this.relaysPublishedString = relaysPublishedString;
+ }
+
+ private String bridgesPublishedString;
+ public void setBridgesPublishedString(String bridgesPublishedString) {
+ this.bridgesPublishedString = bridgesPublishedString;
+ }
+
+ private List<SummaryDocument> orderedRelays =
+ new ArrayList<SummaryDocument>();
+ public void setOrderedRelays(List<SummaryDocument> orderedRelays) {
+ this.orderedRelays = orderedRelays;
+ }
+
+ private List<SummaryDocument> orderedBridges =
+ new ArrayList<SummaryDocument>();
+ public void setOrderedBridges(List<SummaryDocument> orderedBridges) {
+ this.orderedBridges = orderedBridges;
+ }
+
+ private String[] fields;
+ public void setFields(String[] fields) {
+ this.fields = new String[fields.length];
+ System.arraycopy(fields, 0, this.fields, 0, fields.length);
+ }
+
+ public void buildResponse(PrintWriter pw) {
+ writeRelays(orderedRelays, pw);
+ writeBridges(orderedBridges, pw);
+ }
+
+ private void writeRelays(List<SummaryDocument> relays, PrintWriter pw) {
+ pw.write("{\"relays_published\":\"" + relaysPublishedString
+ + "\",\n\"relays\":[");
+ int written = 0;
+ for (SummaryDocument entry : relays) {
+ String lines = this.formatNodeStatus(entry);
+ if (lines.length() > 0) {
+ pw.print((written++ > 0 ? ",\n" : "\n") + lines);
+ }
+ }
+ pw.print("\n],\n");
+ }
+
+ private void writeBridges(List<SummaryDocument> bridges,
+ PrintWriter pw) {
+ pw.write("\"bridges_published\":\"" + bridgesPublishedString
+ + "\",\n\"bridges\":[");
+ int written = 0;
+ for (SummaryDocument entry : bridges) {
+ String lines = this.formatNodeStatus(entry);
+ if (lines.length() > 0) {
+ pw.print((written++ > 0 ? ",\n" : "\n") + lines);
+ }
+ }
+ pw.print("\n]}\n");
+ }
+
+ private String formatNodeStatus(SummaryDocument entry) {
+ if (this.resourceType == null) {
+ return "";
+ } else if (this.resourceType.equals("summary")) {
+ return this.writeSummaryLine(entry);
+ } else if (this.resourceType.equals("details")) {
+ return this.writeDetailsLines(entry);
+ } else if (this.resourceType.equals("bandwidth")) {
+ return this.writeBandwidthLines(entry);
+ } else if (this.resourceType.equals("weights")) {
+ return this.writeWeightsLines(entry);
+ } else if (this.resourceType.equals("clients")) {
+ return this.writeClientsLines(entry);
+ } else if (this.resourceType.equals("uptime")) {
+ return this.writeUptimeLines(entry);
+ } else {
+ return "";
+ }
+ }
+
+ private String writeSummaryLine(SummaryDocument entry) {
+ return entry.isRelay() ? writeRelaySummaryLine(entry)
+ : writeBridgeSummaryLine(entry);
+ }
+
+ private String writeRelaySummaryLine(SummaryDocument entry) {
+ String nickname = !entry.getNickname().equals("Unnamed") ?
+ entry.getNickname() : null;
+ String fingerprint = entry.getFingerprint();
+ String running = entry.isRunning() ? "true" : "false";
+ List<String> addresses = entry.getAddresses();
+ StringBuilder addressesBuilder = new StringBuilder();
+ int written = 0;
+ for (String address : addresses) {
+ addressesBuilder.append((written++ > 0 ? "," : "") + "\""
+ + address.toLowerCase() + "\"");
+ }
+ return String.format("{%s\"f\":\"%s\",\"a\":[%s],\"r\":%s}",
+ (nickname == null ? "" : "\"n\":\"" + nickname + "\","),
+ fingerprint, addressesBuilder.toString(), running);
+ }
+
+ private String writeBridgeSummaryLine(SummaryDocument entry) {
+ String nickname = !entry.getNickname().equals("Unnamed") ?
+ entry.getNickname() : null;
+ String hashedFingerprint = entry.getFingerprint();
+ String running = entry.isRunning() ? "true" : "false";
+ return String.format("{%s\"h\":\"%s\",\"r\":%s}",
+ (nickname == null ? "" : "\"n\":\"" + nickname + "\","),
+ hashedFingerprint, running);
+ }
+
+ private String writeDetailsLines(SummaryDocument entry) {
+ String fingerprint = entry.getFingerprint();
+ if (this.fields != null) {
+ /* TODO Maybe there's a more elegant way (more maintainable, more
+ * efficient, etc.) to implement this? */
+ DetailsDocument detailsDocument = documentStore.retrieve(
+ DetailsDocument.class, true, fingerprint);
+ if (detailsDocument != null) {
+ DetailsDocument dd = new DetailsDocument();
+ for (String field : this.fields) {
+ if (field.equals("nickname")) {
+ dd.setNickname(detailsDocument.getNickname());
+ } else if (field.equals("fingerprint")) {
+ dd.setFingerprint(detailsDocument.getFingerprint());
+ } else if (field.equals("hashed_fingerprint")) {
+ dd.setHashedFingerprint(
+ detailsDocument.getHashedFingerprint());
+ } else if (field.equals("or_addresses")) {
+ dd.setOrAddresses(detailsDocument.getOrAddresses());
+ } else if (field.equals("exit_addresses")) {
+ dd.setExitAddresses(detailsDocument.getExitAddresses());
+ } else if (field.equals("dir_address")) {
+ dd.setDirAddress(detailsDocument.getDirAddress());
+ } else if (field.equals("last_seen")) {
+ dd.setLastSeen(detailsDocument.getLastSeen());
+ } else if (field.equals("last_changed_address_or_port")) {
+ dd.setLastChangedAddressOrPort(
+ detailsDocument.getLastChangedAddressOrPort());
+ } else if (field.equals("first_seen")) {
+ dd.setFirstSeen(detailsDocument.getFirstSeen());
+ } else if (field.equals("running")) {
+ dd.setRunning(detailsDocument.getRunning());
+ } else if (field.equals("flags")) {
+ dd.setFlags(detailsDocument.getFlags());
+ } else if (field.equals("country")) {
+ dd.setCountry(detailsDocument.getCountry());
+ } else if (field.equals("country_name")) {
+ dd.setCountryName(detailsDocument.getCountryName());
+ } else if (field.equals("region_name")) {
+ dd.setRegionName(detailsDocument.getRegionName());
+ } else if (field.equals("city_name")) {
+ dd.setCityName(detailsDocument.getCityName());
+ } else if (field.equals("latitude")) {
+ dd.setLatitude(detailsDocument.getLatitude());
+ } else if (field.equals("longitude")) {
+ dd.setLongitude(detailsDocument.getLongitude());
+ } else if (field.equals("as_number")) {
+ dd.setAsNumber(detailsDocument.getAsNumber());
+ } else if (field.equals("as_name")) {
+ dd.setAsName(detailsDocument.getAsName());
+ } else if (field.equals("consensus_weight")) {
+ dd.setConsensusWeight(detailsDocument.getConsensusWeight());
+ } else if (field.equals("host_name")) {
+ dd.setHostName(detailsDocument.getHostName());
+ } else if (field.equals("last_restarted")) {
+ dd.setLastRestarted(detailsDocument.getLastRestarted());
+ } else if (field.equals("bandwidth_rate")) {
+ dd.setBandwidthRate(detailsDocument.getBandwidthRate());
+ } else if (field.equals("bandwidth_burst")) {
+ dd.setBandwidthBurst(detailsDocument.getBandwidthBurst());
+ } else if (field.equals("observed_bandwidth")) {
+ dd.setObservedBandwidth(
+ detailsDocument.getObservedBandwidth());
+ } else if (field.equals("advertised_bandwidth")) {
+ dd.setAdvertisedBandwidth(
+ detailsDocument.getAdvertisedBandwidth());
+ } else if (field.equals("exit_policy")) {
+ dd.setExitPolicy(detailsDocument.getExitPolicy());
+ } else if (field.equals("exit_policy_summary")) {
+ dd.setExitPolicySummary(
+ detailsDocument.getExitPolicySummary());
+ } else if (field.equals("exit_policy_v6_summary")) {
+ dd.setExitPolicyV6Summary(
+ detailsDocument.getExitPolicyV6Summary());
+ } else if (field.equals("contact")) {
+ dd.setContact(detailsDocument.getContact());
+ } else if (field.equals("platform")) {
+ dd.setPlatform(detailsDocument.getPlatform());
+ } else if (field.equals("family")) {
+ dd.setFamily(detailsDocument.getFamily());
+ } else if (field.equals("advertised_bandwidth_fraction")) {
+ dd.setAdvertisedBandwidthFraction(
+ detailsDocument.getAdvertisedBandwidthFraction());
+ } else if (field.equals("consensus_weight_fraction")) {
+ dd.setConsensusWeightFraction(
+ detailsDocument.getConsensusWeightFraction());
+ } else if (field.equals("guard_probability")) {
+ dd.setGuardProbability(detailsDocument.getGuardProbability());
+ } else if (field.equals("middle_probability")) {
+ dd.setMiddleProbability(
+ detailsDocument.getMiddleProbability());
+ } else if (field.equals("exit_probability")) {
+ dd.setExitProbability(detailsDocument.getExitProbability());
+ } else if (field.equals("recommended_version")) {
+ dd.setRecommendedVersion(
+ detailsDocument.getRecommendedVersion());
+ } else if (field.equals("hibernating")) {
+ dd.setHibernating(detailsDocument.getHibernating());
+ } else if (field.equals("pool_assignment")) {
+ dd.setPoolAssignment(detailsDocument.getPoolAssignment());
+ }
+ }
+ /* Don't escape HTML characters, like < and >, contained in
+ * strings. */
+ Gson gson = new GsonBuilder().disableHtmlEscaping().create();
+ /* Whenever we provide Gson with a string containing an escaped
+ * non-ASCII character like \u00F2, it escapes the \ to \\, which
+ * we need to undo before including the string in a response. */
+ return gson.toJson(dd).replaceAll("\\\\\\\\u", "\\\\u");
+ } else {
+ // TODO We should probably log that we didn't find a details
+ // document that we expected to exist.
+ return "";
+ }
+ } else {
+ DetailsDocument detailsDocument = documentStore.retrieve(
+ DetailsDocument.class, false, fingerprint);
+ if (detailsDocument != null) {
+ return detailsDocument.getDocumentString();
+ } else {
+ // TODO We should probably log that we didn't find a details
+ // document that we expected to exist.
+ return "";
+ }
+ }
+ }
+
+ private String writeBandwidthLines(SummaryDocument entry) {
+ String fingerprint = entry.getFingerprint();
+ BandwidthDocument bandwidthDocument = this.documentStore.retrieve(
+ BandwidthDocument.class, false, fingerprint);
+ if (bandwidthDocument != null &&
+ bandwidthDocument.getDocumentString() != null) {
+ return bandwidthDocument.getDocumentString();
+ } else {
+ return "{\"fingerprint\":\"" + fingerprint.toUpperCase() + "\"}";
+ }
+ }
+
+ private String writeWeightsLines(SummaryDocument entry) {
+ String fingerprint = entry.getFingerprint();
+ WeightsDocument weightsDocument = this.documentStore.retrieve(
+ WeightsDocument.class, false, fingerprint);
+ if (weightsDocument != null &&
+ weightsDocument.getDocumentString() != null) {
+ return weightsDocument.getDocumentString();
+ } else {
+ return "{\"fingerprint\":\"" + fingerprint.toUpperCase() + "\"}";
+ }
+ }
+
+ private String writeClientsLines(SummaryDocument entry) {
+ String fingerprint = entry.getFingerprint();
+ ClientsDocument clientsDocument = this.documentStore.retrieve(
+ ClientsDocument.class, false, fingerprint);
+ if (clientsDocument != null &&
+ clientsDocument.getDocumentString() != null) {
+ return clientsDocument.getDocumentString();
+ } else {
+ return "{\"fingerprint\":\"" + fingerprint.toUpperCase() + "\"}";
+ }
+ }
+
+ private String writeUptimeLines(SummaryDocument entry) {
+ String fingerprint = entry.getFingerprint();
+ UptimeDocument uptimeDocument = this.documentStore.retrieve(
+ UptimeDocument.class, false, fingerprint);
+ if (uptimeDocument != null &&
+ uptimeDocument.getDocumentString() != null) {
+ return uptimeDocument.getDocumentString();
+ } else {
+ return "{\"fingerprint\":\"" + fingerprint.toUpperCase() + "\"}";
+ }
+ }
+}
diff --git a/src/org/torproject/onionoo/updater/BandwidthStatusUpdater.java b/src/org/torproject/onionoo/updater/BandwidthStatusUpdater.java
new file mode 100644
index 0000000..bc7dd74
--- /dev/null
+++ b/src/org/torproject/onionoo/updater/BandwidthStatusUpdater.java
@@ -0,0 +1,149 @@
+/* Copyright 2011--2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.updater;
+
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.ExtraInfoDescriptor;
+import org.torproject.onionoo.docs.BandwidthStatus;
+import org.torproject.onionoo.docs.DocumentStore;
+import org.torproject.onionoo.util.ApplicationFactory;
+import org.torproject.onionoo.util.DateTimeHelper;
+
+public class BandwidthStatusUpdater implements DescriptorListener,
+ StatusUpdater {
+
+ private DescriptorSource descriptorSource;
+
+ private DocumentStore documentStore;
+
+ private long now;
+
+ public BandwidthStatusUpdater() {
+ this.descriptorSource = ApplicationFactory.getDescriptorSource();
+ this.documentStore = ApplicationFactory.getDocumentStore();
+ this.now = ApplicationFactory.getTime().currentTimeMillis();
+ this.registerDescriptorListeners();
+ }
+
+ private void registerDescriptorListeners() {
+ this.descriptorSource.registerDescriptorListener(this,
+ DescriptorType.RELAY_EXTRA_INFOS);
+ this.descriptorSource.registerDescriptorListener(this,
+ DescriptorType.BRIDGE_EXTRA_INFOS);
+ }
+
+ public void processDescriptor(Descriptor descriptor, boolean relay) {
+ if (descriptor instanceof ExtraInfoDescriptor) {
+ this.parseDescriptor((ExtraInfoDescriptor) descriptor);
+ }
+ }
+
+ public void updateStatuses() {
+ /* Status files are already updated while processing descriptors. */
+ }
+
+ private void parseDescriptor(ExtraInfoDescriptor descriptor) {
+ String fingerprint = descriptor.getFingerprint();
+ BandwidthStatus bandwidthStatus = this.documentStore.retrieve(
+ BandwidthStatus.class, true, fingerprint);
+ if (bandwidthStatus == null) {
+ bandwidthStatus = new BandwidthStatus();
+ }
+ if (descriptor.getWriteHistory() != null) {
+ parseHistoryLine(descriptor.getWriteHistory().getLine(),
+ bandwidthStatus.getWriteHistory());
+ }
+ if (descriptor.getReadHistory() != null) {
+ parseHistoryLine(descriptor.getReadHistory().getLine(),
+ bandwidthStatus.getReadHistory());
+ }
+ this.compressHistory(bandwidthStatus.getWriteHistory());
+ this.compressHistory(bandwidthStatus.getReadHistory());
+ this.documentStore.store(bandwidthStatus, fingerprint);
+ }
+
+ private void parseHistoryLine(String line,
+ SortedMap<Long, long[]> history) {
+ String[] parts = line.split(" ");
+ if (parts.length < 6) {
+ return;
+ }
+ long endMillis = DateTimeHelper.parse(parts[1] + " " + parts[2]);
+ if (endMillis < 0L) {
+ System.err.println("Could not parse timestamp in line '" + line
+ + "'. Skipping.");
+ return;
+ }
+ long intervalMillis = Long.parseLong(parts[3].substring(1))
+ * DateTimeHelper.ONE_SECOND;
+ String[] values = parts[5].split(",");
+ for (int i = values.length - 1; i >= 0; i--) {
+ long bandwidthValue = Long.parseLong(values[i]);
+ long startMillis = endMillis - intervalMillis;
+ /* TODO Should we first check whether an interval is already
+ * contained in history? */
+ history.put(startMillis, new long[] { startMillis, endMillis,
+ bandwidthValue });
+ endMillis -= intervalMillis;
+ }
+ }
+
+ private void compressHistory(SortedMap<Long, long[]> history) {
+ SortedMap<Long, long[]> uncompressedHistory =
+ new TreeMap<Long, long[]>(history);
+ history.clear();
+ long lastStartMillis = 0L, lastEndMillis = 0L, lastBandwidth = 0L;
+ String lastMonthString = "1970-01";
+ for (long[] v : uncompressedHistory.values()) {
+ long startMillis = v[0], endMillis = v[1], bandwidth = v[2];
+ long intervalLengthMillis;
+ if (this.now - endMillis <= DateTimeHelper.THREE_DAYS) {
+ intervalLengthMillis = DateTimeHelper.FIFTEEN_MINUTES;
+ } else if (this.now - endMillis <= DateTimeHelper.ONE_WEEK) {
+ intervalLengthMillis = DateTimeHelper.ONE_HOUR;
+ } else if (this.now - endMillis <=
+ DateTimeHelper.ROUGHLY_ONE_MONTH) {
+ intervalLengthMillis = DateTimeHelper.FOUR_HOURS;
+ } else if (this.now - endMillis <=
+ DateTimeHelper.ROUGHLY_THREE_MONTHS) {
+ intervalLengthMillis = DateTimeHelper.TWELVE_HOURS;
+ } else if (this.now - endMillis <=
+ DateTimeHelper.ROUGHLY_ONE_YEAR) {
+ intervalLengthMillis = DateTimeHelper.TWO_DAYS;
+ } else {
+ intervalLengthMillis = DateTimeHelper.TEN_DAYS;
+ }
+ String monthString = DateTimeHelper.format(startMillis,
+ DateTimeHelper.ISO_YEARMONTH_FORMAT);
+ if (lastEndMillis == startMillis &&
+ ((lastEndMillis - 1L) / intervalLengthMillis) ==
+ ((endMillis - 1L) / intervalLengthMillis) &&
+ lastMonthString.equals(monthString)) {
+ lastEndMillis = endMillis;
+ lastBandwidth += bandwidth;
+ } else {
+ if (lastStartMillis > 0L) {
+ history.put(lastStartMillis, new long[] { lastStartMillis,
+ lastEndMillis, lastBandwidth });
+ }
+ lastStartMillis = startMillis;
+ lastEndMillis = endMillis;
+ lastBandwidth = bandwidth;
+ }
+ lastMonthString = monthString;
+ }
+ if (lastStartMillis > 0L) {
+ history.put(lastStartMillis, new long[] { lastStartMillis,
+ lastEndMillis, lastBandwidth });
+ }
+ }
+
+ public String getStatsString() {
+ /* TODO Add statistics string. */
+ return null;
+ }
+}
+
diff --git a/src/org/torproject/onionoo/updater/ClientsStatusUpdater.java b/src/org/torproject/onionoo/updater/ClientsStatusUpdater.java
new file mode 100644
index 0000000..79c1060
--- /dev/null
+++ b/src/org/torproject/onionoo/updater/ClientsStatusUpdater.java
@@ -0,0 +1,230 @@
+/* Copyright 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.updater;
+
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.ExtraInfoDescriptor;
+import org.torproject.onionoo.docs.ClientsHistory;
+import org.torproject.onionoo.docs.ClientsStatus;
+import org.torproject.onionoo.docs.DocumentStore;
+import org.torproject.onionoo.util.ApplicationFactory;
+import org.torproject.onionoo.util.DateTimeHelper;
+import org.torproject.onionoo.util.Logger;
+
+/*
+ * Example extra-info descriptor used as input:
+ *
+ * extra-info ndnop2 DE6397A047ABE5F78B4C87AF725047831B221AAB
+ * dirreq-stats-end 2014-02-16 16:42:11 (86400 s)
+ * dirreq-v3-resp ok=856,not-enough-sigs=0,unavailable=0,not-found=0,
+ * not-modified=40,busy=0
+ * bridge-stats-end 2014-02-16 16:42:17 (86400 s)
+ * bridge-ips ??=8,in=8,se=8
+ * bridge-ip-versions v4=8,v6=0
+ *
+ * Clients status file produced as intermediate output:
+ *
+ * 2014-02-15 16:42:11 2014-02-16 00:00:00
+ * 259.042 in=86.347,se=86.347 v4=259.042
+ * 2014-02-16 00:00:00 2014-02-16 16:42:11
+ * 592.958 in=197.653,se=197.653 v4=592.958
+ */
+public class ClientsStatusUpdater implements DescriptorListener,
+ StatusUpdater {
+
+ private DescriptorSource descriptorSource;
+
+ private DocumentStore documentStore;
+
+ private long now;
+
+ public ClientsStatusUpdater() {
+ this.descriptorSource = ApplicationFactory.getDescriptorSource();
+ this.documentStore = ApplicationFactory.getDocumentStore();
+ this.now = ApplicationFactory.getTime().currentTimeMillis();
+ this.registerDescriptorListeners();
+ }
+
+ private void registerDescriptorListeners() {
+ this.descriptorSource.registerDescriptorListener(this,
+ DescriptorType.BRIDGE_EXTRA_INFOS);
+ }
+
+ public void processDescriptor(Descriptor descriptor, boolean relay) {
+ if (descriptor instanceof ExtraInfoDescriptor && !relay) {
+ this.processBridgeExtraInfoDescriptor(
+ (ExtraInfoDescriptor) descriptor);
+ }
+ }
+
+ private SortedMap<String, SortedSet<ClientsHistory>> newResponses =
+ new TreeMap<String, SortedSet<ClientsHistory>>();
+
+ private void processBridgeExtraInfoDescriptor(
+ ExtraInfoDescriptor descriptor) {
+ long dirreqStatsEndMillis = descriptor.getDirreqStatsEndMillis();
+ long dirreqStatsIntervalLengthMillis =
+ descriptor.getDirreqStatsIntervalLength()
+ * DateTimeHelper.ONE_SECOND;
+ SortedMap<String, Integer> responses = descriptor.getDirreqV3Resp();
+ if (dirreqStatsEndMillis < 0L ||
+ dirreqStatsIntervalLengthMillis != DateTimeHelper.ONE_DAY ||
+ responses == null || !responses.containsKey("ok")) {
+ return;
+ }
+ double okResponses = (double) (responses.get("ok") - 4);
+ if (okResponses < 0.0) {
+ return;
+ }
+ String hashedFingerprint = descriptor.getFingerprint().toUpperCase();
+ long dirreqStatsStartMillis = dirreqStatsEndMillis
+ - dirreqStatsIntervalLengthMillis;
+ long utcBreakMillis = (dirreqStatsEndMillis / DateTimeHelper.ONE_DAY)
+ * DateTimeHelper.ONE_DAY;
+ for (int i = 0; i < 2; i++) {
+ long startMillis = i == 0 ? dirreqStatsStartMillis : utcBreakMillis;
+ long endMillis = i == 0 ? utcBreakMillis : dirreqStatsEndMillis;
+ if (startMillis >= endMillis) {
+ continue;
+ }
+ double totalResponses = okResponses
+ * ((double) (endMillis - startMillis))
+ / ((double) DateTimeHelper.ONE_DAY);
+ SortedMap<String, Double> responsesByCountry =
+ this.weightResponsesWithUniqueIps(totalResponses,
+ descriptor.getBridgeIps(), "??");
+ SortedMap<String, Double> responsesByTransport =
+ this.weightResponsesWithUniqueIps(totalResponses,
+ descriptor.getBridgeIpTransports(), "<??>");
+ SortedMap<String, Double> responsesByVersion =
+ this.weightResponsesWithUniqueIps(totalResponses,
+ descriptor.getBridgeIpVersions(), "");
+ ClientsHistory newResponseHistory = new ClientsHistory(
+ startMillis, endMillis, totalResponses, responsesByCountry,
+ responsesByTransport, responsesByVersion);
+ if (!this.newResponses.containsKey(hashedFingerprint)) {
+ this.newResponses.put(hashedFingerprint,
+ new TreeSet<ClientsHistory>());
+ }
+ this.newResponses.get(hashedFingerprint).add(
+ newResponseHistory);
+ }
+ }
+
+ private SortedMap<String, Double> weightResponsesWithUniqueIps(
+ double totalResponses, SortedMap<String, Integer> uniqueIps,
+ String omitString) {
+ SortedMap<String, Double> weightedResponses =
+ new TreeMap<String, Double>();
+ int totalUniqueIps = 0;
+ if (uniqueIps != null) {
+ for (Map.Entry<String, Integer> e : uniqueIps.entrySet()) {
+ if (e.getValue() > 4) {
+ totalUniqueIps += e.getValue() - 4;
+ }
+ }
+ }
+ if (totalUniqueIps > 0) {
+ for (Map.Entry<String, Integer> e : uniqueIps.entrySet()) {
+ if (!e.getKey().equals(omitString) && e.getValue() > 4) {
+ weightedResponses.put(e.getKey(),
+ (((double) (e.getValue() - 4)) * totalResponses)
+ / ((double) totalUniqueIps));
+ }
+ }
+ }
+ return weightedResponses;
+ }
+
+ public void updateStatuses() {
+ for (Map.Entry<String, SortedSet<ClientsHistory>> e :
+ this.newResponses.entrySet()) {
+ String hashedFingerprint = e.getKey();
+ ClientsStatus clientsStatus = this.documentStore.retrieve(
+ ClientsStatus.class, true, hashedFingerprint);
+ if (clientsStatus == null) {
+ clientsStatus = new ClientsStatus();
+ }
+ this.addToHistory(clientsStatus, e.getValue());
+ this.compressHistory(clientsStatus);
+ this.documentStore.store(clientsStatus, hashedFingerprint);
+ }
+ }
+
+ private void addToHistory(ClientsStatus clientsStatus,
+ SortedSet<ClientsHistory> newIntervals) {
+ SortedSet<ClientsHistory> history = clientsStatus.getHistory();
+ for (ClientsHistory interval : newIntervals) {
+ if ((history.headSet(interval).isEmpty() ||
+ history.headSet(interval).last().getEndMillis() <=
+ interval.getStartMillis()) &&
+ (history.tailSet(interval).isEmpty() ||
+ history.tailSet(interval).first().getStartMillis() >=
+ interval.getEndMillis())) {
+ history.add(interval);
+ }
+ }
+ }
+
+ private void compressHistory(ClientsStatus clientsStatus) {
+ SortedSet<ClientsHistory> history = clientsStatus.getHistory();
+ SortedSet<ClientsHistory> compressedHistory =
+ new TreeSet<ClientsHistory>();
+ ClientsHistory lastResponses = null;
+ String lastMonthString = "1970-01";
+ for (ClientsHistory responses : history) {
+ long intervalLengthMillis;
+ if (this.now - responses.getEndMillis() <=
+ DateTimeHelper.ROUGHLY_THREE_MONTHS) {
+ intervalLengthMillis = DateTimeHelper.ONE_DAY;
+ } else if (this.now - responses.getEndMillis() <=
+ DateTimeHelper.ROUGHLY_ONE_YEAR) {
+ intervalLengthMillis = DateTimeHelper.TWO_DAYS;
+ } else {
+ intervalLengthMillis = DateTimeHelper.TEN_DAYS;
+ }
+ String monthString = DateTimeHelper.format(
+ responses.getStartMillis(),
+ DateTimeHelper.ISO_YEARMONTH_FORMAT);
+ if (lastResponses != null &&
+ lastResponses.getEndMillis() == responses.getStartMillis() &&
+ ((lastResponses.getEndMillis() - 1L) / intervalLengthMillis) ==
+ ((responses.getEndMillis() - 1L) / intervalLengthMillis) &&
+ lastMonthString.equals(monthString)) {
+ lastResponses.addResponses(responses);
+ } else {
+ if (lastResponses != null) {
+ compressedHistory.add(lastResponses);
+ }
+ lastResponses = responses;
+ }
+ lastMonthString = monthString;
+ }
+ if (lastResponses != null) {
+ compressedHistory.add(lastResponses);
+ }
+ clientsStatus.setHistory(compressedHistory);
+ }
+
+ public String getStatsString() {
+ int newIntervals = 0;
+ for (SortedSet<ClientsHistory> hist : this.newResponses.values()) {
+ newIntervals += hist.size();
+ }
+ StringBuilder sb = new StringBuilder();
+ sb.append(" "
+ + Logger.formatDecimalNumber(newIntervals / 2)
+ + " client statistics processed from extra-info descriptors\n");
+ sb.append(" "
+ + Logger.formatDecimalNumber(this.newResponses.size())
+ + " client status files updated\n");
+ return sb.toString();
+ }
+}
+
diff --git a/src/org/torproject/onionoo/updater/DescriptorListener.java b/src/org/torproject/onionoo/updater/DescriptorListener.java
new file mode 100644
index 0000000..3613879
--- /dev/null
+++ b/src/org/torproject/onionoo/updater/DescriptorListener.java
@@ -0,0 +1,7 @@
+package org.torproject.onionoo.updater;
+
+import org.torproject.descriptor.Descriptor;
+
+public interface DescriptorListener {
+ abstract void processDescriptor(Descriptor descriptor, boolean relay);
+}
\ No newline at end of file
diff --git a/src/org/torproject/onionoo/updater/DescriptorSource.java b/src/org/torproject/onionoo/updater/DescriptorSource.java
new file mode 100644
index 0000000..32fbd2a
--- /dev/null
+++ b/src/org/torproject/onionoo/updater/DescriptorSource.java
@@ -0,0 +1,646 @@
+/* Copyright 2013, 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.updater;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.zip.GZIPInputStream;
+
+import org.torproject.descriptor.BridgeNetworkStatus;
+import org.torproject.descriptor.BridgePoolAssignment;
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.DescriptorFile;
+import org.torproject.descriptor.DescriptorReader;
+import org.torproject.descriptor.DescriptorSourceFactory;
+import org.torproject.descriptor.ExitList;
+import org.torproject.descriptor.ExitListEntry;
+import org.torproject.descriptor.ExtraInfoDescriptor;
+import org.torproject.descriptor.RelayNetworkStatusConsensus;
+import org.torproject.descriptor.ServerDescriptor;
+import org.torproject.onionoo.util.Logger;
+
+enum DescriptorHistory {
+ RELAY_CONSENSUS_HISTORY,
+ RELAY_SERVER_HISTORY,
+ RELAY_EXTRAINFO_HISTORY,
+ EXIT_LIST_HISTORY,
+ BRIDGE_STATUS_HISTORY,
+ BRIDGE_SERVER_HISTORY,
+ BRIDGE_EXTRAINFO_HISTORY,
+ BRIDGE_POOLASSIGN_HISTORY,
+}
+
+class DescriptorDownloader {
+
+ private final String protocolHostNameResourcePrefix =
+ "https://collector.torproject.org/recent/";;
+
+ private String directory;
+
+ private final File inDir = new File("in/recent");
+
+ public DescriptorDownloader(DescriptorType descriptorType) {
+ switch (descriptorType) {
+ case RELAY_CONSENSUSES:
+ this.directory = "relay-descriptors/consensuses/";
+ break;
+ case RELAY_SERVER_DESCRIPTORS:
+ this.directory = "relay-descriptors/server-descriptors/";
+ break;
+ case RELAY_EXTRA_INFOS:
+ this.directory = "relay-descriptors/extra-infos/";
+ break;
+ case EXIT_LISTS:
+ this.directory = "exit-lists/";
+ break;
+ case BRIDGE_STATUSES:
+ this.directory = "bridge-descriptors/statuses/";
+ break;
+ case BRIDGE_SERVER_DESCRIPTORS:
+ this.directory = "bridge-descriptors/server-descriptors/";
+ break;
+ case BRIDGE_EXTRA_INFOS:
+ this.directory = "bridge-descriptors/extra-infos/";
+ break;
+ case BRIDGE_POOL_ASSIGNMENTS:
+ this.directory = "bridge-pool-assignments/";
+ break;
+ default:
+ System.err.println("Unknown descriptor type.");
+ return;
+ }
+ }
+
+ private SortedSet<String> localFiles = new TreeSet<String>();
+
+ public int statLocalFiles() {
+ File localDirectory = new File(this.inDir, this.directory);
+ if (localDirectory.exists()) {
+ for (File file : localDirectory.listFiles()) {
+ this.localFiles.add(file.getName());
+ }
+ }
+ return this.localFiles.size();
+ }
+
+ private SortedSet<String> remoteFiles = new TreeSet<String>();
+
+ public int fetchRemoteDirectory() {
+ String directoryUrl = this.protocolHostNameResourcePrefix
+ + this.directory;
+ try {
+ URL u = new URL(directoryUrl);
+ HttpURLConnection huc = (HttpURLConnection) u.openConnection();
+ huc.setRequestMethod("GET");
+ huc.connect();
+ if (huc.getResponseCode() != 200) {
+ System.err.println("Could not fetch " + directoryUrl
+ + ": " + huc.getResponseCode() + " "
+ + huc.getResponseMessage() + ". Skipping.");
+ return 0;
+ }
+ BufferedReader br = new BufferedReader(new InputStreamReader(
+ huc.getInputStream()));
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (!line.trim().startsWith("<tr>") ||
+ !line.contains("<a href=\"")) {
+ continue;
+ }
+ String linePart = line.substring(
+ line.indexOf("<a href=\"") + "<a href=\"".length());
+ if (!linePart.contains("\"")) {
+ continue;
+ }
+ linePart = linePart.substring(0, linePart.indexOf("\""));
+ if (linePart.endsWith("/")) {
+ continue;
+ }
+ this.remoteFiles.add(linePart);
+ }
+ br.close();
+ } catch (IOException e) {
+ System.err.println("Could not fetch or parse " + directoryUrl
+ + ". Skipping.");
+ }
+ return this.remoteFiles.size();
+ }
+
+ public int fetchRemoteFiles() {
+ int fetchedFiles = 0;
+ for (String remoteFile : this.remoteFiles) {
+ if (this.localFiles.contains(remoteFile)) {
+ continue;
+ }
+ String fileUrl = this.protocolHostNameResourcePrefix
+ + this.directory + remoteFile;
+ File localTempFile = new File(this.inDir, this.directory
+ + remoteFile + ".tmp");
+ File localFile = new File(this.inDir, this.directory + remoteFile);
+ try {
+ localFile.getParentFile().mkdirs();
+ URL u = new URL(fileUrl);
+ HttpURLConnection huc = (HttpURLConnection) u.openConnection();
+ huc.setRequestMethod("GET");
+ huc.addRequestProperty("Accept-Encoding", "gzip");
+ huc.connect();
+ if (huc.getResponseCode() != 200) {
+ System.err.println("Could not fetch " + fileUrl
+ + ": " + huc.getResponseCode() + " "
+ + huc.getResponseMessage() + ". Skipping.");
+ continue;
+ }
+ long lastModified = huc.getHeaderFieldDate("Last-Modified", -1L);
+ InputStream is;
+ if (huc.getContentEncoding() != null &&
+ huc.getContentEncoding().equalsIgnoreCase("gzip")) {
+ is = new GZIPInputStream(huc.getInputStream());
+ } else {
+ is = huc.getInputStream();
+ }
+ BufferedInputStream bis = new BufferedInputStream(is);
+ BufferedOutputStream bos = new BufferedOutputStream(
+ new FileOutputStream(localTempFile));
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = bis.read(data, 0, 1024)) >= 0) {
+ bos.write(data, 0, len);
+ }
+ bis.close();
+ bos.close();
+ localTempFile.renameTo(localFile);
+ if (lastModified >= 0) {
+ localFile.setLastModified(lastModified);
+ }
+ fetchedFiles++;
+ } catch (IOException e) {
+ System.err.println("Could not fetch or store " + fileUrl
+ + ". Skipping.");
+ }
+ }
+ return fetchedFiles;
+ }
+
+ public int deleteOldLocalFiles() {
+ int deletedFiles = 0;
+ for (String localFile : this.localFiles) {
+ if (!this.remoteFiles.contains(localFile)) {
+ new File(this.inDir, this.directory + localFile).delete();
+ deletedFiles++;
+ }
+ }
+ return deletedFiles;
+ }
+}
+
+class DescriptorQueue {
+
+ private File inDir;
+
+ private File statusDir;
+
+ private DescriptorReader descriptorReader;
+
+ private File historyFile;
+
+ private Iterator<DescriptorFile> descriptorFiles;
+
+ private List<Descriptor> descriptors;
+
+ private int historySizeBefore;
+ public int getHistorySizeBefore() {
+ return this.historySizeBefore;
+ }
+
+ private int historySizeAfter;
+ public int getHistorySizeAfter() {
+ return this.historySizeAfter;
+ }
+
+ private long returnedDescriptors = 0L;
+ public long getReturnedDescriptors() {
+ return this.returnedDescriptors;
+ }
+
+ private long returnedBytes = 0L;
+ public long getReturnedBytes() {
+ return this.returnedBytes;
+ }
+
+ public DescriptorQueue(File inDir, File statusDir) {
+ this.inDir = inDir;
+ this.statusDir = statusDir;
+ this.descriptorReader =
+ DescriptorSourceFactory.createDescriptorReader();
+ }
+
+ public void addDirectory(DescriptorType descriptorType) {
+ String directoryName = null;
+ switch (descriptorType) {
+ case RELAY_CONSENSUSES:
+ directoryName = "relay-descriptors/consensuses";
+ break;
+ case RELAY_SERVER_DESCRIPTORS:
+ directoryName = "relay-descriptors/server-descriptors";
+ break;
+ case RELAY_EXTRA_INFOS:
+ directoryName = "relay-descriptors/extra-infos";
+ break;
+ case BRIDGE_STATUSES:
+ directoryName = "bridge-descriptors/statuses";
+ break;
+ case BRIDGE_SERVER_DESCRIPTORS:
+ directoryName = "bridge-descriptors/server-descriptors";
+ break;
+ case BRIDGE_EXTRA_INFOS:
+ directoryName = "bridge-descriptors/extra-infos";
+ break;
+ case BRIDGE_POOL_ASSIGNMENTS:
+ directoryName = "bridge-pool-assignments";
+ break;
+ case EXIT_LISTS:
+ directoryName = "exit-lists";
+ break;
+ default:
+ System.err.println("Unknown descriptor type. Not adding directory "
+ + "to descriptor reader.");
+ return;
+ }
+ File directory = new File(this.inDir, directoryName);
+ if (directory.exists() && directory.isDirectory()) {
+ this.descriptorReader.addDirectory(directory);
+ this.descriptorReader.setMaxDescriptorFilesInQueue(1);
+ } else {
+ System.err.println("Directory " + directory.getAbsolutePath()
+ + " either does not exist or is not a directory. Not adding "
+ + "to descriptor reader.");
+ }
+ }
+
+ public void readHistoryFile(DescriptorHistory descriptorHistory) {
+ String historyFileName = null;
+ switch (descriptorHistory) {
+ case RELAY_EXTRAINFO_HISTORY:
+ historyFileName = "relay-extrainfo-history";
+ break;
+ case BRIDGE_EXTRAINFO_HISTORY:
+ historyFileName = "bridge-extrainfo-history";
+ break;
+ case EXIT_LIST_HISTORY:
+ historyFileName = "exit-list-history";
+ break;
+ case BRIDGE_POOLASSIGN_HISTORY:
+ historyFileName = "bridge-poolassign-history";
+ break;
+ case RELAY_CONSENSUS_HISTORY:
+ historyFileName = "relay-consensus-history";
+ break;
+ case BRIDGE_STATUS_HISTORY:
+ historyFileName = "bridge-status-history";
+ break;
+ case RELAY_SERVER_HISTORY:
+ historyFileName = "relay-server-history";
+ break;
+ case BRIDGE_SERVER_HISTORY:
+ historyFileName = "bridge-server-history";
+ break;
+ default:
+ System.err.println("Unknown descriptor history. Not excluding "
+ + "files.");
+ return;
+ }
+ this.historyFile = new File(this.statusDir, historyFileName);
+ if (this.historyFile.exists() && this.historyFile.isFile()) {
+ SortedMap<String, Long> excludedFiles = new TreeMap<String, Long>();
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ this.historyFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ try {
+ String[] parts = line.split(" ", 2);
+ excludedFiles.put(parts[1], Long.parseLong(parts[0]));
+ } catch (NumberFormatException e) {
+ System.err.println("Illegal line '" + line + "' in parse "
+ + "history. Skipping line.");
+ }
+ }
+ br.close();
+ } catch (IOException e) {
+ System.err.println("Could not read history file '"
+ + this.historyFile.getAbsolutePath() + "'. Not excluding "
+ + "descriptors in this execution.");
+ e.printStackTrace();
+ return;
+ }
+ this.historySizeBefore = excludedFiles.size();
+ this.descriptorReader.setExcludedFiles(excludedFiles);
+ }
+ }
+
+ public void writeHistoryFile() {
+ if (this.historyFile == null) {
+ return;
+ }
+ SortedMap<String, Long> excludedAndParsedFiles =
+ new TreeMap<String, Long>();
+ excludedAndParsedFiles.putAll(
+ this.descriptorReader.getExcludedFiles());
+ excludedAndParsedFiles.putAll(this.descriptorReader.getParsedFiles());
+ this.historySizeAfter = excludedAndParsedFiles.size();
+ try {
+ this.historyFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.historyFile));
+ for (Map.Entry<String, Long> e : excludedAndParsedFiles.entrySet()) {
+ String absolutePath = e.getKey();
+ long lastModifiedMillis = e.getValue();
+ bw.write(String.valueOf(lastModifiedMillis) + " " + absolutePath
+ + "\n");
+ }
+ bw.close();
+ } catch (IOException e) {
+ System.err.println("Could not write history file '"
+ + this.historyFile.getAbsolutePath() + "'. Not excluding "
+ + "descriptors in next execution.");
+ return;
+ }
+ }
+
+ public Descriptor nextDescriptor() {
+ Descriptor nextDescriptor = null;
+ if (this.descriptorFiles == null) {
+ this.descriptorFiles = this.descriptorReader.readDescriptors();
+ }
+ while (this.descriptors == null && this.descriptorFiles.hasNext()) {
+ DescriptorFile descriptorFile = this.descriptorFiles.next();
+ if (descriptorFile.getException() != null) {
+ System.err.println("Could not parse "
+ + descriptorFile.getFileName());
+ descriptorFile.getException().printStackTrace();
+ }
+ if (descriptorFile.getDescriptors() != null &&
+ !descriptorFile.getDescriptors().isEmpty()) {
+ this.descriptors = descriptorFile.getDescriptors();
+ }
+ }
+ if (this.descriptors != null) {
+ nextDescriptor = this.descriptors.remove(0);
+ this.returnedDescriptors++;
+ this.returnedBytes += nextDescriptor.getRawDescriptorBytes().length;
+ if (this.descriptors.isEmpty()) {
+ this.descriptors = null;
+ }
+ }
+ return nextDescriptor;
+ }
+}
+
+public class DescriptorSource {
+
+ private final File inDir = new File("in/recent");
+
+ private final File statusDir = new File("status");
+
+ private List<DescriptorQueue> descriptorQueues;
+
+ public DescriptorSource() {
+ this.descriptorQueues = new ArrayList<DescriptorQueue>();
+ this.descriptorListeners =
+ new HashMap<DescriptorType, Set<DescriptorListener>>();
+ this.fingerprintListeners =
+ new HashMap<DescriptorType, Set<FingerprintListener>>();
+ }
+
+ private DescriptorQueue getDescriptorQueue(
+ DescriptorType descriptorType,
+ DescriptorHistory descriptorHistory) {
+ DescriptorQueue descriptorQueue = new DescriptorQueue(this.inDir,
+ this.statusDir);
+ descriptorQueue.addDirectory(descriptorType);
+ if (descriptorHistory != null) {
+ descriptorQueue.readHistoryFile(descriptorHistory);
+ }
+ this.descriptorQueues.add(descriptorQueue);
+ return descriptorQueue;
+ }
+
+ private Map<DescriptorType, Set<DescriptorListener>>
+ descriptorListeners;
+
+ private Map<DescriptorType, Set<FingerprintListener>>
+ fingerprintListeners;
+
+ public void registerDescriptorListener(DescriptorListener listener,
+ DescriptorType descriptorType) {
+ if (!this.descriptorListeners.containsKey(descriptorType)) {
+ this.descriptorListeners.put(descriptorType,
+ new HashSet<DescriptorListener>());
+ }
+ this.descriptorListeners.get(descriptorType).add(listener);
+ }
+
+ public void registerFingerprintListener(FingerprintListener listener,
+ DescriptorType descriptorType) {
+ if (!this.fingerprintListeners.containsKey(descriptorType)) {
+ this.fingerprintListeners.put(descriptorType,
+ new HashSet<FingerprintListener>());
+ }
+ this.fingerprintListeners.get(descriptorType).add(listener);
+ }
+
+ public void downloadDescriptors() {
+ for (DescriptorType descriptorType : DescriptorType.values()) {
+ this.downloadDescriptors(descriptorType);
+ }
+ }
+
+ private int localFilesBefore = 0, foundRemoteFiles = 0,
+ downloadedFiles = 0, deletedLocalFiles = 0;
+
+ private void downloadDescriptors(DescriptorType descriptorType) {
+ if (!this.descriptorListeners.containsKey(descriptorType) &&
+ !this.fingerprintListeners.containsKey(descriptorType)) {
+ return;
+ }
+ DescriptorDownloader descriptorDownloader =
+ new DescriptorDownloader(descriptorType);
+ this.localFilesBefore += descriptorDownloader.statLocalFiles();
+ this.foundRemoteFiles +=
+ descriptorDownloader.fetchRemoteDirectory();
+ this.downloadedFiles += descriptorDownloader.fetchRemoteFiles();
+ this.deletedLocalFiles += descriptorDownloader.deleteOldLocalFiles();
+ }
+
+ public void readDescriptors() {
+ /* Careful when changing the order of parsing descriptor types! The
+ * various status updaters may base assumptions on this order. */
+ this.readDescriptors(DescriptorType.RELAY_SERVER_DESCRIPTORS,
+ DescriptorHistory.RELAY_SERVER_HISTORY, true);
+ this.readDescriptors(DescriptorType.RELAY_EXTRA_INFOS,
+ DescriptorHistory.RELAY_EXTRAINFO_HISTORY, true);
+ this.readDescriptors(DescriptorType.EXIT_LISTS,
+ DescriptorHistory.EXIT_LIST_HISTORY, true);
+ this.readDescriptors(DescriptorType.RELAY_CONSENSUSES,
+ DescriptorHistory.RELAY_CONSENSUS_HISTORY, true);
+ this.readDescriptors(DescriptorType.BRIDGE_SERVER_DESCRIPTORS,
+ DescriptorHistory.BRIDGE_SERVER_HISTORY, false);
+ this.readDescriptors(DescriptorType.BRIDGE_EXTRA_INFOS,
+ DescriptorHistory.BRIDGE_EXTRAINFO_HISTORY, false);
+ this.readDescriptors(DescriptorType.BRIDGE_POOL_ASSIGNMENTS,
+ DescriptorHistory.BRIDGE_POOLASSIGN_HISTORY, false);
+ this.readDescriptors(DescriptorType.BRIDGE_STATUSES,
+ DescriptorHistory.BRIDGE_STATUS_HISTORY, false);
+ }
+
+ private void readDescriptors(DescriptorType descriptorType,
+ DescriptorHistory descriptorHistory, boolean relay) {
+ if (!this.descriptorListeners.containsKey(descriptorType) &&
+ !this.fingerprintListeners.containsKey(descriptorType)) {
+ return;
+ }
+ Set<DescriptorListener> descriptorListeners =
+ this.descriptorListeners.get(descriptorType);
+ Set<FingerprintListener> fingerprintListeners =
+ this.fingerprintListeners.get(descriptorType);
+ DescriptorQueue descriptorQueue = this.getDescriptorQueue(
+ descriptorType, descriptorHistory);
+ Descriptor descriptor;
+ while ((descriptor = descriptorQueue.nextDescriptor()) != null) {
+ for (DescriptorListener descriptorListener : descriptorListeners) {
+ descriptorListener.processDescriptor(descriptor, relay);
+ }
+ if (fingerprintListeners == null) {
+ continue;
+ }
+ SortedSet<String> fingerprints = new TreeSet<String>();
+ if (descriptorType == DescriptorType.RELAY_CONSENSUSES &&
+ descriptor instanceof RelayNetworkStatusConsensus) {
+ fingerprints.addAll(((RelayNetworkStatusConsensus) descriptor).
+ getStatusEntries().keySet());
+ } else if (descriptorType
+ == DescriptorType.RELAY_SERVER_DESCRIPTORS &&
+ descriptor instanceof ServerDescriptor) {
+ fingerprints.add(((ServerDescriptor) descriptor).
+ getFingerprint());
+ } else if (descriptorType == DescriptorType.RELAY_EXTRA_INFOS &&
+ descriptor instanceof ExtraInfoDescriptor) {
+ fingerprints.add(((ExtraInfoDescriptor) descriptor).
+ getFingerprint());
+ } else if (descriptorType == DescriptorType.EXIT_LISTS &&
+ descriptor instanceof ExitList) {
+ for (ExitListEntry entry :
+ ((ExitList) descriptor).getExitListEntries()) {
+ fingerprints.add(entry.getFingerprint());
+ }
+ } else if (descriptorType == DescriptorType.BRIDGE_STATUSES &&
+ descriptor instanceof BridgeNetworkStatus) {
+ fingerprints.addAll(((BridgeNetworkStatus) descriptor).
+ getStatusEntries().keySet());
+ } else if (descriptorType ==
+ DescriptorType.BRIDGE_SERVER_DESCRIPTORS &&
+ descriptor instanceof ServerDescriptor) {
+ fingerprints.add(((ServerDescriptor) descriptor).
+ getFingerprint());
+ } else if (descriptorType == DescriptorType.BRIDGE_EXTRA_INFOS &&
+ descriptor instanceof ExtraInfoDescriptor) {
+ fingerprints.add(((ExtraInfoDescriptor) descriptor).
+ getFingerprint());
+ } else if (descriptorType ==
+ DescriptorType.BRIDGE_POOL_ASSIGNMENTS &&
+ descriptor instanceof BridgePoolAssignment) {
+ fingerprints.addAll(((BridgePoolAssignment) descriptor).
+ getEntries().keySet());
+ }
+ for (FingerprintListener fingerprintListener :
+ fingerprintListeners) {
+ fingerprintListener.processFingerprints(fingerprints, relay);
+ }
+ }
+ switch (descriptorType) {
+ case RELAY_CONSENSUSES:
+ Logger.printStatusTime("Read relay network consensuses");
+ break;
+ case RELAY_SERVER_DESCRIPTORS:
+ Logger.printStatusTime("Read relay server descriptors");
+ break;
+ case RELAY_EXTRA_INFOS:
+ Logger.printStatusTime("Read relay extra-info descriptors");
+ break;
+ case EXIT_LISTS:
+ Logger.printStatusTime("Read exit lists");
+ break;
+ case BRIDGE_STATUSES:
+ Logger.printStatusTime("Read bridge network statuses");
+ break;
+ case BRIDGE_SERVER_DESCRIPTORS:
+ Logger.printStatusTime("Read bridge server descriptors");
+ break;
+ case BRIDGE_EXTRA_INFOS:
+ Logger.printStatusTime("Read bridge extra-info descriptors");
+ break;
+ case BRIDGE_POOL_ASSIGNMENTS:
+ Logger.printStatusTime("Read bridge-pool assignments");
+ break;
+ }
+ }
+
+ public void writeHistoryFiles() {
+ for (DescriptorQueue descriptorQueue : this.descriptorQueues) {
+ descriptorQueue.writeHistoryFile();
+ }
+ }
+
+ public String getStatsString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(" " + this.localFilesBefore + " descriptor files found "
+ + "locally\n");
+ sb.append(" " + this.foundRemoteFiles + " descriptor files found "
+ + "remotely\n");
+ sb.append(" " + this.downloadedFiles + " descriptor files "
+ + "downloaded from remote\n");
+ sb.append(" " + this.deletedLocalFiles + " descriptor files "
+ + "deleted locally\n");
+ sb.append(" " + this.descriptorQueues.size() + " descriptor "
+ + "queues created\n");
+ int historySizeBefore = 0, historySizeAfter = 0;
+ long descriptors = 0L, bytes = 0L;
+ for (DescriptorQueue descriptorQueue : descriptorQueues) {
+ historySizeBefore += descriptorQueue.getHistorySizeBefore();
+ historySizeAfter += descriptorQueue.getHistorySizeAfter();
+ descriptors += descriptorQueue.getReturnedDescriptors();
+ bytes += descriptorQueue.getReturnedBytes();
+ }
+ sb.append(" " + Logger.formatDecimalNumber(historySizeBefore)
+ + " descriptors excluded from this execution\n");
+ sb.append(" " + Logger.formatDecimalNumber(descriptors)
+ + " descriptors provided\n");
+ sb.append(" " + Logger.formatBytes(bytes) + " provided\n");
+ sb.append(" " + Logger.formatDecimalNumber(historySizeAfter)
+ + " descriptors excluded from next execution\n");
+ return sb.toString();
+ }
+}
+
diff --git a/src/org/torproject/onionoo/updater/DescriptorType.java b/src/org/torproject/onionoo/updater/DescriptorType.java
new file mode 100644
index 0000000..41956da
--- /dev/null
+++ b/src/org/torproject/onionoo/updater/DescriptorType.java
@@ -0,0 +1,15 @@
+/* Copyright 2013, 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.updater;
+
+public enum DescriptorType {
+ RELAY_CONSENSUSES,
+ RELAY_SERVER_DESCRIPTORS,
+ RELAY_EXTRA_INFOS,
+ EXIT_LISTS,
+ BRIDGE_STATUSES,
+ BRIDGE_SERVER_DESCRIPTORS,
+ BRIDGE_EXTRA_INFOS,
+ BRIDGE_POOL_ASSIGNMENTS,
+}
+
diff --git a/src/org/torproject/onionoo/updater/FingerprintListener.java b/src/org/torproject/onionoo/updater/FingerprintListener.java
new file mode 100644
index 0000000..5e16eae
--- /dev/null
+++ b/src/org/torproject/onionoo/updater/FingerprintListener.java
@@ -0,0 +1,10 @@
+/* Copyright 2013, 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.updater;
+
+import java.util.SortedSet;
+
+public interface FingerprintListener {
+ abstract void processFingerprints(SortedSet<String> fingerprints,
+ boolean relay);
+}
\ No newline at end of file
diff --git a/src/org/torproject/onionoo/updater/LookupResult.java b/src/org/torproject/onionoo/updater/LookupResult.java
new file mode 100644
index 0000000..dcf3a2a
--- /dev/null
+++ b/src/org/torproject/onionoo/updater/LookupResult.java
@@ -0,0 +1,70 @@
+/* Copyright 2013 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.updater;
+
+public class LookupResult {
+
+ private String countryCode;
+ public void setCountryCode(String countryCode) {
+ this.countryCode = countryCode;
+ }
+ public String getCountryCode() {
+ return this.countryCode;
+ }
+
+ private String countryName;
+ public void setCountryName(String countryName) {
+ this.countryName = countryName;
+ }
+ public String getCountryName() {
+ return this.countryName;
+ }
+
+ private String regionName;
+ public void setRegionName(String regionName) {
+ this.regionName = regionName;
+ }
+ public String getRegionName() {
+ return this.regionName;
+ }
+
+ private String cityName;
+ public void setCityName(String cityName) {
+ this.cityName = cityName;
+ }
+ public String getCityName() {
+ return this.cityName;
+ }
+
+ private Float latitude;
+ public void setLatitude(Float latitude) {
+ this.latitude = latitude;
+ }
+ public Float getLatitude() {
+ return this.latitude;
+ }
+
+ private Float longitude;
+ public void setLongitude(Float longitude) {
+ this.longitude = longitude;
+ }
+ public Float getLongitude() {
+ return this.longitude;
+ }
+
+ private String asNumber;
+ public void setAsNumber(String asNumber) {
+ this.asNumber = asNumber;
+ }
+ public String getAsNumber() {
+ return this.asNumber;
+ }
+
+ private String asName;
+ public void setAsName(String asName) {
+ this.asName = asName;
+ }
+ public String getAsName() {
+ return this.asName;
+ }
+}
\ No newline at end of file
diff --git a/src/org/torproject/onionoo/updater/LookupService.java b/src/org/torproject/onionoo/updater/LookupService.java
new file mode 100644
index 0000000..b816091
--- /dev/null
+++ b/src/org/torproject/onionoo/updater/LookupService.java
@@ -0,0 +1,343 @@
+/* Copyright 2013 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.updater;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.regex.Pattern;
+
+import org.torproject.onionoo.util.Logger;
+
+public class LookupService {
+
+ private File geoipDir;
+ private File geoLite2CityBlocksCsvFile;
+ private File geoLite2CityLocationsCsvFile;
+ private File geoIPASNum2CsvFile;
+ private boolean hasAllFiles = false;
+ public LookupService(File geoipDir) {
+ this.geoipDir = geoipDir;
+ this.findRequiredCsvFiles();
+ }
+
+ /* Make sure we have all required .csv files. */
+ private void findRequiredCsvFiles() {
+ this.geoLite2CityBlocksCsvFile = new File(this.geoipDir,
+ "GeoLite2-City-Blocks.csv");
+ if (!this.geoLite2CityBlocksCsvFile.exists()) {
+ System.err.println("No GeoLite2-City-Blocks.csv file in geoip/.");
+ return;
+ }
+ this.geoLite2CityLocationsCsvFile = new File(this.geoipDir,
+ "GeoLite2-City-Locations.csv");
+ if (!this.geoLite2CityLocationsCsvFile.exists()) {
+ System.err.println("No GeoLite2-City-Locations.csv file in "
+ + "geoip/.");
+ return;
+ }
+ this.geoIPASNum2CsvFile = new File(this.geoipDir, "GeoIPASNum2.csv");
+ if (!this.geoIPASNum2CsvFile.exists()) {
+ System.err.println("No GeoIPASNum2.csv file in geoip/.");
+ return;
+ }
+ this.hasAllFiles = true;
+ }
+
+ private Pattern ipv4Pattern = Pattern.compile("^[0-9\\.]{7,15}$");
+ private long parseAddressString(String addressString) {
+ long addressNumber = -1L;
+ if (ipv4Pattern.matcher(addressString).matches()) {
+ String[] parts = addressString.split("\\.", 4);
+ if (parts.length == 4) {
+ addressNumber = 0L;
+ for (int i = 0; i < 4; i++) {
+ addressNumber *= 256L;
+ int octetValue = -1;
+ try {
+ octetValue = Integer.parseInt(parts[i]);
+ } catch (NumberFormatException e) {
+ }
+ if (octetValue < 0 || octetValue > 255) {
+ addressNumber = -1L;
+ break;
+ }
+ addressNumber += octetValue;
+ }
+ }
+ }
+ return addressNumber;
+ }
+
+ public SortedMap<String, LookupResult> lookup(
+ SortedSet<String> addressStrings) {
+
+ SortedMap<String, LookupResult> lookupResults =
+ new TreeMap<String, LookupResult>();
+
+ if (!this.hasAllFiles) {
+ return lookupResults;
+ }
+
+ /* Obtain a map from relay IP address strings to numbers. */
+ Map<String, Long> addressStringNumbers = new HashMap<String, Long>();
+ for (String addressString : addressStrings) {
+ long addressNumber = this.parseAddressString(addressString);
+ if (addressNumber >= 0L) {
+ addressStringNumbers.put(addressString, addressNumber);
+ }
+ }
+ if (addressStringNumbers.isEmpty()) {
+ return lookupResults;
+ }
+
+ /* Obtain a map from IP address numbers to blocks and to latitudes and
+ longitudes. */
+ Map<Long, Long> addressNumberBlocks = new HashMap<Long, Long>();
+ Map<Long, Float[]> addressNumberLatLong =
+ new HashMap<Long, Float[]>();
+ try {
+ SortedSet<Long> sortedAddressNumbers = new TreeSet<Long>(
+ addressStringNumbers.values());
+ BufferedReader br = new BufferedReader(new InputStreamReader(
+ new FileInputStream(geoLite2CityBlocksCsvFile), "ISO-8859-1"));
+ String line = br.readLine();
+ while ((line = br.readLine()) != null) {
+ if (!line.startsWith("::ffff:")) {
+ /* TODO Make this less hacky and IPv6-ready at some point. */
+ continue;
+ }
+ String[] parts = line.replaceAll("\"", "").split(",", 10);
+ if (parts.length != 10) {
+ System.err.println("Illegal line '" + line + "' in "
+ + geoLite2CityBlocksCsvFile.getAbsolutePath() + ".");
+ br.close();
+ return lookupResults;
+ }
+ try {
+ String startAddressString = parts[0].substring(7); /* ::ffff: */
+ long startIpNum = this.parseAddressString(startAddressString);
+ if (startIpNum < 0L) {
+ System.err.println("Illegal IP address in '" + line
+ + "' in " + geoLite2CityBlocksCsvFile.getAbsolutePath()
+ + ".");
+ br.close();
+ return lookupResults;
+ }
+ int networkMaskLength = Integer.parseInt(parts[1]);
+ if (networkMaskLength < 96 || networkMaskLength > 128) {
+ System.err.println("Illegal network mask in '" + line
+ + "' in " + geoLite2CityBlocksCsvFile.getAbsolutePath()
+ + ".");
+ br.close();
+ return lookupResults;
+ }
+ if (parts[2].length() == 0 && parts[3].length() == 0) {
+ continue;
+ }
+ long endIpNum = startIpNum + (1 << (128 - networkMaskLength))
+ - 1;
+ for (long addressNumber : sortedAddressNumbers.
+ tailSet(startIpNum).headSet(endIpNum + 1L)) {
+ String blockString = parts[2].length() > 0 ? parts[2] :
+ parts[3];
+ long blockNumber = Long.parseLong(blockString);
+ addressNumberBlocks.put(addressNumber, blockNumber);
+ if (parts[6].length() > 0 && parts[7].length() > 0) {
+ addressNumberLatLong.put(addressNumber,
+ new Float[] { Float.parseFloat(parts[6]),
+ Float.parseFloat(parts[7]) });
+ }
+ }
+ } catch (NumberFormatException e) {
+ System.err.println("Number format exception while parsing line "
+ + "'" + line + "' in "
+ + geoLite2CityBlocksCsvFile.getAbsolutePath() + ".");
+ br.close();
+ return lookupResults;
+ }
+ }
+ br.close();
+ } catch (IOException e) {
+ System.err.println("I/O exception while reading "
+ + geoLite2CityBlocksCsvFile.getAbsolutePath() + ".");
+ return lookupResults;
+ }
+
+ /* Obtain a map from relevant blocks to location lines. */
+ Map<Long, String> blockLocations = new HashMap<Long, String>();
+ try {
+ Set<Long> blockNumbers = new HashSet<Long>(
+ addressNumberBlocks.values());
+ BufferedReader br = new BufferedReader(new InputStreamReader(
+ new FileInputStream(geoLite2CityLocationsCsvFile),
+ "ISO-8859-1"));
+ String line = br.readLine();
+ while ((line = br.readLine()) != null) {
+ String[] parts = line.replaceAll("\"", "").split(",", 10);
+ if (parts.length != 10) {
+ System.err.println("Illegal line '" + line + "' in "
+ + geoLite2CityLocationsCsvFile.getAbsolutePath() + ".");
+ br.close();
+ return lookupResults;
+ }
+ try {
+ long locId = Long.parseLong(parts[0]);
+ if (blockNumbers.contains(locId)) {
+ blockLocations.put(locId, line);
+ }
+ } catch (NumberFormatException e) {
+ System.err.println("Number format exception while parsing line "
+ + "'" + line + "' in "
+ + geoLite2CityLocationsCsvFile.getAbsolutePath() + ".");
+ br.close();
+ return lookupResults;
+ }
+ }
+ br.close();
+ } catch (IOException e) {
+ System.err.println("I/O exception while reading "
+ + geoLite2CityLocationsCsvFile.getAbsolutePath() + ".");
+ return lookupResults;
+ }
+
+ /* Obtain a map from IP address numbers to ASN. */
+ Map<Long, String> addressNumberASN = new HashMap<Long, String>();
+ try {
+ SortedSet<Long> sortedAddressNumbers = new TreeSet<Long>(
+ addressStringNumbers.values());
+ long firstAddressNumber = sortedAddressNumbers.first();
+ BufferedReader br = new BufferedReader(new InputStreamReader(
+ new FileInputStream(geoIPASNum2CsvFile), "ISO-8859-1"));
+ String line;
+ long previousStartIpNum = -1L;
+ while ((line = br.readLine()) != null) {
+ String[] parts = line.replaceAll("\"", "").split(",", 3);
+ if (parts.length != 3) {
+ System.err.println("Illegal line '" + line + "' in "
+ + geoIPASNum2CsvFile.getAbsolutePath() + ".");
+ br.close();
+ return lookupResults;
+ }
+ try {
+ long startIpNum = Long.parseLong(parts[0]);
+ if (startIpNum <= previousStartIpNum) {
+ System.err.println("Line '" + line + "' not sorted in "
+ + geoIPASNum2CsvFile.getAbsolutePath() + ".");
+ br.close();
+ return lookupResults;
+ }
+ previousStartIpNum = startIpNum;
+ while (firstAddressNumber < startIpNum &&
+ firstAddressNumber != -1L) {
+ sortedAddressNumbers.remove(firstAddressNumber);
+ if (sortedAddressNumbers.isEmpty()) {
+ firstAddressNumber = -1L;
+ } else {
+ firstAddressNumber = sortedAddressNumbers.first();
+ }
+ }
+ long endIpNum = Long.parseLong(parts[1]);
+ while (firstAddressNumber <= endIpNum &&
+ firstAddressNumber != -1L) {
+ if (parts[2].startsWith("AS") &&
+ parts[2].split(" ", 2).length == 2) {
+ addressNumberASN.put(firstAddressNumber, parts[2]);
+ }
+ sortedAddressNumbers.remove(firstAddressNumber);
+ if (sortedAddressNumbers.isEmpty()) {
+ firstAddressNumber = -1L;
+ } else {
+ firstAddressNumber = sortedAddressNumbers.first();
+ }
+ }
+ if (firstAddressNumber == -1L) {
+ break;
+ }
+ }
+ catch (NumberFormatException e) {
+ System.err.println("Number format exception while parsing line "
+ + "'" + line + "' in "
+ + geoIPASNum2CsvFile.getAbsolutePath() + ".");
+ br.close();
+ return lookupResults;
+ }
+ }
+ br.close();
+ } catch (IOException e) {
+ System.err.println("I/O exception while reading "
+ + geoIPASNum2CsvFile.getAbsolutePath() + ".");
+ return lookupResults;
+ }
+
+ /* Finally, put together lookup results. */
+ for (String addressString : addressStrings) {
+ if (!addressStringNumbers.containsKey(addressString)) {
+ continue;
+ }
+ long addressNumber = addressStringNumbers.get(addressString);
+ if (!addressNumberBlocks.containsKey(addressNumber) &&
+ !addressNumberLatLong.containsKey(addressNumber) &&
+ !addressNumberASN.containsKey(addressNumber)) {
+ continue;
+ }
+ LookupResult lookupResult = new LookupResult();
+ if (addressNumberBlocks.containsKey(addressNumber)) {
+ long blockNumber = addressNumberBlocks.get(addressNumber);
+ if (blockLocations.containsKey(blockNumber)) {
+ String[] parts = blockLocations.get(blockNumber).
+ replaceAll("\"", "").split(",", -1);
+ lookupResult.setCountryCode(parts[3].toLowerCase());
+ if (parts[4].length() > 0) {
+ lookupResult.setCountryName(parts[4]);
+ }
+ if (parts[6].length() > 0) {
+ lookupResult.setRegionName(parts[6]);
+ }
+ if (parts[7].length() > 0) {
+ lookupResult.setCityName(parts[7]);
+ }
+ }
+ }
+ if (addressNumberLatLong.containsKey(addressNumber)) {
+ Float[] latLong = addressNumberLatLong.get(addressNumber);
+ lookupResult.setLatitude(latLong[0]);
+ lookupResult.setLongitude(latLong[1]);
+ }
+ if (addressNumberASN.containsKey(addressNumber)) {
+ String[] parts = addressNumberASN.get(addressNumber).split(" ",
+ 2);
+ lookupResult.setAsNumber(parts[0]);
+ lookupResult.setAsName(parts[1]);
+ }
+ lookupResults.put(addressString, lookupResult);
+ }
+
+ /* Keep statistics. */
+ this.addressesLookedUp += addressStrings.size();
+ this.addressesResolved += lookupResults.size();
+
+ return lookupResults;
+ }
+
+ private int addressesLookedUp = 0, addressesResolved = 0;
+
+ public String getStatsString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(" " + Logger.formatDecimalNumber(addressesLookedUp)
+ + " addresses looked up\n");
+ sb.append(" " + Logger.formatDecimalNumber(addressesResolved)
+ + " addresses resolved\n");
+ return sb.toString();
+ }
+}
diff --git a/src/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java b/src/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java
new file mode 100644
index 0000000..c687704
--- /dev/null
+++ b/src/org/torproject/onionoo/updater/NodeDetailsStatusUpdater.java
@@ -0,0 +1,626 @@
+/* Copyright 2011--2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.updater;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.torproject.descriptor.BridgeNetworkStatus;
+import org.torproject.descriptor.BridgePoolAssignment;
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.ExitList;
+import org.torproject.descriptor.ExitListEntry;
+import org.torproject.descriptor.NetworkStatusEntry;
+import org.torproject.descriptor.RelayNetworkStatusConsensus;
+import org.torproject.descriptor.ServerDescriptor;
+import org.torproject.onionoo.docs.DetailsStatus;
+import org.torproject.onionoo.docs.DocumentStore;
+import org.torproject.onionoo.docs.NodeStatus;
+import org.torproject.onionoo.util.ApplicationFactory;
+import org.torproject.onionoo.util.DateTimeHelper;
+import org.torproject.onionoo.util.Logger;
+
+public class NodeDetailsStatusUpdater implements DescriptorListener,
+ StatusUpdater {
+
+ private DescriptorSource descriptorSource;
+
+ private ReverseDomainNameResolver reverseDomainNameResolver;
+
+ private LookupService lookupService;
+
+ private DocumentStore documentStore;
+
+ private long now;
+
+ private SortedMap<String, NodeStatus> knownNodes =
+ new TreeMap<String, NodeStatus>();
+
+ private SortedMap<String, NodeStatus> relays;
+
+ private SortedMap<String, NodeStatus> bridges;
+
+ private long relaysLastValidAfterMillis = -1L;
+
+ private long bridgesLastPublishedMillis = -1L;
+
+ private SortedMap<String, Integer> lastBandwidthWeights = null;
+
+ private int relayConsensusesProcessed = 0, bridgeStatusesProcessed = 0;
+
+ public NodeDetailsStatusUpdater(
+ ReverseDomainNameResolver reverseDomainNameResolver,
+ LookupService lookupService) {
+ this.descriptorSource = ApplicationFactory.getDescriptorSource();
+ this.reverseDomainNameResolver = reverseDomainNameResolver;
+ this.lookupService = lookupService;
+ this.documentStore = ApplicationFactory.getDocumentStore();
+ this.now = ApplicationFactory.getTime().currentTimeMillis();
+ this.registerDescriptorListeners();
+ }
+
+ private void registerDescriptorListeners() {
+ this.descriptorSource.registerDescriptorListener(this,
+ DescriptorType.RELAY_CONSENSUSES);
+ this.descriptorSource.registerDescriptorListener(this,
+ DescriptorType.RELAY_SERVER_DESCRIPTORS);
+ this.descriptorSource.registerDescriptorListener(this,
+ DescriptorType.BRIDGE_STATUSES);
+ this.descriptorSource.registerDescriptorListener(this,
+ DescriptorType.BRIDGE_SERVER_DESCRIPTORS);
+ this.descriptorSource.registerDescriptorListener(this,
+ DescriptorType.BRIDGE_POOL_ASSIGNMENTS);
+ this.descriptorSource.registerDescriptorListener(this,
+ DescriptorType.EXIT_LISTS);
+ }
+
+ public void processDescriptor(Descriptor descriptor, boolean relay) {
+ if (descriptor instanceof ServerDescriptor && relay) {
+ this.processRelayServerDescriptor((ServerDescriptor) descriptor);
+ } else if (descriptor instanceof ExitList) {
+ this.processExitList((ExitList) descriptor);
+ } else if (descriptor instanceof RelayNetworkStatusConsensus) {
+ this.processRelayNetworkStatusConsensus(
+ (RelayNetworkStatusConsensus) descriptor);
+ } else if (descriptor instanceof ServerDescriptor && !relay) {
+ this.processBridgeServerDescriptor((ServerDescriptor) descriptor);
+ } else if (descriptor instanceof BridgePoolAssignment) {
+ this.processBridgePoolAssignment((BridgePoolAssignment) descriptor);
+ } else if (descriptor instanceof BridgeNetworkStatus) {
+ this.processBridgeNetworkStatus((BridgeNetworkStatus) descriptor);
+ }
+ }
+
+ private void processRelayServerDescriptor(
+ ServerDescriptor descriptor) {
+ String fingerprint = descriptor.getFingerprint();
+ DetailsStatus detailsStatus = this.documentStore.retrieve(
+ DetailsStatus.class, true, fingerprint);
+ String publishedDateTime =
+ DateTimeHelper.format(descriptor.getPublishedMillis());
+ if (detailsStatus == null) {
+ detailsStatus = new DetailsStatus();
+ } else if (detailsStatus.getDescPublished() != null &&
+ publishedDateTime.compareTo(
+ detailsStatus.getDescPublished()) < 0) {
+ return;
+ }
+ String lastRestartedString = DateTimeHelper.format(
+ descriptor.getPublishedMillis() - descriptor.getUptime()
+ * DateTimeHelper.ONE_SECOND);
+ int bandwidthRate = descriptor.getBandwidthRate();
+ int bandwidthBurst = descriptor.getBandwidthBurst();
+ int observedBandwidth = descriptor.getBandwidthObserved();
+ int advertisedBandwidth = Math.min(bandwidthRate,
+ Math.min(bandwidthBurst, observedBandwidth));
+ detailsStatus.setDescPublished(publishedDateTime);
+ detailsStatus.setLastRestarted(lastRestartedString);
+ detailsStatus.setBandwidthRate(bandwidthRate);
+ detailsStatus.setBandwidthBurst(bandwidthBurst);
+ detailsStatus.setObservedBandwidth(observedBandwidth);
+ detailsStatus.setAdvertisedBandwidth(advertisedBandwidth);
+ detailsStatus.setExitPolicy(descriptor.getExitPolicyLines());
+ detailsStatus.setContact(descriptor.getContact());
+ detailsStatus.setPlatform(descriptor.getPlatform());
+ detailsStatus.setFamily(descriptor.getFamilyEntries());
+ if (descriptor.getIpv6DefaultPolicy() != null &&
+ (descriptor.getIpv6DefaultPolicy().equals("accept") ||
+ descriptor.getIpv6DefaultPolicy().equals("reject")) &&
+ descriptor.getIpv6PortList() != null) {
+ Map<String, List<String>> exitPolicyV6Summary =
+ new HashMap<String, List<String>>();
+ List<String> portsOrPortRanges = Arrays.asList(
+ descriptor.getIpv6PortList().split(","));
+ exitPolicyV6Summary.put(descriptor.getIpv6DefaultPolicy(),
+ portsOrPortRanges);
+ detailsStatus.setExitPolicyV6Summary(exitPolicyV6Summary);
+ }
+ if (descriptor.isHibernating()) {
+ detailsStatus.setHibernating(true);
+ }
+ this.documentStore.store(detailsStatus, fingerprint);
+ }
+
+ private Map<String, Map<String, Long>> exitListEntries =
+ new HashMap<String, Map<String, Long>>();
+
+ private void processExitList(ExitList exitList) {
+ for (ExitListEntry exitListEntry : exitList.getExitListEntries()) {
+ String fingerprint = exitListEntry.getFingerprint();
+ if (exitListEntry.getScanMillis() <
+ this.now - DateTimeHelper.ONE_DAY) {
+ continue;
+ }
+ if (!this.exitListEntries.containsKey(fingerprint)) {
+ this.exitListEntries.put(fingerprint,
+ new HashMap<String, Long>());
+ }
+ String exitAddress = exitListEntry.getExitAddress();
+ long scanMillis = exitListEntry.getScanMillis();
+ if (!this.exitListEntries.get(fingerprint).containsKey(exitAddress)
+ || this.exitListEntries.get(fingerprint).get(exitAddress)
+ < scanMillis) {
+ this.exitListEntries.get(fingerprint).put(exitAddress,
+ scanMillis);
+ }
+ }
+ }
+
+ private void processRelayNetworkStatusConsensus(
+ RelayNetworkStatusConsensus consensus) {
+ long validAfterMillis = consensus.getValidAfterMillis();
+ if (validAfterMillis > this.relaysLastValidAfterMillis) {
+ this.relaysLastValidAfterMillis = validAfterMillis;
+ }
+ Set<String> recommendedVersions = null;
+ if (consensus.getRecommendedServerVersions() != null) {
+ recommendedVersions = new HashSet<String>();
+ for (String recommendedVersion :
+ consensus.getRecommendedServerVersions()) {
+ recommendedVersions.add("Tor " + recommendedVersion);
+ }
+ }
+ for (NetworkStatusEntry entry :
+ consensus.getStatusEntries().values()) {
+ String nickname = entry.getNickname();
+ String fingerprint = entry.getFingerprint();
+ String address = entry.getAddress();
+ SortedSet<String> orAddressesAndPorts = new TreeSet<String>(
+ entry.getOrAddresses());
+ int orPort = entry.getOrPort();
+ int dirPort = entry.getDirPort();
+ SortedSet<String> relayFlags = entry.getFlags();
+ long consensusWeight = entry.getBandwidth();
+ String defaultPolicy = entry.getDefaultPolicy();
+ String portList = entry.getPortList();
+ Boolean recommendedVersion = (recommendedVersions == null ||
+ entry.getVersion() == null) ? null :
+ recommendedVersions.contains(entry.getVersion());
+ NodeStatus newNodeStatus = new NodeStatus(true, nickname,
+ fingerprint, address, orAddressesAndPorts, null,
+ validAfterMillis, orPort, dirPort, relayFlags, consensusWeight,
+ null, null, -1L, defaultPolicy, portList, validAfterMillis,
+ validAfterMillis, null, null, recommendedVersion, null);
+ if (this.knownNodes.containsKey(fingerprint)) {
+ this.knownNodes.get(fingerprint).update(newNodeStatus);
+ } else {
+ this.knownNodes.put(fingerprint, newNodeStatus);
+ }
+ }
+ this.relayConsensusesProcessed++;
+ if (this.relaysLastValidAfterMillis == validAfterMillis) {
+ this.lastBandwidthWeights = consensus.getBandwidthWeights();
+ }
+ }
+
+ private void processBridgeServerDescriptor(
+ ServerDescriptor descriptor) {
+ String fingerprint = descriptor.getFingerprint();
+ DetailsStatus detailsStatus = this.documentStore.retrieve(
+ DetailsStatus.class, true, fingerprint);
+ String publishedDateTime =
+ DateTimeHelper.format(descriptor.getPublishedMillis());
+ if (detailsStatus == null) {
+ detailsStatus = new DetailsStatus();
+ } else if (detailsStatus.getDescPublished() != null &&
+ publishedDateTime.compareTo(
+ detailsStatus.getDescPublished()) < 0) {
+ return;
+ }
+ String lastRestartedString = DateTimeHelper.format(
+ descriptor.getPublishedMillis() - descriptor.getUptime()
+ * DateTimeHelper.ONE_SECOND);
+ int advertisedBandwidth = Math.min(descriptor.getBandwidthRate(),
+ Math.min(descriptor.getBandwidthBurst(),
+ descriptor.getBandwidthObserved()));
+ detailsStatus.setDescPublished(publishedDateTime);
+ detailsStatus.setLastRestarted(lastRestartedString);
+ detailsStatus.setAdvertisedBandwidth(advertisedBandwidth);
+ detailsStatus.setPlatform(descriptor.getPlatform());
+ this.documentStore.store(detailsStatus, fingerprint);
+ }
+
+ private void processBridgePoolAssignment(
+ BridgePoolAssignment bridgePoolAssignment) {
+ for (Map.Entry<String, String> e :
+ bridgePoolAssignment.getEntries().entrySet()) {
+ String fingerprint = e.getKey();
+ String details = e.getValue();
+ DetailsStatus detailsStatus = this.documentStore.retrieve(
+ DetailsStatus.class, true, fingerprint);
+ if (detailsStatus == null) {
+ detailsStatus = new DetailsStatus();
+ } else if (details.equals(detailsStatus.getPoolAssignment())) {
+ continue;
+ }
+ detailsStatus.setPoolAssignment(details);
+ this.documentStore.store(detailsStatus, fingerprint);
+ }
+ }
+
+ private void processBridgeNetworkStatus(BridgeNetworkStatus status) {
+ long publishedMillis = status.getPublishedMillis();
+ if (publishedMillis > this.bridgesLastPublishedMillis) {
+ this.bridgesLastPublishedMillis = publishedMillis;
+ }
+ for (NetworkStatusEntry entry : status.getStatusEntries().values()) {
+ String nickname = entry.getNickname();
+ String fingerprint = entry.getFingerprint();
+ String address = entry.getAddress();
+ SortedSet<String> orAddressesAndPorts = new TreeSet<String>(
+ entry.getOrAddresses());
+ int orPort = entry.getOrPort();
+ int dirPort = entry.getDirPort();
+ SortedSet<String> relayFlags = entry.getFlags();
+ NodeStatus newNodeStatus = new NodeStatus(false, nickname,
+ fingerprint, address, orAddressesAndPorts, null,
+ publishedMillis, orPort, dirPort, relayFlags, -1L, "??", null,
+ -1L, null, null, publishedMillis, -1L, null, null, null, null);
+ if (this.knownNodes.containsKey(fingerprint)) {
+ this.knownNodes.get(fingerprint).update(newNodeStatus);
+ } else {
+ this.knownNodes.put(fingerprint, newNodeStatus);
+ }
+ }
+ this.bridgeStatusesProcessed++;
+ }
+
+ public void updateStatuses() {
+ this.readStatusSummary();
+ Logger.printStatusTime("Read status summary");
+ this.setCurrentNodes();
+ Logger.printStatusTime("Set current node fingerprints");
+ this.startReverseDomainNameLookups();
+ Logger.printStatusTime("Started reverse domain name lookups");
+ this.lookUpCitiesAndASes();
+ Logger.printStatusTime("Looked up cities and ASes");
+ this.setDescriptorPartsOfNodeStatus();
+ Logger.printStatusTime("Set descriptor parts of node statuses.");
+ this.calculatePathSelectionProbabilities();
+ Logger.printStatusTime("Calculated path selection probabilities");
+ this.finishReverseDomainNameLookups();
+ Logger.printStatusTime("Finished reverse domain name lookups");
+ this.writeStatusSummary();
+ Logger.printStatusTime("Wrote status summary");
+ this.updateDetailsStatuses();
+ Logger.printStatusTime("Updated exit addresses in details statuses");
+ }
+
+ private void readStatusSummary() {
+ SortedSet<String> fingerprints = this.documentStore.list(
+ NodeStatus.class);
+ for (String fingerprint : fingerprints) {
+ NodeStatus node = this.documentStore.retrieve(NodeStatus.class,
+ true, fingerprint);
+ if (node.isRelay()) {
+ this.relaysLastValidAfterMillis = Math.max(
+ this.relaysLastValidAfterMillis, node.getLastSeenMillis());
+ } else {
+ this.bridgesLastPublishedMillis = Math.max(
+ this.bridgesLastPublishedMillis, node.getLastSeenMillis());
+ }
+ if (this.knownNodes.containsKey(fingerprint)) {
+ this.knownNodes.get(fingerprint).update(node);
+ } else {
+ this.knownNodes.put(fingerprint, node);
+ }
+ }
+ }
+
+ private void setCurrentNodes() {
+ long cutoff = Math.max(this.relaysLastValidAfterMillis,
+ this.bridgesLastPublishedMillis) - 7L * 24L * 60L * 60L * 1000L;
+ SortedMap<String, NodeStatus> currentNodes =
+ new TreeMap<String, NodeStatus>();
+ for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) {
+ if (e.getValue().getLastSeenMillis() >= cutoff) {
+ currentNodes.put(e.getKey(), e.getValue());
+ }
+ }
+ this.relays = new TreeMap<String, NodeStatus>();
+ this.bridges = new TreeMap<String, NodeStatus>();
+ for (Map.Entry<String, NodeStatus> e : currentNodes.entrySet()) {
+ if (e.getValue().isRelay()) {
+ this.relays.put(e.getKey(), e.getValue());
+ } else {
+ this.bridges.put(e.getKey(), e.getValue());
+ }
+ }
+ }
+
+ private void startReverseDomainNameLookups() {
+ Map<String, Long> addressLastLookupTimes =
+ new HashMap<String, Long>();
+ for (NodeStatus relay : relays.values()) {
+ addressLastLookupTimes.put(relay.getAddress(),
+ relay.getLastRdnsLookup());
+ }
+ this.reverseDomainNameResolver.setAddresses(addressLastLookupTimes);
+ this.reverseDomainNameResolver.startReverseDomainNameLookups();
+ }
+
+ private void lookUpCitiesAndASes() {
+ SortedSet<String> addressStrings = new TreeSet<String>();
+ for (NodeStatus node : this.knownNodes.values()) {
+ if (node.isRelay()) {
+ addressStrings.add(node.getAddress());
+ }
+ }
+ if (addressStrings.isEmpty()) {
+ System.err.println("No relay IP addresses to resolve to cities or "
+ + "ASN.");
+ return;
+ }
+ SortedMap<String, LookupResult> lookupResults =
+ this.lookupService.lookup(addressStrings);
+ for (NodeStatus node : knownNodes.values()) {
+ if (!node.isRelay()) {
+ continue;
+ }
+ String addressString = node.getAddress();
+ if (lookupResults.containsKey(addressString)) {
+ LookupResult lookupResult = lookupResults.get(addressString);
+ node.setCountryCode(lookupResult.getCountryCode());
+ node.setCountryName(lookupResult.getCountryName());
+ node.setRegionName(lookupResult.getRegionName());
+ node.setCityName(lookupResult.getCityName());
+ node.setLatitude(lookupResult.getLatitude());
+ node.setLongitude(lookupResult.getLongitude());
+ node.setASNumber(lookupResult.getAsNumber());
+ node.setASName(lookupResult.getAsName());
+ }
+ }
+ }
+
+ private void setDescriptorPartsOfNodeStatus() {
+ for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) {
+ String fingerprint = e.getKey();
+ NodeStatus node = e.getValue();
+ if (node.isRelay()) {
+ if (node.getRelayFlags().contains("Running") &&
+ node.getLastSeenMillis() == this.relaysLastValidAfterMillis) {
+ node.setRunning(true);
+ }
+ DetailsStatus detailsStatus = this.documentStore.retrieve(
+ DetailsStatus.class, true, fingerprint);
+ if (detailsStatus != null) {
+ node.setContact(detailsStatus.getContact());
+ if (detailsStatus.getExitAddresses() != null) {
+ for (Map.Entry<String, Long> ea :
+ detailsStatus.getExitAddresses().entrySet()) {
+ if (ea.getValue() >= this.now - DateTimeHelper.ONE_DAY) {
+ node.addExitAddress(ea.getKey());
+ }
+ }
+ }
+ if (detailsStatus.getFamily() != null &&
+ !detailsStatus.getFamily().isEmpty()) {
+ SortedSet<String> familyFingerprints = new TreeSet<String>();
+ for (String familyMember : detailsStatus.getFamily()) {
+ if (familyMember.startsWith("$") &&
+ familyMember.length() == 41) {
+ familyFingerprints.add(familyMember.substring(1));
+ }
+ }
+ if (!familyFingerprints.isEmpty()) {
+ node.setFamilyFingerprints(familyFingerprints);
+ }
+ }
+ }
+ }
+ if (!node.isRelay() && node.getRelayFlags().contains("Running") &&
+ node.getLastSeenMillis() == this.bridgesLastPublishedMillis) {
+ node.setRunning(true);
+ }
+ }
+ }
+
+ private void calculatePathSelectionProbabilities() {
+ boolean consensusContainsBandwidthWeights = false;
+ double wgg = 0.0, wgd = 0.0, wmg = 0.0, wmm = 0.0, wme = 0.0,
+ wmd = 0.0, wee = 0.0, wed = 0.0;
+ if (this.lastBandwidthWeights != null) {
+ SortedSet<String> weightKeys = new TreeSet<String>(Arrays.asList(
+ "Wgg,Wgd,Wmg,Wmm,Wme,Wmd,Wee,Wed".split(",")));
+ weightKeys.removeAll(this.lastBandwidthWeights.keySet());
+ if (weightKeys.isEmpty()) {
+ consensusContainsBandwidthWeights = true;
+ wgg = ((double) this.lastBandwidthWeights.get("Wgg")) / 10000.0;
+ wgd = ((double) this.lastBandwidthWeights.get("Wgd")) / 10000.0;
+ wmg = ((double) this.lastBandwidthWeights.get("Wmg")) / 10000.0;
+ wmm = ((double) this.lastBandwidthWeights.get("Wmm")) / 10000.0;
+ wme = ((double) this.lastBandwidthWeights.get("Wme")) / 10000.0;
+ wmd = ((double) this.lastBandwidthWeights.get("Wmd")) / 10000.0;
+ wee = ((double) this.lastBandwidthWeights.get("Wee")) / 10000.0;
+ wed = ((double) this.lastBandwidthWeights.get("Wed")) / 10000.0;
+ }
+ } else {
+ System.err.println("Could not determine most recent Wxx parameter "
+ + "values, probably because we didn't parse a consensus in "
+ + "this execution. All relays' guard/middle/exit weights are "
+ + "going to be 0.0.");
+ }
+ SortedMap<String, Double>
+ advertisedBandwidths = new TreeMap<String, Double>(),
+ consensusWeights = new TreeMap<String, Double>(),
+ guardWeights = new TreeMap<String, Double>(),
+ middleWeights = new TreeMap<String, Double>(),
+ exitWeights = new TreeMap<String, Double>();
+ double totalAdvertisedBandwidth = 0.0;
+ double totalConsensusWeight = 0.0;
+ double totalGuardWeight = 0.0;
+ double totalMiddleWeight = 0.0;
+ double totalExitWeight = 0.0;
+ for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) {
+ String fingerprint = e.getKey();
+ NodeStatus relay = e.getValue();
+ if (!relay.getRunning()) {
+ continue;
+ }
+ boolean isExit = relay.getRelayFlags().contains("Exit") &&
+ !relay.getRelayFlags().contains("BadExit");
+ boolean isGuard = relay.getRelayFlags().contains("Guard");
+ DetailsStatus detailsStatus = this.documentStore.retrieve(
+ DetailsStatus.class, true, fingerprint);
+ if (detailsStatus != null) {
+ double advertisedBandwidth =
+ detailsStatus.getAdvertisedBandwidth();
+ if (advertisedBandwidth >= 0.0) {
+ advertisedBandwidths.put(fingerprint, advertisedBandwidth);
+ totalAdvertisedBandwidth += advertisedBandwidth;
+ }
+ }
+ double consensusWeight = (double) relay.getConsensusWeight();
+ consensusWeights.put(fingerprint, consensusWeight);
+ totalConsensusWeight += consensusWeight;
+ if (consensusContainsBandwidthWeights) {
+ double guardWeight = consensusWeight,
+ middleWeight = consensusWeight,
+ exitWeight = consensusWeight;
+ if (isGuard && isExit) {
+ guardWeight *= wgd;
+ middleWeight *= wmd;
+ exitWeight *= wed;
+ } else if (isGuard) {
+ guardWeight *= wgg;
+ middleWeight *= wmg;
+ exitWeight = 0.0;
+ } else if (isExit) {
+ guardWeight = 0.0;
+ middleWeight *= wme;
+ exitWeight *= wee;
+ } else {
+ guardWeight = 0.0;
+ middleWeight *= wmm;
+ exitWeight = 0.0;
+ }
+ guardWeights.put(fingerprint, guardWeight);
+ middleWeights.put(fingerprint, middleWeight);
+ exitWeights.put(fingerprint, exitWeight);
+ totalGuardWeight += guardWeight;
+ totalMiddleWeight += middleWeight;
+ totalExitWeight += exitWeight;
+ }
+ }
+ for (Map.Entry<String, NodeStatus> e : this.relays.entrySet()) {
+ String fingerprint = e.getKey();
+ NodeStatus relay = e.getValue();
+ if (advertisedBandwidths.containsKey(fingerprint)) {
+ relay.setAdvertisedBandwidthFraction(advertisedBandwidths.get(
+ fingerprint) / totalAdvertisedBandwidth);
+ }
+ if (consensusWeights.containsKey(fingerprint)) {
+ relay.setConsensusWeightFraction(consensusWeights.get(fingerprint)
+ / totalConsensusWeight);
+ }
+ if (guardWeights.containsKey(fingerprint)) {
+ relay.setGuardProbability(guardWeights.get(fingerprint)
+ / totalGuardWeight);
+ }
+ if (middleWeights.containsKey(fingerprint)) {
+ relay.setMiddleProbability(middleWeights.get(fingerprint)
+ / totalMiddleWeight);
+ }
+ if (exitWeights.containsKey(fingerprint)) {
+ relay.setExitProbability(exitWeights.get(fingerprint)
+ / totalExitWeight);
+ }
+ }
+ }
+
+ private void finishReverseDomainNameLookups() {
+ this.reverseDomainNameResolver.finishReverseDomainNameLookups();
+ Map<String, String> lookupResults =
+ this.reverseDomainNameResolver.getLookupResults();
+ long startedRdnsLookups =
+ this.reverseDomainNameResolver.getLookupStartMillis();
+ for (NodeStatus relay : relays.values()) {
+ if (lookupResults.containsKey(relay.getAddress())) {
+ relay.setHostName(lookupResults.get(relay.getAddress()));
+ relay.setLastRdnsLookup(startedRdnsLookups);
+ }
+ }
+ }
+
+ private void writeStatusSummary() {
+ for (Map.Entry<String, NodeStatus> e : this.knownNodes.entrySet()) {
+ this.documentStore.store(e.getValue(), e.getKey());
+ }
+ }
+
+ private void updateDetailsStatuses() {
+ SortedSet<String> fingerprints = new TreeSet<String>();
+ fingerprints.addAll(this.exitListEntries.keySet());
+ for (String fingerprint : fingerprints) {
+ DetailsStatus detailsStatus = this.documentStore.retrieve(
+ DetailsStatus.class, true, fingerprint);
+ if (detailsStatus == null) {
+ detailsStatus = new DetailsStatus();
+ }
+ Map<String, Long> exitAddresses = new HashMap<String, Long>();
+ if (detailsStatus.getExitAddresses() != null) {
+ for (Map.Entry<String, Long> e :
+ detailsStatus.getExitAddresses().entrySet()) {
+ if (e.getValue() >= this.now - DateTimeHelper.ONE_DAY) {
+ exitAddresses.put(e.getKey(), e.getValue());
+ }
+ }
+ }
+ if (this.exitListEntries.containsKey(fingerprint)) {
+ for (Map.Entry<String, Long> e :
+ this.exitListEntries.get(fingerprint).entrySet()) {
+ if (!exitAddresses.containsKey(e.getKey()) ||
+ exitAddresses.get(e.getKey()) < e.getValue()) {
+ exitAddresses.put(e.getKey(), e.getValue());
+ }
+ }
+ }
+ if (this.knownNodes.containsKey(fingerprint)) {
+ for (String orAddress :
+ this.knownNodes.get(fingerprint).getOrAddresses()) {
+ this.exitListEntries.remove(orAddress);
+ }
+ }
+ detailsStatus.setExitAddresses(exitAddresses);
+ this.documentStore.store(detailsStatus, fingerprint);
+ }
+ }
+
+ public String getStatsString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(" " + Logger.formatDecimalNumber(
+ relayConsensusesProcessed) + " relay consensuses processed\n");
+ sb.append(" " + Logger.formatDecimalNumber(bridgeStatusesProcessed)
+ + " bridge statuses processed\n");
+ return sb.toString();
+ }
+}
+
diff --git a/src/org/torproject/onionoo/updater/ReverseDomainNameResolver.java b/src/org/torproject/onionoo/updater/ReverseDomainNameResolver.java
new file mode 100644
index 0000000..8ca7eb4
--- /dev/null
+++ b/src/org/torproject/onionoo/updater/ReverseDomainNameResolver.java
@@ -0,0 +1,179 @@
+/* Copyright 2013 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.updater;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.torproject.onionoo.util.ApplicationFactory;
+import org.torproject.onionoo.util.DateTimeHelper;
+import org.torproject.onionoo.util.Logger;
+import org.torproject.onionoo.util.Time;
+
+public class ReverseDomainNameResolver {
+
+ private class RdnsLookupWorker extends Thread {
+ public void run() {
+ while (time.currentTimeMillis() - RDNS_LOOKUP_MAX_DURATION_MILLIS
+ <= startedRdnsLookups) {
+ String rdnsLookupJob = null;
+ synchronized (rdnsLookupJobs) {
+ for (String job : rdnsLookupJobs) {
+ rdnsLookupJob = job;
+ rdnsLookupJobs.remove(job);
+ break;
+ }
+ }
+ if (rdnsLookupJob == null) {
+ break;
+ }
+ RdnsLookupRequest request = new RdnsLookupRequest(this,
+ rdnsLookupJob);
+ request.setDaemon(true);
+ request.start();
+ try {
+ Thread.sleep(RDNS_LOOKUP_MAX_REQUEST_MILLIS);
+ } catch (InterruptedException e) {
+ /* Getting interrupted should be the default case. */
+ }
+ String hostName = request.getHostName();
+ if (hostName != null) {
+ synchronized (rdnsLookupResults) {
+ rdnsLookupResults.put(rdnsLookupJob, hostName);
+ }
+ }
+ long lookupMillis = request.getLookupMillis();
+ if (lookupMillis >= 0L) {
+ synchronized (rdnsLookupMillis) {
+ rdnsLookupMillis.add(lookupMillis);
+ }
+ }
+ }
+ }
+ }
+
+ private class RdnsLookupRequest extends Thread {
+ private RdnsLookupWorker parent;
+ private String address, hostName;
+ private long lookupStartedMillis = -1L, lookupCompletedMillis = -1L;
+ public RdnsLookupRequest(RdnsLookupWorker parent, String address) {
+ this.parent = parent;
+ this.address = address;
+ }
+ public void run() {
+ this.lookupStartedMillis = time.currentTimeMillis();
+ try {
+ String result = InetAddress.getByName(this.address).getHostName();
+ synchronized (this) {
+ this.hostName = result;
+ }
+ } catch (UnknownHostException e) {
+ /* We'll try again the next time. */
+ }
+ this.lookupCompletedMillis = time.currentTimeMillis();
+ this.parent.interrupt();
+ }
+ public synchronized String getHostName() {
+ return hostName;
+ }
+ public synchronized long getLookupMillis() {
+ return this.lookupCompletedMillis - this.lookupStartedMillis;
+ }
+ }
+
+ private Time time;
+
+ public ReverseDomainNameResolver() {
+ this.time = ApplicationFactory.getTime();
+ }
+
+ private static final long RDNS_LOOKUP_MAX_REQUEST_MILLIS =
+ DateTimeHelper.TEN_SECONDS;
+ private static final long RDNS_LOOKUP_MAX_DURATION_MILLIS =
+ DateTimeHelper.FIVE_MINUTES;
+ private static final long RDNS_LOOKUP_MAX_AGE_MILLIS =
+ DateTimeHelper.TWELVE_HOURS;
+ private static final int RDNS_LOOKUP_WORKERS_NUM = 5;
+
+ private Map<String, Long> addressLastLookupTimes;
+
+ private Set<String> rdnsLookupJobs;
+
+ private Map<String, String> rdnsLookupResults;
+
+ private List<Long> rdnsLookupMillis;
+
+ private long startedRdnsLookups;
+
+ private List<RdnsLookupWorker> rdnsLookupWorkers;
+
+ public void setAddresses(Map<String, Long> addressLastLookupTimes) {
+ this.addressLastLookupTimes = addressLastLookupTimes;
+ }
+
+ public void startReverseDomainNameLookups() {
+ this.startedRdnsLookups = this.time.currentTimeMillis();
+ this.rdnsLookupJobs = new HashSet<String>();
+ for (Map.Entry<String, Long> e :
+ this.addressLastLookupTimes.entrySet()) {
+ if (e.getValue() < this.startedRdnsLookups
+ - RDNS_LOOKUP_MAX_AGE_MILLIS) {
+ this.rdnsLookupJobs.add(e.getKey());
+ }
+ }
+ this.rdnsLookupResults = new HashMap<String, String>();
+ this.rdnsLookupMillis = new ArrayList<Long>();
+ this.rdnsLookupWorkers = new ArrayList<RdnsLookupWorker>();
+ for (int i = 0; i < RDNS_LOOKUP_WORKERS_NUM; i++) {
+ RdnsLookupWorker rdnsLookupWorker = new RdnsLookupWorker();
+ this.rdnsLookupWorkers.add(rdnsLookupWorker);
+ rdnsLookupWorker.setDaemon(true);
+ rdnsLookupWorker.start();
+ }
+ }
+
+ public void finishReverseDomainNameLookups() {
+ for (RdnsLookupWorker rdnsLookupWorker : this.rdnsLookupWorkers) {
+ try {
+ rdnsLookupWorker.join();
+ } catch (InterruptedException e) {
+ /* This is not something that we can take care of. Just leave the
+ * worker thread alone. */
+ }
+ }
+ }
+
+ public Map<String, String> getLookupResults() {
+ synchronized (this.rdnsLookupResults) {
+ return new HashMap<String, String>(this.rdnsLookupResults);
+ }
+ }
+
+ public long getLookupStartMillis() {
+ return this.startedRdnsLookups;
+ }
+
+ public String getStatsString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(" " + Logger.formatDecimalNumber(rdnsLookupMillis.size())
+ + " lookups performed\n");
+ if (rdnsLookupMillis.size() > 0) {
+ Collections.sort(rdnsLookupMillis);
+ sb.append(" " + Logger.formatMillis(rdnsLookupMillis.get(0))
+ + " minimum lookup time\n");
+ sb.append(" " + Logger.formatMillis(rdnsLookupMillis.get(
+ rdnsLookupMillis.size() / 2)) + " median lookup time\n");
+ sb.append(" " + Logger.formatMillis(rdnsLookupMillis.get(
+ rdnsLookupMillis.size() - 1)) + " maximum lookup time\n");
+ }
+ return sb.toString();
+ }
+}
+
diff --git a/src/org/torproject/onionoo/updater/StatusUpdater.java b/src/org/torproject/onionoo/updater/StatusUpdater.java
new file mode 100644
index 0000000..9fc34d3
--- /dev/null
+++ b/src/org/torproject/onionoo/updater/StatusUpdater.java
@@ -0,0 +1,11 @@
+/* Copyright 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.updater;
+
+public interface StatusUpdater {
+
+ public abstract void updateStatuses();
+
+ public abstract String getStatsString();
+}
+
diff --git a/src/org/torproject/onionoo/updater/UptimeStatusUpdater.java b/src/org/torproject/onionoo/updater/UptimeStatusUpdater.java
new file mode 100644
index 0000000..dd71e74
--- /dev/null
+++ b/src/org/torproject/onionoo/updater/UptimeStatusUpdater.java
@@ -0,0 +1,130 @@
+/* Copyright 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.updater;
+
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.torproject.descriptor.BridgeNetworkStatus;
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.NetworkStatusEntry;
+import org.torproject.descriptor.RelayNetworkStatusConsensus;
+import org.torproject.onionoo.docs.UptimeStatus;
+import org.torproject.onionoo.util.ApplicationFactory;
+import org.torproject.onionoo.util.DateTimeHelper;
+import org.torproject.onionoo.util.Logger;
+
+public class UptimeStatusUpdater implements DescriptorListener,
+ StatusUpdater {
+
+ private DescriptorSource descriptorSource;
+
+ public UptimeStatusUpdater() {
+ this.descriptorSource = ApplicationFactory.getDescriptorSource();
+ this.registerDescriptorListeners();
+ }
+
+ private void registerDescriptorListeners() {
+ this.descriptorSource.registerDescriptorListener(this,
+ DescriptorType.RELAY_CONSENSUSES);
+ this.descriptorSource.registerDescriptorListener(this,
+ DescriptorType.BRIDGE_STATUSES);
+ }
+
+ public void processDescriptor(Descriptor descriptor, boolean relay) {
+ if (descriptor instanceof RelayNetworkStatusConsensus) {
+ this.processRelayNetworkStatusConsensus(
+ (RelayNetworkStatusConsensus) descriptor);
+ } else if (descriptor instanceof BridgeNetworkStatus) {
+ this.processBridgeNetworkStatus(
+ (BridgeNetworkStatus) descriptor);
+ }
+ }
+
+ private SortedSet<Long> newRelayStatuses = new TreeSet<Long>(),
+ newBridgeStatuses = new TreeSet<Long>();
+ private SortedMap<String, SortedSet<Long>>
+ newRunningRelays = new TreeMap<String, SortedSet<Long>>(),
+ newRunningBridges = new TreeMap<String, SortedSet<Long>>();
+
+ private void processRelayNetworkStatusConsensus(
+ RelayNetworkStatusConsensus consensus) {
+ SortedSet<String> fingerprints = new TreeSet<String>();
+ for (NetworkStatusEntry entry :
+ consensus.getStatusEntries().values()) {
+ if (entry.getFlags().contains("Running")) {
+ fingerprints.add(entry.getFingerprint());
+ }
+ }
+ if (!fingerprints.isEmpty()) {
+ long dateHourMillis = (consensus.getValidAfterMillis()
+ / DateTimeHelper.ONE_HOUR) * DateTimeHelper.ONE_HOUR;
+ for (String fingerprint : fingerprints) {
+ if (!this.newRunningRelays.containsKey(fingerprint)) {
+ this.newRunningRelays.put(fingerprint, new TreeSet<Long>());
+ }
+ this.newRunningRelays.get(fingerprint).add(dateHourMillis);
+ }
+ this.newRelayStatuses.add(dateHourMillis);
+ }
+ }
+
+ private void processBridgeNetworkStatus(BridgeNetworkStatus status) {
+ SortedSet<String> fingerprints = new TreeSet<String>();
+ for (NetworkStatusEntry entry :
+ status.getStatusEntries().values()) {
+ if (entry.getFlags().contains("Running")) {
+ fingerprints.add(entry.getFingerprint());
+ }
+ }
+ if (!fingerprints.isEmpty()) {
+ long dateHourMillis = (status.getPublishedMillis()
+ / DateTimeHelper.ONE_HOUR) * DateTimeHelper.ONE_HOUR;
+ for (String fingerprint : fingerprints) {
+ if (!this.newRunningBridges.containsKey(fingerprint)) {
+ this.newRunningBridges.put(fingerprint, new TreeSet<Long>());
+ }
+ this.newRunningBridges.get(fingerprint).add(dateHourMillis);
+ }
+ this.newBridgeStatuses.add(dateHourMillis);
+ }
+ }
+
+ public void updateStatuses() {
+ for (Map.Entry<String, SortedSet<Long>> e :
+ this.newRunningRelays.entrySet()) {
+ this.updateStatus(true, e.getKey(), e.getValue());
+ }
+ this.updateStatus(true, null, this.newRelayStatuses);
+ for (Map.Entry<String, SortedSet<Long>> e :
+ this.newRunningBridges.entrySet()) {
+ this.updateStatus(false, e.getKey(), e.getValue());
+ }
+ this.updateStatus(false, null, this.newBridgeStatuses);
+ }
+
+ private void updateStatus(boolean relay, String fingerprint,
+ SortedSet<Long> newUptimeHours) {
+ UptimeStatus uptimeStatus = UptimeStatus.loadOrCreate(fingerprint);
+ uptimeStatus.addToHistory(relay, newUptimeHours);
+ uptimeStatus.storeIfChanged();
+ }
+
+ public String getStatsString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(" " + Logger.formatDecimalNumber(
+ this.newRelayStatuses.size()) + " hours of relay uptimes "
+ + "processed\n");
+ sb.append(" " + Logger.formatDecimalNumber(
+ this.newBridgeStatuses.size()) + " hours of bridge uptimes "
+ + "processed\n");
+ sb.append(" " + Logger.formatDecimalNumber(
+ this.newRunningRelays.size() + this.newRunningBridges.size())
+ + " uptime status files updated\n");
+ return sb.toString();
+ }
+}
+
diff --git a/src/org/torproject/onionoo/updater/WeightsStatusUpdater.java b/src/org/torproject/onionoo/updater/WeightsStatusUpdater.java
new file mode 100644
index 0000000..333afcc
--- /dev/null
+++ b/src/org/torproject/onionoo/updater/WeightsStatusUpdater.java
@@ -0,0 +1,332 @@
+/* Copyright 2012--2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.updater;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.NetworkStatusEntry;
+import org.torproject.descriptor.RelayNetworkStatusConsensus;
+import org.torproject.descriptor.ServerDescriptor;
+import org.torproject.onionoo.docs.DocumentStore;
+import org.torproject.onionoo.docs.WeightsStatus;
+import org.torproject.onionoo.util.ApplicationFactory;
+import org.torproject.onionoo.util.DateTimeHelper;
+
+public class WeightsStatusUpdater implements DescriptorListener,
+ StatusUpdater {
+
+ private DescriptorSource descriptorSource;
+
+ private DocumentStore documentStore;
+
+ private long now;
+
+ public WeightsStatusUpdater() {
+ this.descriptorSource = ApplicationFactory.getDescriptorSource();
+ this.documentStore = ApplicationFactory.getDocumentStore();
+ this.now = ApplicationFactory.getTime().currentTimeMillis();
+ this.registerDescriptorListeners();
+ }
+
+ private void registerDescriptorListeners() {
+ this.descriptorSource.registerDescriptorListener(this,
+ DescriptorType.RELAY_CONSENSUSES);
+ this.descriptorSource.registerDescriptorListener(this,
+ DescriptorType.RELAY_SERVER_DESCRIPTORS);
+ }
+
+ public void processDescriptor(Descriptor descriptor, boolean relay) {
+ if (descriptor instanceof ServerDescriptor) {
+ this.processRelayServerDescriptor((ServerDescriptor) descriptor);
+ } else if (descriptor instanceof RelayNetworkStatusConsensus) {
+ this.processRelayNetworkConsensus(
+ (RelayNetworkStatusConsensus) descriptor);
+ }
+ }
+
+ public void updateStatuses() {
+ /* Nothing to do. */
+ }
+
+ private void processRelayNetworkConsensus(
+ RelayNetworkStatusConsensus consensus) {
+ long validAfterMillis = consensus.getValidAfterMillis(),
+ freshUntilMillis = consensus.getFreshUntilMillis();
+ SortedMap<String, double[]> pathSelectionWeights =
+ this.calculatePathSelectionProbabilities(consensus);
+ this.updateWeightsHistory(validAfterMillis, freshUntilMillis,
+ pathSelectionWeights);
+ }
+
+ private void processRelayServerDescriptor(
+ ServerDescriptor serverDescriptor) {
+ String digest = serverDescriptor.getServerDescriptorDigest().
+ toUpperCase();
+ int advertisedBandwidth = Math.min(Math.min(
+ serverDescriptor.getBandwidthBurst(),
+ serverDescriptor.getBandwidthObserved()),
+ serverDescriptor.getBandwidthRate());
+ String fingerprint = serverDescriptor.getFingerprint();
+ WeightsStatus weightsStatus = this.documentStore.retrieve(
+ WeightsStatus.class, true, fingerprint);
+ if (weightsStatus == null) {
+ weightsStatus = new WeightsStatus();
+ }
+ weightsStatus.getAdvertisedBandwidths().put(digest,
+ advertisedBandwidth);
+ this.documentStore.store(weightsStatus, fingerprint);
+}
+
+ private void updateWeightsHistory(long validAfterMillis,
+ long freshUntilMillis,
+ SortedMap<String, double[]> pathSelectionWeights) {
+ String fingerprint = null;
+ double[] weights = null;
+ do {
+ fingerprint = null;
+ synchronized (pathSelectionWeights) {
+ if (!pathSelectionWeights.isEmpty()) {
+ fingerprint = pathSelectionWeights.firstKey();
+ weights = pathSelectionWeights.remove(fingerprint);
+ }
+ }
+ if (fingerprint != null) {
+ this.addToHistory(fingerprint, validAfterMillis,
+ freshUntilMillis, weights);
+ }
+ } while (fingerprint != null);
+ }
+
+ private SortedMap<String, double[]> calculatePathSelectionProbabilities(
+ RelayNetworkStatusConsensus consensus) {
+ boolean containsBandwidthWeights = false;
+ double wgg = 1.0, wgd = 1.0, wmg = 1.0, wmm = 1.0, wme = 1.0,
+ wmd = 1.0, wee = 1.0, wed = 1.0;
+ SortedMap<String, Integer> bandwidthWeights =
+ consensus.getBandwidthWeights();
+ if (bandwidthWeights != null) {
+ SortedSet<String> missingWeightKeys = new TreeSet<String>(
+ Arrays.asList("Wgg,Wgd,Wmg,Wmm,Wme,Wmd,Wee,Wed".split(",")));
+ missingWeightKeys.removeAll(bandwidthWeights.keySet());
+ if (missingWeightKeys.isEmpty()) {
+ wgg = ((double) bandwidthWeights.get("Wgg")) / 10000.0;
+ wgd = ((double) bandwidthWeights.get("Wgd")) / 10000.0;
+ wmg = ((double) bandwidthWeights.get("Wmg")) / 10000.0;
+ wmm = ((double) bandwidthWeights.get("Wmm")) / 10000.0;
+ wme = ((double) bandwidthWeights.get("Wme")) / 10000.0;
+ wmd = ((double) bandwidthWeights.get("Wmd")) / 10000.0;
+ wee = ((double) bandwidthWeights.get("Wee")) / 10000.0;
+ wed = ((double) bandwidthWeights.get("Wed")) / 10000.0;
+ containsBandwidthWeights = true;
+ }
+ }
+ SortedMap<String, Double>
+ advertisedBandwidths = new TreeMap<String, Double>(),
+ consensusWeights = new TreeMap<String, Double>(),
+ guardWeights = new TreeMap<String, Double>(),
+ middleWeights = new TreeMap<String, Double>(),
+ exitWeights = new TreeMap<String, Double>();
+ double totalAdvertisedBandwidth = 0.0;
+ double totalConsensusWeight = 0.0;
+ double totalGuardWeight = 0.0;
+ double totalMiddleWeight = 0.0;
+ double totalExitWeight = 0.0;
+ for (NetworkStatusEntry relay :
+ consensus.getStatusEntries().values()) {
+ String fingerprint = relay.getFingerprint();
+ if (!relay.getFlags().contains("Running")) {
+ continue;
+ }
+ String digest = relay.getDescriptor().toUpperCase();
+ WeightsStatus weightsStatus = this.documentStore.retrieve(
+ WeightsStatus.class, true, fingerprint);
+ if (weightsStatus != null &&
+ weightsStatus.getAdvertisedBandwidths() != null &&
+ weightsStatus.getAdvertisedBandwidths().containsKey(digest)) {
+ /* Read advertised bandwidth from weights status file. Server
+ * descriptors are parsed before consensuses, so we're sure that
+ * if there's a server descriptor for this relay, it'll be
+ * contained in the weights status file by now. */
+ double advertisedBandwidth =
+ (double) weightsStatus.getAdvertisedBandwidths().get(digest);
+ advertisedBandwidths.put(fingerprint, advertisedBandwidth);
+ totalAdvertisedBandwidth += advertisedBandwidth;
+ }
+ if (relay.getBandwidth() >= 0L) {
+ double consensusWeight = (double) relay.getBandwidth();
+ consensusWeights.put(fingerprint, consensusWeight);
+ totalConsensusWeight += consensusWeight;
+ if (containsBandwidthWeights) {
+ double guardWeight = (double) relay.getBandwidth();
+ double middleWeight = (double) relay.getBandwidth();
+ double exitWeight = (double) relay.getBandwidth();
+ boolean isExit = relay.getFlags().contains("Exit") &&
+ !relay.getFlags().contains("BadExit");
+ boolean isGuard = relay.getFlags().contains("Guard");
+ if (isGuard && isExit) {
+ guardWeight *= wgd;
+ middleWeight *= wmd;
+ exitWeight *= wed;
+ } else if (isGuard) {
+ guardWeight *= wgg;
+ middleWeight *= wmg;
+ exitWeight = 0.0;
+ } else if (isExit) {
+ guardWeight = 0.0;
+ middleWeight *= wme;
+ exitWeight *= wee;
+ } else {
+ guardWeight = 0.0;
+ middleWeight *= wmm;
+ exitWeight = 0.0;
+ }
+ guardWeights.put(fingerprint, guardWeight);
+ middleWeights.put(fingerprint, middleWeight);
+ exitWeights.put(fingerprint, exitWeight);
+ totalGuardWeight += guardWeight;
+ totalMiddleWeight += middleWeight;
+ totalExitWeight += exitWeight;
+ }
+ }
+ }
+ SortedMap<String, double[]> pathSelectionProbabilities =
+ new TreeMap<String, double[]>();
+ SortedSet<String> fingerprints = new TreeSet<String>();
+ fingerprints.addAll(consensusWeights.keySet());
+ fingerprints.addAll(advertisedBandwidths.keySet());
+ for (String fingerprint : fingerprints) {
+ double[] probabilities = new double[] { -1.0, -1.0, -1.0, -1.0,
+ -1.0, -1.0, -1.0 };
+ if (consensusWeights.containsKey(fingerprint) &&
+ totalConsensusWeight > 0.0) {
+ probabilities[1] = consensusWeights.get(fingerprint) /
+ totalConsensusWeight;
+ probabilities[6] = consensusWeights.get(fingerprint);
+ }
+ if (guardWeights.containsKey(fingerprint) &&
+ totalGuardWeight > 0.0) {
+ probabilities[2] = guardWeights.get(fingerprint) /
+ totalGuardWeight;
+ }
+ if (middleWeights.containsKey(fingerprint) &&
+ totalMiddleWeight > 0.0) {
+ probabilities[3] = middleWeights.get(fingerprint) /
+ totalMiddleWeight;
+ }
+ if (exitWeights.containsKey(fingerprint) &&
+ totalExitWeight > 0.0) {
+ probabilities[4] = exitWeights.get(fingerprint) /
+ totalExitWeight;
+ }
+ if (advertisedBandwidths.containsKey(fingerprint) &&
+ totalAdvertisedBandwidth > 0.0) {
+ probabilities[0] = advertisedBandwidths.get(fingerprint)
+ / totalAdvertisedBandwidth;
+ probabilities[5] = advertisedBandwidths.get(fingerprint);
+ }
+ pathSelectionProbabilities.put(fingerprint, probabilities);
+ }
+ return pathSelectionProbabilities;
+ }
+
+ private void addToHistory(String fingerprint, long validAfterMillis,
+ long freshUntilMillis, double[] weights) {
+ WeightsStatus weightsStatus = this.documentStore.retrieve(
+ WeightsStatus.class, true, fingerprint);
+ if (weightsStatus == null) {
+ weightsStatus = new WeightsStatus();
+ }
+ SortedMap<long[], double[]> history = weightsStatus.getHistory();
+ long[] interval = new long[] { validAfterMillis, freshUntilMillis };
+ if ((history.headMap(interval).isEmpty() ||
+ history.headMap(interval).lastKey()[1] <= validAfterMillis) &&
+ (history.tailMap(interval).isEmpty() ||
+ history.tailMap(interval).firstKey()[0] >= freshUntilMillis)) {
+ history.put(interval, weights);
+ this.compressHistory(weightsStatus);
+ this.documentStore.store(weightsStatus, fingerprint);
+ }
+ }
+
+ private void compressHistory(WeightsStatus weightsStatus) {
+ SortedMap<long[], double[]> history = weightsStatus.getHistory();
+ SortedMap<long[], double[]> compressedHistory =
+ new TreeMap<long[], double[]>(history.comparator());
+ long lastStartMillis = 0L, lastEndMillis = 0L;
+ double[] lastWeights = null;
+ String lastMonthString = "1970-01";
+ int lastMissingValues = -1;
+ for (Map.Entry<long[], double[]> e : history.entrySet()) {
+ long startMillis = e.getKey()[0], endMillis = e.getKey()[1];
+ double[] weights = e.getValue();
+ long intervalLengthMillis;
+ if (this.now - endMillis <= DateTimeHelper.ONE_WEEK) {
+ intervalLengthMillis = DateTimeHelper.ONE_HOUR;
+ } else if (this.now - endMillis <=
+ DateTimeHelper.ROUGHLY_ONE_MONTH) {
+ intervalLengthMillis = DateTimeHelper.FOUR_HOURS;
+ } else if (this.now - endMillis <=
+ DateTimeHelper.ROUGHLY_THREE_MONTHS) {
+ intervalLengthMillis = DateTimeHelper.TWELVE_HOURS;
+ } else if (this.now - endMillis <=
+ DateTimeHelper.ROUGHLY_ONE_YEAR) {
+ intervalLengthMillis = DateTimeHelper.TWO_DAYS;
+ } else {
+ intervalLengthMillis = DateTimeHelper.TEN_DAYS;
+ }
+ String monthString = DateTimeHelper.format(startMillis,
+ DateTimeHelper.ISO_YEARMONTH_FORMAT);
+ int missingValues = 0;
+ for (int i = 0; i < weights.length; i++) {
+ if (weights[i] < -0.5) {
+ missingValues += 1 << i;
+ }
+ }
+ if (lastEndMillis == startMillis &&
+ ((lastEndMillis - 1L) / intervalLengthMillis) ==
+ ((endMillis - 1L) / intervalLengthMillis) &&
+ lastMonthString.equals(monthString) &&
+ lastMissingValues == missingValues) {
+ double lastIntervalInHours = (double) ((lastEndMillis
+ - lastStartMillis) / DateTimeHelper.ONE_HOUR);
+ double currentIntervalInHours = (double) ((endMillis
+ - startMillis) / DateTimeHelper.ONE_HOUR);
+ double newIntervalInHours = (double) ((endMillis
+ - lastStartMillis) / DateTimeHelper.ONE_HOUR);
+ for (int i = 0; i < lastWeights.length; i++) {
+ lastWeights[i] *= lastIntervalInHours;
+ lastWeights[i] += weights[i] * currentIntervalInHours;
+ lastWeights[i] /= newIntervalInHours;
+ }
+ lastEndMillis = endMillis;
+ } else {
+ if (lastStartMillis > 0L) {
+ compressedHistory.put(new long[] { lastStartMillis,
+ lastEndMillis }, lastWeights);
+ }
+ lastStartMillis = startMillis;
+ lastEndMillis = endMillis;
+ lastWeights = weights;
+ }
+ lastMonthString = monthString;
+ lastMissingValues = missingValues;
+ }
+ if (lastStartMillis > 0L) {
+ compressedHistory.put(new long[] { lastStartMillis, lastEndMillis },
+ lastWeights);
+ }
+ weightsStatus.setHistory(compressedHistory);
+ }
+
+ public String getStatsString() {
+ /* TODO Add statistics string. */
+ return null;
+ }
+}
+
diff --git a/src/org/torproject/onionoo/util/ApplicationFactory.java b/src/org/torproject/onionoo/util/ApplicationFactory.java
new file mode 100644
index 0000000..8eafca9
--- /dev/null
+++ b/src/org/torproject/onionoo/util/ApplicationFactory.java
@@ -0,0 +1,55 @@
+/* Copyright 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.util;
+
+import org.torproject.onionoo.docs.DocumentStore;
+import org.torproject.onionoo.server.NodeIndexer;
+import org.torproject.onionoo.updater.DescriptorSource;
+
+public class ApplicationFactory {
+
+ private static Time timeInstance;
+ public static void setTime(Time time) {
+ timeInstance = time;
+ }
+ public static Time getTime() {
+ if (timeInstance == null) {
+ timeInstance = new Time();
+ }
+ return timeInstance;
+ }
+
+ private static DescriptorSource descriptorSourceInstance;
+ public static void setDescriptorSource(
+ DescriptorSource descriptorSource) {
+ descriptorSourceInstance = descriptorSource;
+ }
+ public static DescriptorSource getDescriptorSource() {
+ if (descriptorSourceInstance == null) {
+ descriptorSourceInstance = new DescriptorSource();
+ }
+ return descriptorSourceInstance;
+ }
+
+ private static DocumentStore documentStoreInstance;
+ public static void setDocumentStore(DocumentStore documentStore) {
+ documentStoreInstance = documentStore;
+ }
+ public static DocumentStore getDocumentStore() {
+ if (documentStoreInstance == null) {
+ documentStoreInstance = new DocumentStore();
+ }
+ return documentStoreInstance;
+ }
+
+ private static NodeIndexer nodeIndexerInstance;
+ public static void setNodeIndexer(NodeIndexer nodeIndexer) {
+ nodeIndexerInstance = nodeIndexer;
+ }
+ public static NodeIndexer getNodeIndexer() {
+ if (nodeIndexerInstance == null) {
+ nodeIndexerInstance = new NodeIndexer();
+ }
+ return nodeIndexerInstance;
+ }
+}
diff --git a/src/org/torproject/onionoo/util/DateTimeHelper.java b/src/org/torproject/onionoo/util/DateTimeHelper.java
new file mode 100644
index 0000000..1fcf6e1
--- /dev/null
+++ b/src/org/torproject/onionoo/util/DateTimeHelper.java
@@ -0,0 +1,92 @@
+/* Copyright 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.util;
+
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TimeZone;
+
+public class DateTimeHelper {
+
+ private DateTimeHelper() {
+ }
+
+ public static final long ONE_SECOND = 1000L,
+ TEN_SECONDS = 10L * ONE_SECOND,
+ ONE_MINUTE = 60L * ONE_SECOND,
+ FIVE_MINUTES = 5L * ONE_MINUTE,
+ FIFTEEN_MINUTES = 15L * ONE_MINUTE,
+ ONE_HOUR = 60L * ONE_MINUTE,
+ FOUR_HOURS = 4L * ONE_HOUR,
+ SIX_HOURS = 6L * ONE_HOUR,
+ TWELVE_HOURS = 12L * ONE_HOUR,
+ ONE_DAY = 24L * ONE_HOUR,
+ TWO_DAYS = 2L * ONE_DAY,
+ THREE_DAYS = 3L * ONE_DAY,
+ ONE_WEEK = 7L * ONE_DAY,
+ TEN_DAYS = 10L * ONE_DAY,
+ ROUGHLY_ONE_MONTH = 31L * ONE_DAY,
+ ROUGHLY_THREE_MONTHS = 92L * ONE_DAY,
+ ROUGHLY_ONE_YEAR = 366L * ONE_DAY,
+ ROUGHLY_FIVE_YEARS = 5L * ROUGHLY_ONE_YEAR;
+
+ public static final String ISO_DATETIME_FORMAT = "yyyy-MM-dd HH:mm:ss";
+
+ public static final String ISO_DATETIME_TAB_FORMAT =
+ "yyyy-MM-dd\tHH:mm:ss";
+
+ public static final String ISO_YEARMONTH_FORMAT = "yyyy-MM";
+
+ public static final String DATEHOUR_NOSPACE_FORMAT = "yyyy-MM-dd-HH";
+
+ private static ThreadLocal<Map<String, DateFormat>> dateFormats =
+ new ThreadLocal<Map<String, DateFormat>> () {
+ public Map<String, DateFormat> get() {
+ return super.get();
+ }
+ protected Map<String, DateFormat> initialValue() {
+ return new HashMap<String, DateFormat>();
+ }
+ public void remove() {
+ super.remove();
+ }
+ public void set(Map<String, DateFormat> value) {
+ super.set(value);
+ }
+ };
+
+ private static DateFormat getDateFormat(String format) {
+ Map<String, DateFormat> threadDateFormats = dateFormats.get();
+ if (!threadDateFormats.containsKey(format)) {
+ DateFormat dateFormat = new SimpleDateFormat(format);
+ dateFormat.setLenient(false);
+ dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ threadDateFormats.put(format, dateFormat);
+ }
+ return threadDateFormats.get(format);
+ }
+
+ public static String format(long millis, String format) {
+ return getDateFormat(format).format(millis);
+ }
+
+ public static String format(long millis) {
+ return format(millis, ISO_DATETIME_FORMAT);
+ }
+
+ public static long parse(String string, String format) {
+ try {
+ return getDateFormat(format).parse(string).getTime();
+ } catch (ParseException e) {
+ return -1L;
+ }
+ }
+
+ public static long parse(String string) {
+ return parse(string, ISO_DATETIME_FORMAT);
+ }
+}
+
diff --git a/src/org/torproject/onionoo/util/LockFile.java b/src/org/torproject/onionoo/util/LockFile.java
new file mode 100644
index 0000000..01c4dcb
--- /dev/null
+++ b/src/org/torproject/onionoo/util/LockFile.java
@@ -0,0 +1,43 @@
+/* Copyright 2013 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.util;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+
+public class LockFile {
+
+ private final File lockFile = new File("lock");
+
+ public boolean acquireLock() {
+ Time time = ApplicationFactory.getTime();
+ try {
+ if (this.lockFile.exists()) {
+ return false;
+ }
+ if (this.lockFile.getParentFile() != null) {
+ this.lockFile.getParentFile().mkdirs();
+ }
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.lockFile));
+ bw.append("" + time.currentTimeMillis() + "\n");
+ bw.close();
+ return true;
+ } catch (IOException e) {
+ System.err.println("Caught exception while trying to acquire "
+ + "lock!");
+ e.printStackTrace();
+ return false;
+ }
+ }
+
+ public boolean releaseLock() {
+ if (this.lockFile.exists()) {
+ this.lockFile.delete();
+ }
+ return !this.lockFile.exists();
+ }
+}
+
diff --git a/src/org/torproject/onionoo/util/Logger.java b/src/org/torproject/onionoo/util/Logger.java
new file mode 100644
index 0000000..443c1ca
--- /dev/null
+++ b/src/org/torproject/onionoo/util/Logger.java
@@ -0,0 +1,81 @@
+/* Copyright 2013 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.util;
+
+import java.util.Date;
+
+public class Logger {
+
+ private Logger() {
+ }
+
+ private static Time time;
+
+ public static void setTime() {
+ time = ApplicationFactory.getTime();
+ }
+
+ private static long currentTimeMillis() {
+ if (time == null) {
+ return System.currentTimeMillis();
+ } else {
+ return time.currentTimeMillis();
+ }
+ }
+
+ public static String formatDecimalNumber(long decimalNumber) {
+ return String.format("%,d", decimalNumber);
+ }
+
+ public static String formatMillis(long millis) {
+ return String.format("%02d:%02d.%03d minutes",
+ millis / DateTimeHelper.ONE_MINUTE,
+ (millis % DateTimeHelper.ONE_MINUTE) / DateTimeHelper.ONE_SECOND,
+ millis % DateTimeHelper.ONE_SECOND);
+ }
+
+ public static String formatBytes(long bytes) {
+ if (bytes < 1024) {
+ return bytes + " B";
+ } else {
+ int exp = (int) (Math.log(bytes) / Math.log(1024));
+ return String.format("%.1f %siB", bytes / Math.pow(1024, exp),
+ "KMGTPE".charAt(exp-1));
+ }
+ }
+
+ private static long printedLastStatusMessage = -1L;
+
+ public static void printStatus(String message) {
+ System.out.println(new Date() + ": " + message);
+ printedLastStatusMessage = currentTimeMillis();
+ }
+
+ public static void printStatistics(String component, String message) {
+ System.out.print(" " + component + " statistics:\n" + message);
+ }
+
+ public static void printStatusTime(String message) {
+ printStatusOrErrorTime(message, false);
+ }
+
+ public static void printErrorTime(String message) {
+ printStatusOrErrorTime(message, true);
+ }
+
+ private static void printStatusOrErrorTime(String message,
+ boolean printToSystemErr) {
+ long now = currentTimeMillis();
+ long millis = printedLastStatusMessage < 0 ? 0 :
+ now - printedLastStatusMessage;
+ String line = " " + message + " (" + Logger.formatMillis(millis)
+ + ").";
+ if (printToSystemErr) {
+ System.err.println(line);
+ } else {
+ System.out.println(line);
+ }
+ printedLastStatusMessage = now;
+ }
+}
+
diff --git a/src/org/torproject/onionoo/util/Time.java b/src/org/torproject/onionoo/util/Time.java
new file mode 100644
index 0000000..126a910
--- /dev/null
+++ b/src/org/torproject/onionoo/util/Time.java
@@ -0,0 +1,14 @@
+/* Copyright 2013 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.util;
+
+/*
+ * Wrapper for System.currentTimeMillis() that can be replaced with a
+ * custom time source for testing.
+ */
+public class Time {
+ public long currentTimeMillis() {
+ return System.currentTimeMillis();
+ }
+}
+
diff --git a/src/org/torproject/onionoo/writer/BandwidthDocumentWriter.java b/src/org/torproject/onionoo/writer/BandwidthDocumentWriter.java
new file mode 100644
index 0000000..908ec7c
--- /dev/null
+++ b/src/org/torproject/onionoo/writer/BandwidthDocumentWriter.java
@@ -0,0 +1,201 @@
+/* Copyright 2011--2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.writer;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+
+import org.torproject.onionoo.docs.BandwidthDocument;
+import org.torproject.onionoo.docs.BandwidthStatus;
+import org.torproject.onionoo.docs.DocumentStore;
+import org.torproject.onionoo.docs.GraphHistory;
+import org.torproject.onionoo.updater.DescriptorSource;
+import org.torproject.onionoo.updater.DescriptorType;
+import org.torproject.onionoo.updater.FingerprintListener;
+import org.torproject.onionoo.util.ApplicationFactory;
+import org.torproject.onionoo.util.DateTimeHelper;
+import org.torproject.onionoo.util.Logger;
+
+public class BandwidthDocumentWriter implements FingerprintListener,
+ DocumentWriter{
+
+ private DescriptorSource descriptorSource;
+
+ private DocumentStore documentStore;
+
+ private long now;
+
+ public BandwidthDocumentWriter() {
+ this.descriptorSource = ApplicationFactory.getDescriptorSource();
+ this.documentStore = ApplicationFactory.getDocumentStore();
+ this.now = ApplicationFactory.getTime().currentTimeMillis();
+ this.registerFingerprintListeners();
+ }
+
+ private void registerFingerprintListeners() {
+ this.descriptorSource.registerFingerprintListener(this,
+ DescriptorType.RELAY_EXTRA_INFOS);
+ this.descriptorSource.registerFingerprintListener(this,
+ DescriptorType.BRIDGE_EXTRA_INFOS);
+ }
+
+ private Set<String> updateBandwidthDocuments = new HashSet<String>();
+
+ public void processFingerprints(SortedSet<String> fingerprints,
+ boolean relay) {
+ this.updateBandwidthDocuments.addAll(fingerprints);
+ }
+
+ public void writeDocuments() {
+ for (String fingerprint : this.updateBandwidthDocuments) {
+ BandwidthStatus bandwidthStatus = this.documentStore.retrieve(
+ BandwidthStatus.class, true, fingerprint);
+ if (bandwidthStatus == null) {
+ continue;
+ }
+ BandwidthDocument bandwidthDocument = this.compileBandwidthDocument(
+ fingerprint, bandwidthStatus);
+ this.documentStore.store(bandwidthDocument, fingerprint);
+ }
+ Logger.printStatusTime("Wrote bandwidth document files");
+ }
+
+
+ private BandwidthDocument compileBandwidthDocument(String fingerprint,
+ BandwidthStatus bandwidthStatus) {
+ BandwidthDocument bandwidthDocument = new BandwidthDocument();
+ bandwidthDocument.setFingerprint(fingerprint);
+ bandwidthDocument.setWriteHistory(this.compileGraphType(
+ bandwidthStatus.getWriteHistory()));
+ bandwidthDocument.setReadHistory(this.compileGraphType(
+ bandwidthStatus.getReadHistory()));
+ return bandwidthDocument;
+ }
+
+ private String[] graphNames = new String[] {
+ "3_days",
+ "1_week",
+ "1_month",
+ "3_months",
+ "1_year",
+ "5_years" };
+
+ private long[] graphIntervals = new long[] {
+ DateTimeHelper.THREE_DAYS,
+ DateTimeHelper.ONE_WEEK,
+ DateTimeHelper.ROUGHLY_ONE_MONTH,
+ DateTimeHelper.ROUGHLY_THREE_MONTHS,
+ DateTimeHelper.ROUGHLY_ONE_YEAR,
+ DateTimeHelper.ROUGHLY_FIVE_YEARS };
+
+ private long[] dataPointIntervals = new long[] {
+ DateTimeHelper.FIFTEEN_MINUTES,
+ DateTimeHelper.ONE_HOUR,
+ DateTimeHelper.FOUR_HOURS,
+ DateTimeHelper.TWELVE_HOURS,
+ DateTimeHelper.TWO_DAYS,
+ DateTimeHelper.TEN_DAYS };
+
+ private Map<String, GraphHistory> compileGraphType(
+ SortedMap<Long, long[]> history) {
+ Map<String, GraphHistory> graphs =
+ new LinkedHashMap<String, GraphHistory>();
+ for (int i = 0; i < this.graphIntervals.length; i++) {
+ String graphName = this.graphNames[i];
+ long graphInterval = this.graphIntervals[i];
+ long dataPointInterval = this.dataPointIntervals[i];
+ List<Long> dataPoints = new ArrayList<Long>();
+ long intervalStartMillis = ((this.now - graphInterval)
+ / dataPointInterval) * dataPointInterval;
+ long totalMillis = 0L, totalBandwidth = 0L;
+ for (long[] v : history.values()) {
+ long startMillis = v[0], endMillis = v[1], bandwidth = v[2];
+ if (endMillis < intervalStartMillis) {
+ continue;
+ }
+ while ((intervalStartMillis / dataPointInterval) !=
+ (endMillis / dataPointInterval)) {
+ dataPoints.add(totalMillis * 5L < dataPointInterval
+ ? -1L : (totalBandwidth * DateTimeHelper.ONE_SECOND)
+ / totalMillis);
+ totalBandwidth = 0L;
+ totalMillis = 0L;
+ intervalStartMillis += dataPointInterval;
+ }
+ totalBandwidth += bandwidth;
+ totalMillis += (endMillis - startMillis);
+ }
+ dataPoints.add(totalMillis * 5L < dataPointInterval
+ ? -1L : (totalBandwidth * DateTimeHelper.ONE_SECOND)
+ / totalMillis);
+ long maxValue = 1L;
+ int firstNonNullIndex = -1, lastNonNullIndex = -1;
+ for (int j = 0; j < dataPoints.size(); j++) {
+ long dataPoint = dataPoints.get(j);
+ if (dataPoint >= 0L) {
+ if (firstNonNullIndex < 0) {
+ firstNonNullIndex = j;
+ }
+ lastNonNullIndex = j;
+ if (dataPoint > maxValue) {
+ maxValue = dataPoint;
+ }
+ }
+ }
+ if (firstNonNullIndex < 0) {
+ continue;
+ }
+ long firstDataPointMillis = (((this.now - graphInterval)
+ / dataPointInterval) + firstNonNullIndex) * dataPointInterval
+ + dataPointInterval / 2L;
+ if (i > 0 &&
+ firstDataPointMillis >= this.now - graphIntervals[i - 1]) {
+ /* Skip bandwidth history object, because it doesn't contain
+ * anything new that wasn't already contained in the last
+ * bandwidth history object(s). */
+ continue;
+ }
+ long lastDataPointMillis = firstDataPointMillis
+ + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval;
+ double factor = ((double) maxValue) / 999.0;
+ int count = lastNonNullIndex - firstNonNullIndex + 1;
+ GraphHistory graphHistory = new GraphHistory();
+ graphHistory.setFirst(DateTimeHelper.format(firstDataPointMillis));
+ graphHistory.setLast(DateTimeHelper.format(lastDataPointMillis));
+ graphHistory.setInterval((int) (dataPointInterval
+ / DateTimeHelper.ONE_SECOND));
+ graphHistory.setFactor(factor);
+ graphHistory.setCount(count);
+ int previousNonNullIndex = -2;
+ boolean foundTwoAdjacentDataPoints = false;
+ List<Integer> values = new ArrayList<Integer>();
+ for (int j = firstNonNullIndex; j <= lastNonNullIndex; j++) {
+ long dataPoint = dataPoints.get(j);
+ if (dataPoint >= 0L) {
+ if (j - previousNonNullIndex == 1) {
+ foundTwoAdjacentDataPoints = true;
+ }
+ previousNonNullIndex = j;
+ }
+ values.add(dataPoint < 0L ? null :
+ (int) ((dataPoint * 999L) / maxValue));
+ }
+ graphHistory.setValues(values);
+ if (foundTwoAdjacentDataPoints) {
+ graphs.put(graphName, graphHistory);
+ }
+ }
+ return graphs;
+ }
+
+ public String getStatsString() {
+ /* TODO Add statistics string. */
+ return null;
+ }
+}
diff --git a/src/org/torproject/onionoo/writer/ClientsDocumentWriter.java b/src/org/torproject/onionoo/writer/ClientsDocumentWriter.java
new file mode 100644
index 0000000..976804c
--- /dev/null
+++ b/src/org/torproject/onionoo/writer/ClientsDocumentWriter.java
@@ -0,0 +1,296 @@
+/* Copyright 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.writer;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.torproject.onionoo.docs.ClientsDocument;
+import org.torproject.onionoo.docs.ClientsGraphHistory;
+import org.torproject.onionoo.docs.ClientsHistory;
+import org.torproject.onionoo.docs.ClientsStatus;
+import org.torproject.onionoo.docs.DocumentStore;
+import org.torproject.onionoo.updater.DescriptorSource;
+import org.torproject.onionoo.updater.DescriptorType;
+import org.torproject.onionoo.updater.FingerprintListener;
+import org.torproject.onionoo.util.ApplicationFactory;
+import org.torproject.onionoo.util.DateTimeHelper;
+import org.torproject.onionoo.util.Logger;
+
+/*
+ * Clients status file produced as intermediate output:
+ *
+ * 2014-02-15 16:42:11 2014-02-16 00:00:00
+ * 259.042 in=86.347,se=86.347 v4=259.042
+ * 2014-02-16 00:00:00 2014-02-16 16:42:11
+ * 592.958 in=197.653,se=197.653 v4=592.958
+ *
+ * Clients document file produced as output:
+ *
+ * "1_month":{
+ * "first":"2014-02-03 12:00:00",
+ * "last":"2014-02-28 12:00:00",
+ * "interval":86400,
+ * "factor":0.139049349,
+ * "count":26,
+ * "values":[371,354,349,374,432,null,485,458,493,536,null,null,524,576,
+ * 607,622,null,635,null,566,774,999,945,690,656,681],
+ * "countries":{"cn":0.0192,"in":0.1768,"ir":0.2487,"ru":0.0104,
+ * "se":0.1698,"sy":0.0325,"us":0.0406},
+ * "transports":{"obfs2":0.4581},
+ * "versions":{"v4":1.0000}}
+ */
+public class ClientsDocumentWriter implements FingerprintListener,
+ DocumentWriter {
+
+ private DescriptorSource descriptorSource;
+
+ private DocumentStore documentStore;
+
+ private long now;
+
+ public ClientsDocumentWriter() {
+ this.descriptorSource = ApplicationFactory.getDescriptorSource();
+ this.documentStore = ApplicationFactory.getDocumentStore();
+ this.now = ApplicationFactory.getTime().currentTimeMillis();
+ this.registerFingerprintListeners();
+ }
+
+ private void registerFingerprintListeners() {
+ this.descriptorSource.registerFingerprintListener(this,
+ DescriptorType.BRIDGE_EXTRA_INFOS);
+ }
+
+ private SortedSet<String> updateDocuments = new TreeSet<String>();
+
+ public void processFingerprints(SortedSet<String> fingerprints,
+ boolean relay) {
+ if (!relay) {
+ this.updateDocuments.addAll(fingerprints);
+ }
+ }
+
+ private int writtenDocuments = 0;
+
+ public void writeDocuments() {
+ for (String hashedFingerprint : this.updateDocuments) {
+ ClientsStatus clientsStatus = this.documentStore.retrieve(
+ ClientsStatus.class, true, hashedFingerprint);
+ if (clientsStatus == null) {
+ continue;
+ }
+ SortedSet<ClientsHistory> history = clientsStatus.getHistory();
+ ClientsDocument clientsDocument = this.compileClientsDocument(
+ hashedFingerprint, history);
+ this.documentStore.store(clientsDocument, hashedFingerprint);
+ this.writtenDocuments++;
+ }
+ Logger.printStatusTime("Wrote clients document files");
+ }
+
+ private String[] graphNames = new String[] {
+ "1_week",
+ "1_month",
+ "3_months",
+ "1_year",
+ "5_years" };
+
+ private long[] graphIntervals = new long[] {
+ DateTimeHelper.ONE_WEEK,
+ DateTimeHelper.ROUGHLY_ONE_MONTH,
+ DateTimeHelper.ROUGHLY_THREE_MONTHS,
+ DateTimeHelper.ROUGHLY_ONE_YEAR,
+ DateTimeHelper.ROUGHLY_FIVE_YEARS };
+
+ private long[] dataPointIntervals = new long[] {
+ DateTimeHelper.ONE_DAY,
+ DateTimeHelper.ONE_DAY,
+ DateTimeHelper.ONE_DAY,
+ DateTimeHelper.TWO_DAYS,
+ DateTimeHelper.TEN_DAYS };
+
+ private ClientsDocument compileClientsDocument(String hashedFingerprint,
+ SortedSet<ClientsHistory> history) {
+ ClientsDocument clientsDocument = new ClientsDocument();
+ clientsDocument.setFingerprint(hashedFingerprint);
+ Map<String, ClientsGraphHistory> averageClients =
+ new LinkedHashMap<String, ClientsGraphHistory>();
+ for (int graphIntervalIndex = 0; graphIntervalIndex <
+ this.graphIntervals.length; graphIntervalIndex++) {
+ String graphName = this.graphNames[graphIntervalIndex];
+ ClientsGraphHistory graphHistory = this.compileClientsHistory(
+ graphIntervalIndex, history);
+ if (graphHistory != null) {
+ averageClients.put(graphName, graphHistory);
+ }
+ }
+ clientsDocument.setAverageClients(averageClients);
+ return clientsDocument;
+ }
+
+ private ClientsGraphHistory compileClientsHistory(
+ int graphIntervalIndex, SortedSet<ClientsHistory> history) {
+ long graphInterval = this.graphIntervals[graphIntervalIndex];
+ long dataPointInterval =
+ this.dataPointIntervals[graphIntervalIndex];
+ List<Double> dataPoints = new ArrayList<Double>();
+ long intervalStartMillis = ((this.now - graphInterval)
+ / dataPointInterval) * dataPointInterval;
+ long millis = 0L;
+ double responses = 0.0, totalResponses = 0.0;
+ SortedMap<String, Double>
+ totalResponsesByCountry = new TreeMap<String, Double>(),
+ totalResponsesByTransport = new TreeMap<String, Double>(),
+ totalResponsesByVersion = new TreeMap<String, Double>();
+ for (ClientsHistory hist : history) {
+ if (hist.getEndMillis() < intervalStartMillis) {
+ continue;
+ }
+ while ((intervalStartMillis / dataPointInterval) !=
+ (hist.getEndMillis() / dataPointInterval)) {
+ dataPoints.add(millis * 2L < dataPointInterval
+ ? -1.0 : responses * ((double) DateTimeHelper.ONE_DAY)
+ / (((double) millis) * 10.0));
+ responses = 0.0;
+ millis = 0L;
+ intervalStartMillis += dataPointInterval;
+ }
+ responses += hist.getTotalResponses();
+ totalResponses += hist.getTotalResponses();
+ for (Map.Entry<String, Double> e :
+ hist.getResponsesByCountry().entrySet()) {
+ if (!totalResponsesByCountry.containsKey(e.getKey())) {
+ totalResponsesByCountry.put(e.getKey(), 0.0);
+ }
+ totalResponsesByCountry.put(e.getKey(), e.getValue()
+ + totalResponsesByCountry.get(e.getKey()));
+ }
+ for (Map.Entry<String, Double> e :
+ hist.getResponsesByTransport().entrySet()) {
+ if (!totalResponsesByTransport.containsKey(e.getKey())) {
+ totalResponsesByTransport.put(e.getKey(), 0.0);
+ }
+ totalResponsesByTransport.put(e.getKey(), e.getValue()
+ + totalResponsesByTransport.get(e.getKey()));
+ }
+ for (Map.Entry<String, Double> e :
+ hist.getResponsesByVersion().entrySet()) {
+ if (!totalResponsesByVersion.containsKey(e.getKey())) {
+ totalResponsesByVersion.put(e.getKey(), 0.0);
+ }
+ totalResponsesByVersion.put(e.getKey(), e.getValue()
+ + totalResponsesByVersion.get(e.getKey()));
+ }
+ millis += (hist.getEndMillis() - hist.getStartMillis());
+ }
+ dataPoints.add(millis * 2L < dataPointInterval
+ ? -1.0 : responses * ((double) DateTimeHelper.ONE_DAY)
+ / (((double) millis) * 10.0));
+ double maxValue = 0.0;
+ int firstNonNullIndex = -1, lastNonNullIndex = -1;
+ for (int dataPointIndex = 0; dataPointIndex < dataPoints.size();
+ dataPointIndex++) {
+ double dataPoint = dataPoints.get(dataPointIndex);
+ if (dataPoint >= 0.0) {
+ if (firstNonNullIndex < 0) {
+ firstNonNullIndex = dataPointIndex;
+ }
+ lastNonNullIndex = dataPointIndex;
+ if (dataPoint > maxValue) {
+ maxValue = dataPoint;
+ }
+ }
+ }
+ if (firstNonNullIndex < 0) {
+ return null;
+ }
+ long firstDataPointMillis = (((this.now - graphInterval)
+ / dataPointInterval) + firstNonNullIndex) * dataPointInterval
+ + dataPointInterval / 2L;
+ if (graphIntervalIndex > 0 && firstDataPointMillis >=
+ this.now - graphIntervals[graphIntervalIndex - 1]) {
+ /* Skip clients history object, because it doesn't contain
+ * anything new that wasn't already contained in the last
+ * clients history object(s). */
+ return null;
+ }
+ long lastDataPointMillis = firstDataPointMillis
+ + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval;
+ double factor = ((double) maxValue) / 999.0;
+ int count = lastNonNullIndex - firstNonNullIndex + 1;
+ ClientsGraphHistory graphHistory = new ClientsGraphHistory();
+ graphHistory.setFirst(DateTimeHelper.format(firstDataPointMillis));
+ graphHistory.setLast(DateTimeHelper.format(lastDataPointMillis));
+ graphHistory.setInterval((int) (dataPointInterval
+ / DateTimeHelper.ONE_SECOND));
+ graphHistory.setFactor(factor);
+ graphHistory.setCount(count);
+ int previousNonNullIndex = -2;
+ boolean foundTwoAdjacentDataPoints = false;
+ List<Integer> values = new ArrayList<Integer>();
+ for (int dataPointIndex = firstNonNullIndex; dataPointIndex <=
+ lastNonNullIndex; dataPointIndex++) {
+ double dataPoint = dataPoints.get(dataPointIndex);
+ if (dataPoint >= 0.0) {
+ if (dataPointIndex - previousNonNullIndex == 1) {
+ foundTwoAdjacentDataPoints = true;
+ }
+ previousNonNullIndex = dataPointIndex;
+ }
+ values.add(dataPoint < 0.0 ? null :
+ (int) ((dataPoint * 999.0) / maxValue));
+ }
+ graphHistory.setValues(values);
+ if (!totalResponsesByCountry.isEmpty()) {
+ SortedMap<String, Float> countries = new TreeMap<String, Float>();
+ for (Map.Entry<String, Double> e :
+ totalResponsesByCountry.entrySet()) {
+ if (e.getValue() > totalResponses / 100.0) {
+ countries.put(e.getKey(),
+ (float) (e.getValue() / totalResponses));
+ }
+ }
+ graphHistory.setCountries(countries);
+ }
+ if (!totalResponsesByTransport.isEmpty()) {
+ SortedMap<String, Float> transports = new TreeMap<String, Float>();
+ for (Map.Entry<String, Double> e :
+ totalResponsesByTransport.entrySet()) {
+ if (e.getValue() > totalResponses / 100.0) {
+ transports.put(e.getKey(),
+ (float) (e.getValue() / totalResponses));
+ }
+ }
+ graphHistory.setTransports(transports);
+ }
+ if (!totalResponsesByVersion.isEmpty()) {
+ SortedMap<String, Float> versions = new TreeMap<String, Float>();
+ for (Map.Entry<String, Double> e :
+ totalResponsesByVersion.entrySet()) {
+ if (e.getValue() > totalResponses / 100.0) {
+ versions.put(e.getKey(),
+ (float) (e.getValue() / totalResponses));
+ }
+ }
+ graphHistory.setVersions(versions);
+ }
+ if (foundTwoAdjacentDataPoints) {
+ return graphHistory;
+ } else {
+ return null;
+ }
+ }
+
+ public String getStatsString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(" " + Logger.formatDecimalNumber(this.writtenDocuments)
+ + " clients document files updated\n");
+ return sb.toString();
+ }
+}
diff --git a/src/org/torproject/onionoo/writer/DetailsDocumentWriter.java b/src/org/torproject/onionoo/writer/DetailsDocumentWriter.java
new file mode 100644
index 0000000..03f7024
--- /dev/null
+++ b/src/org/torproject/onionoo/writer/DetailsDocumentWriter.java
@@ -0,0 +1,233 @@
+package org.torproject.onionoo.writer;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.torproject.onionoo.docs.DetailsDocument;
+import org.torproject.onionoo.docs.DetailsStatus;
+import org.torproject.onionoo.docs.DocumentStore;
+import org.torproject.onionoo.docs.NodeStatus;
+import org.torproject.onionoo.updater.DescriptorSource;
+import org.torproject.onionoo.updater.DescriptorType;
+import org.torproject.onionoo.updater.FingerprintListener;
+import org.torproject.onionoo.util.ApplicationFactory;
+import org.torproject.onionoo.util.DateTimeHelper;
+import org.torproject.onionoo.util.Logger;
+
+public class DetailsDocumentWriter implements FingerprintListener,
+ DocumentWriter {
+
+ private DescriptorSource descriptorSource;
+
+ private DocumentStore documentStore;
+
+ private long now;
+
+ public DetailsDocumentWriter() {
+ this.descriptorSource = ApplicationFactory.getDescriptorSource();
+ this.documentStore = ApplicationFactory.getDocumentStore();
+ this.now = ApplicationFactory.getTime().currentTimeMillis();
+ this.registerFingerprintListeners();
+ }
+
+ private void registerFingerprintListeners() {
+ this.descriptorSource.registerFingerprintListener(this,
+ DescriptorType.RELAY_CONSENSUSES);
+ this.descriptorSource.registerFingerprintListener(this,
+ DescriptorType.RELAY_SERVER_DESCRIPTORS);
+ this.descriptorSource.registerFingerprintListener(this,
+ DescriptorType.BRIDGE_STATUSES);
+ this.descriptorSource.registerFingerprintListener(this,
+ DescriptorType.BRIDGE_SERVER_DESCRIPTORS);
+ this.descriptorSource.registerFingerprintListener(this,
+ DescriptorType.BRIDGE_POOL_ASSIGNMENTS);
+ this.descriptorSource.registerFingerprintListener(this,
+ DescriptorType.EXIT_LISTS);
+ }
+
+ private SortedSet<String> newRelays = new TreeSet<String>(),
+ newBridges = new TreeSet<String>();
+
+ public void processFingerprints(SortedSet<String> fingerprints,
+ boolean relay) {
+ if (relay) {
+ this.newRelays.addAll(fingerprints);
+ } else {
+ this.newBridges.addAll(fingerprints);
+ }
+ }
+
+ public void writeDocuments() {
+ this.updateRelayDetailsFiles();
+ this.updateBridgeDetailsFiles();
+ Logger.printStatusTime("Wrote details document files");
+ }
+
+ private void updateRelayDetailsFiles() {
+ for (String fingerprint : this.newRelays) {
+
+ /* Generate network-status-specific part. */
+ NodeStatus entry = this.documentStore.retrieve(NodeStatus.class,
+ true, fingerprint);
+ if (entry == null) {
+ continue;
+ }
+ DetailsDocument detailsDocument = new DetailsDocument();
+ detailsDocument.setNickname(entry.getNickname());
+ detailsDocument.setFingerprint(fingerprint);
+ List<String> orAddresses = new ArrayList<String>();
+ orAddresses.add(entry.getAddress() + ":" + entry.getOrPort());
+ for (String orAddress : entry.getOrAddressesAndPorts()) {
+ orAddresses.add(orAddress.toLowerCase());
+ }
+ detailsDocument.setOrAddresses(orAddresses);
+ if (entry.getDirPort() != 0) {
+ detailsDocument.setDirAddress(entry.getAddress() + ":"
+ + entry.getDirPort());
+ }
+ detailsDocument.setLastSeen(DateTimeHelper.format(
+ entry.getLastSeenMillis()));
+ detailsDocument.setFirstSeen(DateTimeHelper.format(
+ entry.getFirstSeenMillis()));
+ detailsDocument.setLastChangedAddressOrPort(
+ DateTimeHelper.format(entry.getLastChangedOrAddress()));
+ detailsDocument.setRunning(entry.getRunning());
+ if (!entry.getRelayFlags().isEmpty()) {
+ detailsDocument.setFlags(new ArrayList<String>(
+ entry.getRelayFlags()));
+ }
+ detailsDocument.setCountry(entry.getCountryCode());
+ detailsDocument.setLatitude(entry.getLatitude());
+ detailsDocument.setLongitude(entry.getLongitude());
+ detailsDocument.setCountryName(entry.getCountryName());
+ detailsDocument.setRegionName(entry.getRegionName());
+ detailsDocument.setCityName(entry.getCityName());
+ detailsDocument.setAsNumber(entry.getASNumber());
+ detailsDocument.setAsName(entry.getASName());
+ detailsDocument.setConsensusWeight(entry.getConsensusWeight());
+ detailsDocument.setHostName(entry.getHostName());
+ detailsDocument.setAdvertisedBandwidthFraction(
+ (float) entry.getAdvertisedBandwidthFraction());
+ detailsDocument.setConsensusWeightFraction(
+ (float) entry.getConsensusWeightFraction());
+ detailsDocument.setGuardProbability(
+ (float) entry.getGuardProbability());
+ detailsDocument.setMiddleProbability(
+ (float) entry.getMiddleProbability());
+ detailsDocument.setExitProbability(
+ (float) entry.getExitProbability());
+ String defaultPolicy = entry.getDefaultPolicy();
+ String portList = entry.getPortList();
+ if (defaultPolicy != null && (defaultPolicy.equals("accept") ||
+ defaultPolicy.equals("reject")) && portList != null) {
+ Map<String, List<String>> exitPolicySummary =
+ new HashMap<String, List<String>>();
+ List<String> portsOrPortRanges = Arrays.asList(
+ portList.split(","));
+ exitPolicySummary.put(defaultPolicy, portsOrPortRanges);
+ detailsDocument.setExitPolicySummary(exitPolicySummary);
+ }
+ detailsDocument.setRecommendedVersion(
+ entry.getRecommendedVersion());
+
+ /* Append descriptor-specific part and exit addresses from details
+ * status file. */
+ DetailsStatus detailsStatus = this.documentStore.retrieve(
+ DetailsStatus.class, true, fingerprint);
+ if (detailsStatus != null) {
+ detailsDocument.setLastRestarted(
+ detailsStatus.getLastRestarted());
+ detailsDocument.setBandwidthRate(
+ detailsStatus.getBandwidthRate());
+ detailsDocument.setBandwidthBurst(
+ detailsStatus.getBandwidthBurst());
+ detailsDocument.setObservedBandwidth(
+ detailsStatus.getObservedBandwidth());
+ detailsDocument.setAdvertisedBandwidth(
+ detailsStatus.getAdvertisedBandwidth());
+ detailsDocument.setExitPolicy(detailsStatus.getExitPolicy());
+ detailsDocument.setContact(detailsStatus.getContact());
+ detailsDocument.setPlatform(detailsStatus.getPlatform());
+ detailsDocument.setFamily(detailsStatus.getFamily());
+ detailsDocument.setExitPolicyV6Summary(
+ detailsStatus.getExitPolicyV6Summary());
+ detailsDocument.setHibernating(detailsStatus.getHibernating());
+ if (detailsStatus.getExitAddresses() != null) {
+ SortedSet<String> exitAddresses = new TreeSet<String>();
+ for (Map.Entry<String, Long> e :
+ detailsStatus.getExitAddresses().entrySet()) {
+ String exitAddress = e.getKey().toLowerCase();
+ long scanMillis = e.getValue();
+ if (!entry.getAddress().equals(exitAddress) &&
+ !entry.getOrAddresses().contains(exitAddress) &&
+ scanMillis >= this.now - DateTimeHelper.ONE_DAY) {
+ exitAddresses.add(exitAddress);
+ }
+ }
+ if (!exitAddresses.isEmpty()) {
+ detailsDocument.setExitAddresses(new ArrayList<String>(
+ exitAddresses));
+ }
+ }
+ }
+
+ /* Write details file to disk. */
+ this.documentStore.store(detailsDocument, fingerprint);
+ }
+ }
+
+ private void updateBridgeDetailsFiles() {
+ for (String fingerprint : this.newBridges) {
+
+ /* Generate network-status-specific part. */
+ NodeStatus entry = this.documentStore.retrieve(NodeStatus.class,
+ true, fingerprint);
+ if (entry == null) {
+ continue;
+ }
+ DetailsDocument detailsDocument = new DetailsDocument();
+ detailsDocument.setNickname(entry.getNickname());
+ detailsDocument.setHashedFingerprint(fingerprint);
+ String address = entry.getAddress();
+ List<String> orAddresses = new ArrayList<String>();
+ orAddresses.add(address + ":" + entry.getOrPort());
+ for (String orAddress : entry.getOrAddressesAndPorts()) {
+ orAddresses.add(orAddress.toLowerCase());
+ }
+ detailsDocument.setOrAddresses(orAddresses);
+ detailsDocument.setLastSeen(DateTimeHelper.format(
+ entry.getLastSeenMillis()));
+ detailsDocument.setFirstSeen(DateTimeHelper.format(
+ entry.getFirstSeenMillis()));
+ detailsDocument.setRunning(entry.getRunning());
+ detailsDocument.setFlags(new ArrayList<String>(
+ entry.getRelayFlags()));
+
+ /* Append descriptor-specific part from details status file. */
+ DetailsStatus detailsStatus = this.documentStore.retrieve(
+ DetailsStatus.class, true, fingerprint);
+ if (detailsStatus != null) {
+ detailsDocument.setLastRestarted(
+ detailsStatus.getLastRestarted());
+ detailsDocument.setAdvertisedBandwidth(
+ detailsStatus.getAdvertisedBandwidth());
+ detailsDocument.setPlatform(detailsStatus.getPlatform());
+ detailsDocument.setPoolAssignment(
+ detailsStatus.getPoolAssignment());
+ }
+
+ /* Write details file to disk. */
+ this.documentStore.store(detailsDocument, fingerprint);
+ }
+ }
+
+ public String getStatsString() {
+ /* TODO Add statistics string. */
+ return null;
+ }
+}
diff --git a/src/org/torproject/onionoo/writer/DocumentWriter.java b/src/org/torproject/onionoo/writer/DocumentWriter.java
new file mode 100644
index 0000000..c238170
--- /dev/null
+++ b/src/org/torproject/onionoo/writer/DocumentWriter.java
@@ -0,0 +1,11 @@
+/* Copyright 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.writer;
+
+public interface DocumentWriter {
+
+ public abstract void writeDocuments();
+
+ public abstract String getStatsString();
+}
+
diff --git a/src/org/torproject/onionoo/writer/SummaryDocumentWriter.java b/src/org/torproject/onionoo/writer/SummaryDocumentWriter.java
new file mode 100644
index 0000000..1b4630e
--- /dev/null
+++ b/src/org/torproject/onionoo/writer/SummaryDocumentWriter.java
@@ -0,0 +1,94 @@
+/* Copyright 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.writer;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.SortedSet;
+
+import org.torproject.onionoo.docs.DocumentStore;
+import org.torproject.onionoo.docs.NodeStatus;
+import org.torproject.onionoo.docs.SummaryDocument;
+import org.torproject.onionoo.util.ApplicationFactory;
+import org.torproject.onionoo.util.DateTimeHelper;
+import org.torproject.onionoo.util.Logger;
+
+public class SummaryDocumentWriter implements DocumentWriter {
+
+ private DocumentStore documentStore;
+
+ public SummaryDocumentWriter() {
+ this.documentStore = ApplicationFactory.getDocumentStore();
+ }
+
+ private int writtenDocuments = 0, deletedDocuments = 0;
+
+ public void writeDocuments() {
+ long maxLastSeenMillis = 0L;
+ for (String fingerprint : this.documentStore.list(NodeStatus.class)) {
+ NodeStatus nodeStatus = this.documentStore.retrieve(
+ NodeStatus.class, true, fingerprint);
+ if (nodeStatus != null &&
+ nodeStatus.getLastSeenMillis() > maxLastSeenMillis) {
+ maxLastSeenMillis = nodeStatus.getLastSeenMillis();
+ }
+ }
+ long cutoff = maxLastSeenMillis - DateTimeHelper.ONE_WEEK;
+ for (String fingerprint : this.documentStore.list(NodeStatus.class)) {
+ NodeStatus nodeStatus = this.documentStore.retrieve(
+ NodeStatus.class,
+ true, fingerprint);
+ if (nodeStatus == null) {
+ continue;
+ }
+ if (nodeStatus.getLastSeenMillis() < cutoff) {
+ if (this.documentStore.remove(SummaryDocument.class,
+ fingerprint)) {
+ this.deletedDocuments++;
+ }
+ continue;
+ }
+ boolean isRelay = nodeStatus.isRelay();
+ String nickname = nodeStatus.getNickname();
+ List<String> addresses = new ArrayList<String>();
+ addresses.add(nodeStatus.getAddress());
+ for (String orAddress : nodeStatus.getOrAddresses()) {
+ if (!addresses.contains(orAddress)) {
+ addresses.add(orAddress);
+ }
+ }
+ for (String exitAddress : nodeStatus.getExitAddresses()) {
+ if (!addresses.contains(exitAddress)) {
+ addresses.add(exitAddress);
+ }
+ }
+ long lastSeenMillis = nodeStatus.getLastSeenMillis();
+ boolean running = nodeStatus.getRunning();
+ SortedSet<String> relayFlags = nodeStatus.getRelayFlags();
+ long consensusWeight = nodeStatus.getConsensusWeight();
+ String countryCode = nodeStatus.getCountryCode();
+ long firstSeenMillis = nodeStatus.getFirstSeenMillis();
+ String aSNumber = nodeStatus.getASNumber();
+ String contact = nodeStatus.getContact();
+ SortedSet<String> familyFingerprints =
+ nodeStatus.getFamilyFingerprints();
+ SummaryDocument summaryDocument = new SummaryDocument(isRelay,
+ nickname, fingerprint, addresses, lastSeenMillis, running,
+ relayFlags, consensusWeight, countryCode, firstSeenMillis,
+ aSNumber, contact, familyFingerprints);
+ if (this.documentStore.store(summaryDocument, fingerprint)) {
+ this.writtenDocuments++;
+ };
+ }
+ Logger.printStatusTime("Wrote summary document files");
+ }
+
+ public String getStatsString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(" " + Logger.formatDecimalNumber(this.writtenDocuments)
+ + " summary document files written\n");
+ sb.append(" " + Logger.formatDecimalNumber(this.deletedDocuments)
+ + " summary document files deleted\n");
+ return sb.toString();
+ }
+}
diff --git a/src/org/torproject/onionoo/writer/UptimeDocumentWriter.java b/src/org/torproject/onionoo/writer/UptimeDocumentWriter.java
new file mode 100644
index 0000000..3e04abb
--- /dev/null
+++ b/src/org/torproject/onionoo/writer/UptimeDocumentWriter.java
@@ -0,0 +1,303 @@
+/* Copyright 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.writer;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.torproject.onionoo.docs.DocumentStore;
+import org.torproject.onionoo.docs.GraphHistory;
+import org.torproject.onionoo.docs.UptimeDocument;
+import org.torproject.onionoo.docs.UptimeHistory;
+import org.torproject.onionoo.docs.UptimeStatus;
+import org.torproject.onionoo.updater.DescriptorSource;
+import org.torproject.onionoo.updater.DescriptorType;
+import org.torproject.onionoo.updater.FingerprintListener;
+import org.torproject.onionoo.util.ApplicationFactory;
+import org.torproject.onionoo.util.DateTimeHelper;
+import org.torproject.onionoo.util.Logger;
+
+public class UptimeDocumentWriter implements FingerprintListener,
+ DocumentWriter {
+
+ private DescriptorSource descriptorSource;
+
+ private DocumentStore documentStore;
+
+ private long now;
+
+ public UptimeDocumentWriter() {
+ this.descriptorSource = ApplicationFactory.getDescriptorSource();
+ this.documentStore = ApplicationFactory.getDocumentStore();
+ this.now = ApplicationFactory.getTime().currentTimeMillis();
+ this.registerFingerprintListeners();
+ }
+
+ private void registerFingerprintListeners() {
+ this.descriptorSource.registerFingerprintListener(this,
+ DescriptorType.RELAY_CONSENSUSES);
+ this.descriptorSource.registerFingerprintListener(this,
+ DescriptorType.BRIDGE_STATUSES);
+ }
+
+ private SortedSet<String> newRelayFingerprints = new TreeSet<String>(),
+ newBridgeFingerprints = new TreeSet<String>();
+
+ public void processFingerprints(SortedSet<String> fingerprints,
+ boolean relay) {
+ if (relay) {
+ this.newRelayFingerprints.addAll(fingerprints);
+ } else {
+ this.newBridgeFingerprints.addAll(fingerprints);
+ }
+ }
+
+ public void writeDocuments() {
+ UptimeStatus uptimeStatus = this.documentStore.retrieve(
+ UptimeStatus.class, true);
+ if (uptimeStatus == null) {
+ return;
+ }
+ for (String fingerprint : this.newRelayFingerprints) {
+ this.updateDocument(true, fingerprint,
+ uptimeStatus.getRelayHistory());
+ }
+ for (String fingerprint : this.newBridgeFingerprints) {
+ this.updateDocument(false, fingerprint,
+ uptimeStatus.getBridgeHistory());
+ }
+ Logger.printStatusTime("Wrote uptime document files");
+ }
+
+ private int writtenDocuments = 0;
+
+ private void updateDocument(boolean relay, String fingerprint,
+ SortedSet<UptimeHistory> knownStatuses) {
+ UptimeStatus uptimeStatus = this.documentStore.retrieve(
+ UptimeStatus.class, true, fingerprint);
+ if (uptimeStatus != null) {
+ SortedSet<UptimeHistory> history = relay
+ ? uptimeStatus.getRelayHistory()
+ : uptimeStatus.getBridgeHistory();
+ UptimeDocument uptimeDocument = this.compileUptimeDocument(relay,
+ fingerprint, history, knownStatuses);
+ this.documentStore.store(uptimeDocument, fingerprint);
+ this.writtenDocuments++;
+ }
+ }
+
+ private String[] graphNames = new String[] {
+ "1_week",
+ "1_month",
+ "3_months",
+ "1_year",
+ "5_years" };
+
+ private long[] graphIntervals = new long[] {
+ DateTimeHelper.ONE_WEEK,
+ DateTimeHelper.ROUGHLY_ONE_MONTH,
+ DateTimeHelper.ROUGHLY_THREE_MONTHS,
+ DateTimeHelper.ROUGHLY_ONE_YEAR,
+ DateTimeHelper.ROUGHLY_FIVE_YEARS };
+
+ private long[] dataPointIntervals = new long[] {
+ DateTimeHelper.ONE_HOUR,
+ DateTimeHelper.FOUR_HOURS,
+ DateTimeHelper.TWELVE_HOURS,
+ DateTimeHelper.TWO_DAYS,
+ DateTimeHelper.TEN_DAYS };
+
+ private UptimeDocument compileUptimeDocument(boolean relay,
+ String fingerprint, SortedSet<UptimeHistory> history,
+ SortedSet<UptimeHistory> knownStatuses) {
+ UptimeDocument uptimeDocument = new UptimeDocument();
+ uptimeDocument.setFingerprint(fingerprint);
+ Map<String, GraphHistory> uptime =
+ new LinkedHashMap<String, GraphHistory>();
+ for (int graphIntervalIndex = 0; graphIntervalIndex <
+ this.graphIntervals.length; graphIntervalIndex++) {
+ String graphName = this.graphNames[graphIntervalIndex];
+ GraphHistory graphHistory = this.compileUptimeHistory(
+ graphIntervalIndex, relay, history, knownStatuses);
+ if (graphHistory != null) {
+ uptime.put(graphName, graphHistory);
+ }
+ }
+ uptimeDocument.setUptime(uptime);
+ return uptimeDocument;
+ }
+
+ private GraphHistory compileUptimeHistory(int graphIntervalIndex,
+ boolean relay, SortedSet<UptimeHistory> history,
+ SortedSet<UptimeHistory> knownStatuses) {
+ long graphInterval = this.graphIntervals[graphIntervalIndex];
+ long dataPointInterval =
+ this.dataPointIntervals[graphIntervalIndex];
+ int dataPointIntervalHours = (int) (dataPointInterval
+ / DateTimeHelper.ONE_HOUR);
+ List<Integer> uptimeDataPoints = new ArrayList<Integer>();
+ long intervalStartMillis = ((this.now - graphInterval)
+ / dataPointInterval) * dataPointInterval;
+ int uptimeHours = 0;
+ long firstStatusStartMillis = -1L;
+ for (UptimeHistory hist : history) {
+ if (hist.isRelay() != relay) {
+ continue;
+ }
+ if (firstStatusStartMillis < 0L) {
+ firstStatusStartMillis = hist.getStartMillis();
+ }
+ long histEndMillis = hist.getStartMillis() + DateTimeHelper.ONE_HOUR
+ * hist.getUptimeHours();
+ if (histEndMillis < intervalStartMillis) {
+ continue;
+ }
+ while (hist.getStartMillis() >= intervalStartMillis
+ + dataPointInterval) {
+ if (firstStatusStartMillis < intervalStartMillis
+ + dataPointInterval) {
+ uptimeDataPoints.add(uptimeHours);
+ } else {
+ uptimeDataPoints.add(-1);
+ }
+ uptimeHours = 0;
+ intervalStartMillis += dataPointInterval;
+ }
+ while (histEndMillis >= intervalStartMillis + dataPointInterval) {
+ uptimeHours += (int) ((intervalStartMillis + dataPointInterval
+ - Math.max(hist.getStartMillis(), intervalStartMillis))
+ / DateTimeHelper.ONE_HOUR);
+ uptimeDataPoints.add(uptimeHours);
+ uptimeHours = 0;
+ intervalStartMillis += dataPointInterval;
+ }
+ uptimeHours += (int) ((histEndMillis - Math.max(
+ hist.getStartMillis(), intervalStartMillis))
+ / DateTimeHelper.ONE_HOUR);
+ }
+ uptimeDataPoints.add(uptimeHours);
+ List<Integer> statusDataPoints = new ArrayList<Integer>();
+ intervalStartMillis = ((this.now - graphInterval)
+ / dataPointInterval) * dataPointInterval;
+ int statusHours = -1;
+ for (UptimeHistory hist : knownStatuses) {
+ if (hist.isRelay() != relay) {
+ continue;
+ }
+ long histEndMillis = hist.getStartMillis() + DateTimeHelper.ONE_HOUR
+ * hist.getUptimeHours();
+ if (histEndMillis < intervalStartMillis) {
+ continue;
+ }
+ while (hist.getStartMillis() >= intervalStartMillis
+ + dataPointInterval) {
+ statusDataPoints.add(statusHours * 5 > dataPointIntervalHours
+ ? statusHours : -1);
+ statusHours = -1;
+ intervalStartMillis += dataPointInterval;
+ }
+ while (histEndMillis >= intervalStartMillis + dataPointInterval) {
+ if (statusHours < 0) {
+ statusHours = 0;
+ }
+ statusHours += (int) ((intervalStartMillis + dataPointInterval
+ - Math.max(Math.max(hist.getStartMillis(),
+ firstStatusStartMillis), intervalStartMillis))
+ / DateTimeHelper.ONE_HOUR);
+ statusDataPoints.add(statusHours * 5 > dataPointIntervalHours
+ ? statusHours : -1);
+ statusHours = -1;
+ intervalStartMillis += dataPointInterval;
+ }
+ if (statusHours < 0) {
+ statusHours = 0;
+ }
+ statusHours += (int) ((histEndMillis - Math.max(Math.max(
+ hist.getStartMillis(), firstStatusStartMillis),
+ intervalStartMillis)) / DateTimeHelper.ONE_HOUR);
+ }
+ if (statusHours > 0) {
+ statusDataPoints.add(statusHours * 5 > dataPointIntervalHours
+ ? statusHours : -1);
+ }
+ List<Double> dataPoints = new ArrayList<Double>();
+ for (int dataPointIndex = 0; dataPointIndex < statusDataPoints.size();
+ dataPointIndex++) {
+ if (dataPointIndex >= uptimeDataPoints.size()) {
+ dataPoints.add(0.0);
+ } else if (uptimeDataPoints.get(dataPointIndex) >= 0 &&
+ statusDataPoints.get(dataPointIndex) > 0) {
+ dataPoints.add(((double) uptimeDataPoints.get(dataPointIndex))
+ / ((double) statusDataPoints.get(dataPointIndex)));
+ } else {
+ dataPoints.add(-1.0);
+ }
+ }
+ int firstNonNullIndex = -1, lastNonNullIndex = -1;
+ for (int dataPointIndex = 0; dataPointIndex < dataPoints.size();
+ dataPointIndex++) {
+ double dataPoint = dataPoints.get(dataPointIndex);
+ if (dataPoint >= 0.0) {
+ if (firstNonNullIndex < 0) {
+ firstNonNullIndex = dataPointIndex;
+ }
+ lastNonNullIndex = dataPointIndex;
+ }
+ }
+ if (firstNonNullIndex < 0) {
+ return null;
+ }
+ long firstDataPointMillis = (((this.now - graphInterval)
+ / dataPointInterval) + firstNonNullIndex)
+ * dataPointInterval + dataPointInterval / 2L;
+ if (graphIntervalIndex > 0 && firstDataPointMillis >=
+ this.now - graphIntervals[graphIntervalIndex - 1]) {
+ /* Skip uptime history object, because it doesn't contain
+ * anything new that wasn't already contained in the last
+ * uptime history object(s). */
+ return null;
+ }
+ long lastDataPointMillis = firstDataPointMillis
+ + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval;
+ int count = lastNonNullIndex - firstNonNullIndex + 1;
+ GraphHistory graphHistory = new GraphHistory();
+ graphHistory.setFirst(DateTimeHelper.format(firstDataPointMillis));
+ graphHistory.setLast(DateTimeHelper.format(lastDataPointMillis));
+ graphHistory.setInterval((int) (dataPointInterval
+ / DateTimeHelper.ONE_SECOND));
+ graphHistory.setFactor(1.0 / 999.0);
+ graphHistory.setCount(count);
+ int previousNonNullIndex = -2;
+ boolean foundTwoAdjacentDataPoints = false;
+ List<Integer> values = new ArrayList<Integer>();
+ for (int dataPointIndex = firstNonNullIndex; dataPointIndex <=
+ lastNonNullIndex; dataPointIndex++) {
+ double dataPoint = dataPoints.get(dataPointIndex);
+ if (dataPoint >= 0.0) {
+ if (dataPointIndex - previousNonNullIndex == 1) {
+ foundTwoAdjacentDataPoints = true;
+ }
+ previousNonNullIndex = dataPointIndex;
+ }
+ values.add(dataPoint < -0.5 ? null : ((int) (dataPoint * 999.0)));
+ }
+ graphHistory.setValues(values);
+ if (foundTwoAdjacentDataPoints) {
+ return graphHistory;
+ } else {
+ return null;
+ }
+ }
+
+ public String getStatsString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(" " + Logger.formatDecimalNumber(this.writtenDocuments)
+ + " uptime document files written\n");
+ return sb.toString();
+ }
+}
+
diff --git a/src/org/torproject/onionoo/writer/WeightsDocumentWriter.java b/src/org/torproject/onionoo/writer/WeightsDocumentWriter.java
new file mode 100644
index 0000000..ddd2774
--- /dev/null
+++ b/src/org/torproject/onionoo/writer/WeightsDocumentWriter.java
@@ -0,0 +1,233 @@
+/* Copyright 2012--2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.writer;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.SortedSet;
+
+import org.torproject.onionoo.docs.DocumentStore;
+import org.torproject.onionoo.docs.GraphHistory;
+import org.torproject.onionoo.docs.WeightsDocument;
+import org.torproject.onionoo.docs.WeightsStatus;
+import org.torproject.onionoo.updater.DescriptorSource;
+import org.torproject.onionoo.updater.DescriptorType;
+import org.torproject.onionoo.updater.FingerprintListener;
+import org.torproject.onionoo.util.ApplicationFactory;
+import org.torproject.onionoo.util.DateTimeHelper;
+import org.torproject.onionoo.util.Logger;
+
+public class WeightsDocumentWriter implements FingerprintListener,
+ DocumentWriter {
+
+ private DescriptorSource descriptorSource;
+
+ private DocumentStore documentStore;
+
+ private long now;
+
+ public WeightsDocumentWriter() {
+ this.descriptorSource = ApplicationFactory.getDescriptorSource();
+ this.documentStore = ApplicationFactory.getDocumentStore();
+ this.now = ApplicationFactory.getTime().currentTimeMillis();
+ this.registerFingerprintListeners();
+ }
+
+ private void registerFingerprintListeners() {
+ this.descriptorSource.registerFingerprintListener(this,
+ DescriptorType.RELAY_CONSENSUSES);
+ this.descriptorSource.registerFingerprintListener(this,
+ DescriptorType.RELAY_SERVER_DESCRIPTORS);
+ }
+
+ private Set<String> updateWeightsDocuments = new HashSet<String>();
+
+ public void processFingerprints(SortedSet<String> fingerprints,
+ boolean relay) {
+ if (relay) {
+ this.updateWeightsDocuments.addAll(fingerprints);
+ }
+ }
+
+ public void writeDocuments() {
+ this.writeWeightsDataFiles();
+ Logger.printStatusTime("Wrote weights document files");
+ }
+
+ private void writeWeightsDataFiles() {
+ for (String fingerprint : this.updateWeightsDocuments) {
+ WeightsStatus weightsStatus = this.documentStore.retrieve(
+ WeightsStatus.class, true, fingerprint);
+ if (weightsStatus == null) {
+ continue;
+ }
+ SortedMap<long[], double[]> history = weightsStatus.getHistory();
+ WeightsDocument weightsDocument = this.compileWeightsDocument(
+ fingerprint, history);
+ this.documentStore.store(weightsDocument, fingerprint);
+ }
+ Logger.printStatusTime("Wrote weights document files");
+ }
+
+ private String[] graphNames = new String[] {
+ "1_week",
+ "1_month",
+ "3_months",
+ "1_year",
+ "5_years" };
+
+ private long[] graphIntervals = new long[] {
+ DateTimeHelper.ONE_WEEK,
+ DateTimeHelper.ROUGHLY_ONE_MONTH,
+ DateTimeHelper.ROUGHLY_THREE_MONTHS,
+ DateTimeHelper.ROUGHLY_ONE_YEAR,
+ DateTimeHelper.ROUGHLY_FIVE_YEARS };
+
+ private long[] dataPointIntervals = new long[] {
+ DateTimeHelper.ONE_HOUR,
+ DateTimeHelper.FOUR_HOURS,
+ DateTimeHelper.TWELVE_HOURS,
+ DateTimeHelper.TWO_DAYS,
+ DateTimeHelper.TEN_DAYS };
+
+ private WeightsDocument compileWeightsDocument(String fingerprint,
+ SortedMap<long[], double[]> history) {
+ WeightsDocument weightsDocument = new WeightsDocument();
+ weightsDocument.setFingerprint(fingerprint);
+ weightsDocument.setAdvertisedBandwidthFraction(
+ this.compileGraphType(history, 0));
+ weightsDocument.setConsensusWeightFraction(
+ this.compileGraphType(history, 1));
+ weightsDocument.setGuardProbability(
+ this.compileGraphType(history, 2));
+ weightsDocument.setMiddleProbability(
+ this.compileGraphType(history, 3));
+ weightsDocument.setExitProbability(
+ this.compileGraphType(history, 4));
+ weightsDocument.setAdvertisedBandwidth(
+ this.compileGraphType(history, 5));
+ weightsDocument.setConsensusWeight(
+ this.compileGraphType(history, 6));
+ return weightsDocument;
+ }
+
+ private Map<String, GraphHistory> compileGraphType(
+ SortedMap<long[], double[]> history, int graphTypeIndex) {
+ Map<String, GraphHistory> graphs =
+ new LinkedHashMap<String, GraphHistory>();
+ for (int graphIntervalIndex = 0; graphIntervalIndex <
+ this.graphIntervals.length; graphIntervalIndex++) {
+ String graphName = this.graphNames[graphIntervalIndex];
+ GraphHistory graphHistory = this.compileWeightsHistory(
+ graphTypeIndex, graphIntervalIndex, history);
+ if (graphHistory != null) {
+ graphs.put(graphName, graphHistory);
+ }
+ }
+ return graphs;
+ }
+
+ private GraphHistory compileWeightsHistory(int graphTypeIndex,
+ int graphIntervalIndex, SortedMap<long[], double[]> history) {
+ long graphInterval = this.graphIntervals[graphIntervalIndex];
+ long dataPointInterval =
+ this.dataPointIntervals[graphIntervalIndex];
+ List<Double> dataPoints = new ArrayList<Double>();
+ long intervalStartMillis = ((this.now - graphInterval)
+ / dataPointInterval) * dataPointInterval;
+ long totalMillis = 0L;
+ double totalWeightTimesMillis = 0.0;
+ for (Map.Entry<long[], double[]> e : history.entrySet()) {
+ long startMillis = e.getKey()[0], endMillis = e.getKey()[1];
+ double weight = e.getValue()[graphTypeIndex];
+ if (endMillis < intervalStartMillis) {
+ continue;
+ }
+ while ((intervalStartMillis / dataPointInterval) !=
+ (endMillis / dataPointInterval)) {
+ dataPoints.add(totalMillis * 5L < dataPointInterval
+ ? -1.0 : totalWeightTimesMillis / (double) totalMillis);
+ totalWeightTimesMillis = 0.0;
+ totalMillis = 0L;
+ intervalStartMillis += dataPointInterval;
+ }
+ if (weight >= 0.0) {
+ totalWeightTimesMillis += weight
+ * ((double) (endMillis - startMillis));
+ totalMillis += (endMillis - startMillis);
+ }
+ }
+ dataPoints.add(totalMillis * 5L < dataPointInterval
+ ? -1.0 : totalWeightTimesMillis / (double) totalMillis);
+ double maxValue = 0.0;
+ int firstNonNullIndex = -1, lastNonNullIndex = -1;
+ for (int dataPointIndex = 0; dataPointIndex < dataPoints.size();
+ dataPointIndex++) {
+ double dataPoint = dataPoints.get(dataPointIndex);
+ if (dataPoint >= 0.0) {
+ if (firstNonNullIndex < 0) {
+ firstNonNullIndex = dataPointIndex;
+ }
+ lastNonNullIndex = dataPointIndex;
+ if (dataPoint > maxValue) {
+ maxValue = dataPoint;
+ }
+ }
+ }
+ if (firstNonNullIndex < 0) {
+ return null;
+ }
+ long firstDataPointMillis = (((this.now - graphInterval)
+ / dataPointInterval) + firstNonNullIndex) * dataPointInterval
+ + dataPointInterval / 2L;
+ if (graphIntervalIndex > 0 && firstDataPointMillis >=
+ this.now - graphIntervals[graphIntervalIndex - 1]) {
+ /* Skip weights history object, because it doesn't contain
+ * anything new that wasn't already contained in the last
+ * weights history object(s). */
+ return null;
+ }
+ long lastDataPointMillis = firstDataPointMillis
+ + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval;
+ double factor = ((double) maxValue) / 999.0;
+ int count = lastNonNullIndex - firstNonNullIndex + 1;
+ GraphHistory graphHistory = new GraphHistory();
+ graphHistory.setFirst(DateTimeHelper.format(firstDataPointMillis));
+ graphHistory.setLast(DateTimeHelper.format(lastDataPointMillis));
+ graphHistory.setInterval((int) (dataPointInterval
+ / DateTimeHelper.ONE_SECOND));
+ graphHistory.setFactor(factor);
+ graphHistory.setCount(count);
+ int previousNonNullIndex = -2;
+ boolean foundTwoAdjacentDataPoints = false;
+ List<Integer> values = new ArrayList<Integer>();
+ for (int dataPointIndex = firstNonNullIndex; dataPointIndex <=
+ lastNonNullIndex; dataPointIndex++) {
+ double dataPoint = dataPoints.get(dataPointIndex);
+ if (dataPoint >= 0.0) {
+ if (dataPointIndex - previousNonNullIndex == 1) {
+ foundTwoAdjacentDataPoints = true;
+ }
+ previousNonNullIndex = dataPointIndex;
+ }
+ values.add(dataPoint < 0.0 ? null :
+ (int) ((dataPoint * 999.0) / maxValue));
+ }
+ graphHistory.setValues(values);
+ if (foundTwoAdjacentDataPoints) {
+ return graphHistory;
+ } else {
+ return null;
+ }
+ }
+
+ public String getStatsString() {
+ /* TODO Add statistics string. */
+ return null;
+ }
+}
diff --git a/test/org/torproject/onionoo/DummyDescriptorSource.java b/test/org/torproject/onionoo/DummyDescriptorSource.java
index 06ec499..e93b063 100644
--- a/test/org/torproject/onionoo/DummyDescriptorSource.java
+++ b/test/org/torproject/onionoo/DummyDescriptorSource.java
@@ -9,6 +9,10 @@ import java.util.SortedSet;
import java.util.TreeSet;
import org.torproject.descriptor.Descriptor;
+import org.torproject.onionoo.updater.DescriptorListener;
+import org.torproject.onionoo.updater.DescriptorSource;
+import org.torproject.onionoo.updater.DescriptorType;
+import org.torproject.onionoo.updater.FingerprintListener;
public class DummyDescriptorSource extends DescriptorSource {
diff --git a/test/org/torproject/onionoo/DummyDocumentStore.java b/test/org/torproject/onionoo/DummyDocumentStore.java
index 5a5905b..54311aa 100644
--- a/test/org/torproject/onionoo/DummyDocumentStore.java
+++ b/test/org/torproject/onionoo/DummyDocumentStore.java
@@ -7,6 +7,9 @@ import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
+import org.torproject.onionoo.docs.Document;
+import org.torproject.onionoo.docs.DocumentStore;
+
public class DummyDocumentStore extends DocumentStore {
private Map<Class<? extends Document>, SortedMap<String, Document>>
diff --git a/test/org/torproject/onionoo/DummyTime.java b/test/org/torproject/onionoo/DummyTime.java
index 7178ed1..ffbd6e3 100644
--- a/test/org/torproject/onionoo/DummyTime.java
+++ b/test/org/torproject/onionoo/DummyTime.java
@@ -2,6 +2,8 @@
* See LICENSE for licensing information */
package org.torproject.onionoo;
+import org.torproject.onionoo.util.Time;
+
public class DummyTime extends Time {
private long currentTimeMillis;
public DummyTime(long currentTimeMillis) {
diff --git a/test/org/torproject/onionoo/LookupServiceTest.java b/test/org/torproject/onionoo/LookupServiceTest.java
index 23b5a91..052b4c0 100644
--- a/test/org/torproject/onionoo/LookupServiceTest.java
+++ b/test/org/torproject/onionoo/LookupServiceTest.java
@@ -22,6 +22,8 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+import org.torproject.onionoo.updater.LookupResult;
+import org.torproject.onionoo.updater.LookupService;
public class LookupServiceTest {
diff --git a/test/org/torproject/onionoo/ResourceServletTest.java b/test/org/torproject/onionoo/ResourceServletTest.java
index f2ace5d..8cd3752 100644
--- a/test/org/torproject/onionoo/ResourceServletTest.java
+++ b/test/org/torproject/onionoo/ResourceServletTest.java
@@ -21,8 +21,14 @@ import java.util.TreeSet;
import org.junit.Before;
import org.junit.Test;
-import org.torproject.onionoo.ResourceServlet.HttpServletRequestWrapper;
-import org.torproject.onionoo.ResourceServlet.HttpServletResponseWrapper;
+import org.torproject.onionoo.docs.UpdateStatus;
+import org.torproject.onionoo.server.NodeIndexer;
+import org.torproject.onionoo.server.ResourceServlet;
+import org.torproject.onionoo.server.ResourceServlet.HttpServletRequestWrapper;
+import org.torproject.onionoo.server.ResourceServlet.HttpServletResponseWrapper;
+import org.torproject.onionoo.util.ApplicationFactory;
+import org.torproject.onionoo.util.DateTimeHelper;
+import org.torproject.onionoo.util.Time;
import com.google.gson.Gson;
@@ -31,7 +37,7 @@ import com.google.gson.Gson;
* which tests servlet specifics. */
public class ResourceServletTest {
- private SortedMap<String, org.torproject.onionoo.SummaryDocument>
+ private SortedMap<String, org.torproject.onionoo.docs.SummaryDocument>
relays, bridges;
private long currentTimeMillis = DateTimeHelper.parse(
@@ -102,8 +108,8 @@ public class ResourceServletTest {
@Before
public void createSampleRelaysAndBridges() {
- org.torproject.onionoo.SummaryDocument relayTorkaZ =
- new org.torproject.onionoo.SummaryDocument(true, "TorkaZ",
+ org.torproject.onionoo.docs.SummaryDocument relayTorkaZ =
+ new org.torproject.onionoo.docs.SummaryDocument(true, "TorkaZ",
"000C5F55BD4814B917CC474BD537F1A3B33CCE2A", Arrays.asList(
new String[] { "62.216.201.221", "62.216.201.222",
"62.216.201.223" }), DateTimeHelper.parse("2013-04-19 05:00:00"),
@@ -114,8 +120,8 @@ public class ResourceServletTest {
+ "<fb-token:np5_g_83jmf=>", new TreeSet<String>(Arrays.asList(
new String[] { "001C13B3A55A71B977CA65EC85539D79C653A3FC",
"0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B" })));
- org.torproject.onionoo.SummaryDocument relayFerrari458 =
- new org.torproject.onionoo.SummaryDocument(true, "Ferrari458",
+ org.torproject.onionoo.docs.SummaryDocument relayFerrari458 =
+ new org.torproject.onionoo.docs.SummaryDocument(true, "Ferrari458",
"001C13B3A55A71B977CA65EC85539D79C653A3FC", Arrays.asList(
new String[] { "68.38.171.200", "[2001:4f8:3:2e::51]" }),
DateTimeHelper.parse("2013-04-24 12:00:00"), true,
@@ -124,8 +130,8 @@ public class ResourceServletTest {
DateTimeHelper.parse("2013-02-12 16:00:00"), "AS7922", null,
new TreeSet<String>(Arrays.asList(new String[] {
"000C5F55BD4814B917CC474BD537F1A3B33CCE2A" })));
- org.torproject.onionoo.SummaryDocument relayTimMayTribute =
- new org.torproject.onionoo.SummaryDocument(true, "TimMayTribute",
+ org.torproject.onionoo.docs.SummaryDocument relayTimMayTribute =
+ new org.torproject.onionoo.docs.SummaryDocument(true, "TimMayTribute",
"0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B", Arrays.asList(
new String[] { "89.69.68.246" }),
DateTimeHelper.parse("2013-04-22 20:00:00"), false,
@@ -135,24 +141,24 @@ public class ResourceServletTest {
"1024D/51E2A1C7 steven j. murdoch "
+ "<tor+steven.murdoch@xxxxxxxxxxxx> <fb-token:5sr_k_zs2wm=>",
new TreeSet<String>());
- org.torproject.onionoo.SummaryDocument bridgeec2bridgercc7f31fe =
- new org.torproject.onionoo.SummaryDocument(false,
+ org.torproject.onionoo.docs.SummaryDocument bridgeec2bridgercc7f31fe =
+ new org.torproject.onionoo.docs.SummaryDocument(false,
"ec2bridgercc7f31fe", "0000831B236DFF73D409AD17B40E2A728A53994F",
Arrays.asList(new String[] { "10.199.7.176" }),
DateTimeHelper.parse("2013-04-21 18:07:03"), false,
new TreeSet<String>(Arrays.asList(new String[] { "Valid" })), -1L,
null, DateTimeHelper.parse("2013-04-20 15:37:04"), null, null,
null);
- org.torproject.onionoo.SummaryDocument bridgeUnnamed =
- new org.torproject.onionoo.SummaryDocument(false, "Unnamed",
+ org.torproject.onionoo.docs.SummaryDocument bridgeUnnamed =
+ new org.torproject.onionoo.docs.SummaryDocument(false, "Unnamed",
"0002D9BDBBC230BD9C78FF502A16E0033EF87E0C", Arrays.asList(
new String[] { "10.0.52.84" }),
DateTimeHelper.parse("2013-04-20 17:37:04"), false,
new TreeSet<String>(Arrays.asList(new String[] { "Valid" })), -1L,
null, DateTimeHelper.parse("2013-04-14 07:07:05"), null, null,
null);
- org.torproject.onionoo.SummaryDocument bridgegummy =
- new org.torproject.onionoo.SummaryDocument(false, "gummy",
+ org.torproject.onionoo.docs.SummaryDocument bridgegummy =
+ new org.torproject.onionoo.docs.SummaryDocument(false, "gummy",
"1FEDE50ED8DBA1DD9F9165F78C8131E4A44AB756", Arrays.asList(
new String[] { "10.63.169.98" }),
DateTimeHelper.parse("2013-04-24 01:07:04"), true,
@@ -160,7 +166,7 @@ public class ResourceServletTest {
"Valid" })), -1L, null,
DateTimeHelper.parse("2013-01-16 21:07:04"), null, null, null);
this.relays =
- new TreeMap<String, org.torproject.onionoo.SummaryDocument>();
+ new TreeMap<String, org.torproject.onionoo.docs.SummaryDocument>();
this.relays.put("000C5F55BD4814B917CC474BD537F1A3B33CCE2A",
relayTorkaZ);
this.relays.put("001C13B3A55A71B977CA65EC85539D79C653A3FC",
@@ -168,7 +174,7 @@ public class ResourceServletTest {
this.relays.put("0025C136C1F3A9EEFE2AE3F918F03BFA21B5070B",
relayTimMayTribute);
this.bridges =
- new TreeMap<String, org.torproject.onionoo.SummaryDocument>();
+ new TreeMap<String, org.torproject.onionoo.docs.SummaryDocument>();
this.bridges.put("0000831B236DFF73D409AD17B40E2A728A53994F",
bridgeec2bridgercc7f31fe);
this.bridges.put("0002D9BDBBC230BD9C78FF502A16E0033EF87E0C",
@@ -201,11 +207,11 @@ public class ResourceServletTest {
updateStatus.setDocumentString(String.valueOf(
this.currentTimeMillis));
documentStore.addDocument(updateStatus, null);
- for (Map.Entry<String, org.torproject.onionoo.SummaryDocument> e :
+ for (Map.Entry<String, org.torproject.onionoo.docs.SummaryDocument> e :
this.relays.entrySet()) {
documentStore.addDocument(e.getValue(), e.getKey());
}
- for (Map.Entry<String, org.torproject.onionoo.SummaryDocument> e :
+ for (Map.Entry<String, org.torproject.onionoo.docs.SummaryDocument> e :
this.bridges.entrySet()) {
documentStore.addDocument(e.getValue(), e.getKey());
}
diff --git a/test/org/torproject/onionoo/UptimeDocumentWriterTest.java b/test/org/torproject/onionoo/UptimeDocumentWriterTest.java
index 5065e4d..5a77514 100644
--- a/test/org/torproject/onionoo/UptimeDocumentWriterTest.java
+++ b/test/org/torproject/onionoo/UptimeDocumentWriterTest.java
@@ -10,6 +10,13 @@ import java.util.List;
import org.junit.Before;
import org.junit.Test;
+import org.torproject.onionoo.docs.GraphHistory;
+import org.torproject.onionoo.docs.UptimeDocument;
+import org.torproject.onionoo.docs.UptimeStatus;
+import org.torproject.onionoo.updater.DescriptorType;
+import org.torproject.onionoo.util.ApplicationFactory;
+import org.torproject.onionoo.util.DateTimeHelper;
+import org.torproject.onionoo.writer.UptimeDocumentWriter;
public class UptimeDocumentWriterTest {
diff --git a/test/org/torproject/onionoo/UptimeStatusTest.java b/test/org/torproject/onionoo/UptimeStatusTest.java
index 884ccc5..671ffa3 100644
--- a/test/org/torproject/onionoo/UptimeStatusTest.java
+++ b/test/org/torproject/onionoo/UptimeStatusTest.java
@@ -9,6 +9,10 @@ import java.util.TreeSet;
import org.junit.Before;
import org.junit.Test;
+import org.torproject.onionoo.docs.UptimeHistory;
+import org.torproject.onionoo.docs.UptimeStatus;
+import org.torproject.onionoo.util.ApplicationFactory;
+import org.torproject.onionoo.util.DateTimeHelper;
public class UptimeStatusTest {
diff --git a/test/org/torproject/onionoo/UptimeStatusUpdaterTest.java b/test/org/torproject/onionoo/UptimeStatusUpdaterTest.java
index a34292b..8070ae4 100644
--- a/test/org/torproject/onionoo/UptimeStatusUpdaterTest.java
+++ b/test/org/torproject/onionoo/UptimeStatusUpdaterTest.java
@@ -6,6 +6,12 @@ import static org.junit.Assert.assertEquals;
import org.junit.Before;
import org.junit.Test;
+import org.torproject.onionoo.docs.UptimeHistory;
+import org.torproject.onionoo.docs.UptimeStatus;
+import org.torproject.onionoo.updater.DescriptorType;
+import org.torproject.onionoo.updater.UptimeStatusUpdater;
+import org.torproject.onionoo.util.ApplicationFactory;
+import org.torproject.onionoo.util.DateTimeHelper;
public class UptimeStatusUpdaterTest {
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits