[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [onionoo/master] Make all inner classes first-class classes.
commit 45359de3263c928c6cd7e8f2e3916bb565eeafbd
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Wed Jul 23 18:03:28 2014 +0200
Make all inner classes first-class classes.
---
.../onionoo/server/HttpServletRequestWrapper.java | 24 ++
.../onionoo/server/HttpServletResponseWrapper.java | 30 ++
src/org/torproject/onionoo/server/NodeIndex.java | 142 +++++++
src/org/torproject/onionoo/server/NodeIndexer.java | 134 -------
.../torproject/onionoo/server/ResourceServlet.java | 39 --
.../onionoo/updater/DescriptorDownloader.java | 178 +++++++++
.../onionoo/updater/DescriptorHistory.java | 12 +
.../onionoo/updater/DescriptorQueue.java | 221 +++++++++++
.../onionoo/updater/DescriptorSource.java | 396 --------------------
.../onionoo/updater/RdnsLookupRequest.java | 43 +++
.../onionoo/updater/RdnsLookupWorker.java | 55 +++
.../onionoo/updater/ReverseDomainNameResolver.java | 87 +----
.../torproject/onionoo/ResourceServletTest.java | 4 +-
13 files changed, 715 insertions(+), 650 deletions(-)
diff --git a/src/org/torproject/onionoo/server/HttpServletRequestWrapper.java b/src/org/torproject/onionoo/server/HttpServletRequestWrapper.java
new file mode 100644
index 0000000..3349acd
--- /dev/null
+++ b/src/org/torproject/onionoo/server/HttpServletRequestWrapper.java
@@ -0,0 +1,24 @@
+/* Copyright 2011, 2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.server;
+
+import java.util.Map;
+
+import javax.servlet.http.HttpServletRequest;
+
+public class HttpServletRequestWrapper {
+ private HttpServletRequest request;
+ protected HttpServletRequestWrapper(HttpServletRequest request) {
+ this.request = request;
+ }
+ protected String getRequestURI() {
+ return this.request.getRequestURI();
+ }
+ @SuppressWarnings("rawtypes")
+ protected Map getParameterMap() {
+ return this.request.getParameterMap();
+ }
+ protected String[] getParameterValues(String parameterKey) {
+ return this.request.getParameterValues(parameterKey);
+ }
+}
\ No newline at end of file
diff --git a/src/org/torproject/onionoo/server/HttpServletResponseWrapper.java b/src/org/torproject/onionoo/server/HttpServletResponseWrapper.java
new file mode 100644
index 0000000..58d9f03
--- /dev/null
+++ b/src/org/torproject/onionoo/server/HttpServletResponseWrapper.java
@@ -0,0 +1,30 @@
+/* Copyright 2011, 2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.server;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+
+import javax.servlet.http.HttpServletResponse;
+
+public class HttpServletResponseWrapper {
+ private HttpServletResponse response = null;
+ protected HttpServletResponseWrapper(HttpServletResponse response) {
+ this.response = response;
+ }
+ protected void sendError(int errorStatusCode) throws IOException {
+ this.response.sendError(errorStatusCode);
+ }
+ protected void setHeader(String headerName, String headerValue) {
+ this.response.setHeader(headerName, headerValue);
+ }
+ protected void setContentType(String contentType) {
+ this.response.setContentType(contentType);
+ }
+ protected void setCharacterEncoding(String characterEncoding) {
+ this.response.setCharacterEncoding(characterEncoding);
+ }
+ protected PrintWriter getWriter() throws IOException {
+ return this.response.getWriter();
+ }
+}
\ No newline at end of file
diff --git a/src/org/torproject/onionoo/server/NodeIndex.java b/src/org/torproject/onionoo/server/NodeIndex.java
new file mode 100644
index 0000000..7b95d2e
--- /dev/null
+++ b/src/org/torproject/onionoo/server/NodeIndex.java
@@ -0,0 +1,142 @@
+package org.torproject.onionoo.server;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedMap;
+
+import org.torproject.onionoo.docs.SummaryDocument;
+
+class NodeIndex {
+
+ private String relaysPublishedString;
+ public void setRelaysPublishedString(String relaysPublishedString) {
+ this.relaysPublishedString = relaysPublishedString;
+ }
+ public String getRelaysPublishedString() {
+ return relaysPublishedString;
+ }
+
+ private String bridgesPublishedString;
+ public void setBridgesPublishedString(String bridgesPublishedString) {
+ this.bridgesPublishedString = bridgesPublishedString;
+ }
+ public String getBridgesPublishedString() {
+ return bridgesPublishedString;
+ }
+
+ private List<String> relaysByConsensusWeight;
+ public void setRelaysByConsensusWeight(
+ List<String> relaysByConsensusWeight) {
+ this.relaysByConsensusWeight = relaysByConsensusWeight;
+ }
+ public List<String> getRelaysByConsensusWeight() {
+ return relaysByConsensusWeight;
+ }
+
+
+ private Map<String, SummaryDocument> relayFingerprintSummaryLines;
+ public void setRelayFingerprintSummaryLines(
+ Map<String, SummaryDocument> relayFingerprintSummaryLines) {
+ this.relayFingerprintSummaryLines = relayFingerprintSummaryLines;
+ }
+ public Map<String, SummaryDocument> getRelayFingerprintSummaryLines() {
+ return this.relayFingerprintSummaryLines;
+ }
+
+ private Map<String, SummaryDocument> bridgeFingerprintSummaryLines;
+ public void setBridgeFingerprintSummaryLines(
+ Map<String, SummaryDocument> bridgeFingerprintSummaryLines) {
+ this.bridgeFingerprintSummaryLines = bridgeFingerprintSummaryLines;
+ }
+ public Map<String, SummaryDocument> getBridgeFingerprintSummaryLines() {
+ return this.bridgeFingerprintSummaryLines;
+ }
+
+ private Map<String, Set<String>> relaysByCountryCode = null;
+ public void setRelaysByCountryCode(
+ Map<String, Set<String>> relaysByCountryCode) {
+ this.relaysByCountryCode = relaysByCountryCode;
+ }
+ public Map<String, Set<String>> getRelaysByCountryCode() {
+ return relaysByCountryCode;
+ }
+
+ private Map<String, Set<String>> relaysByASNumber = null;
+ public void setRelaysByASNumber(
+ Map<String, Set<String>> relaysByASNumber) {
+ this.relaysByASNumber = relaysByASNumber;
+ }
+ public Map<String, Set<String>> getRelaysByASNumber() {
+ return relaysByASNumber;
+ }
+
+ private Map<String, Set<String>> relaysByFlag = null;
+ public void setRelaysByFlag(Map<String, Set<String>> relaysByFlag) {
+ this.relaysByFlag = relaysByFlag;
+ }
+ public Map<String, Set<String>> getRelaysByFlag() {
+ return relaysByFlag;
+ }
+
+ private Map<String, Set<String>> bridgesByFlag = null;
+ public void setBridgesByFlag(Map<String, Set<String>> bridgesByFlag) {
+ this.bridgesByFlag = bridgesByFlag;
+ }
+ public Map<String, Set<String>> getBridgesByFlag() {
+ return bridgesByFlag;
+ }
+
+ private Map<String, Set<String>> relaysByContact = null;
+ public void setRelaysByContact(
+ Map<String, Set<String>> relaysByContact) {
+ this.relaysByContact = relaysByContact;
+ }
+ public Map<String, Set<String>> getRelaysByContact() {
+ return relaysByContact;
+ }
+
+ private Map<String, Set<String>> relaysByFamily = null;
+ public void setRelaysByFamily(Map<String, Set<String>> relaysByFamily) {
+ this.relaysByFamily = relaysByFamily;
+ }
+ public Map<String, Set<String>> getRelaysByFamily() {
+ return this.relaysByFamily;
+ }
+
+ private SortedMap<Integer, Set<String>> relaysByFirstSeenDays;
+ public void setRelaysByFirstSeenDays(
+ SortedMap<Integer, Set<String>> relaysByFirstSeenDays) {
+ this.relaysByFirstSeenDays = relaysByFirstSeenDays;
+ }
+ public SortedMap<Integer, Set<String>> getRelaysByFirstSeenDays() {
+ return relaysByFirstSeenDays;
+ }
+
+ private SortedMap<Integer, Set<String>> bridgesByFirstSeenDays;
+ public void setBridgesByFirstSeenDays(
+ SortedMap<Integer, Set<String>> bridgesByFirstSeenDays) {
+ this.bridgesByFirstSeenDays = bridgesByFirstSeenDays;
+ }
+ public SortedMap<Integer, Set<String>> getBridgesByFirstSeenDays() {
+ return bridgesByFirstSeenDays;
+ }
+
+ private SortedMap<Integer, Set<String>> relaysByLastSeenDays;
+ public void setRelaysByLastSeenDays(
+ SortedMap<Integer, Set<String>> relaysByLastSeenDays) {
+ this.relaysByLastSeenDays = relaysByLastSeenDays;
+ }
+ public SortedMap<Integer, Set<String>> getRelaysByLastSeenDays() {
+ return relaysByLastSeenDays;
+ }
+
+ private SortedMap<Integer, Set<String>> bridgesByLastSeenDays;
+ public void setBridgesByLastSeenDays(
+ SortedMap<Integer, Set<String>> bridgesByLastSeenDays) {
+ this.bridgesByLastSeenDays = bridgesByLastSeenDays;
+ }
+ public SortedMap<Integer, Set<String>> getBridgesByLastSeenDays() {
+ return bridgesByLastSeenDays;
+ }
+}
\ No newline at end of file
diff --git a/src/org/torproject/onionoo/server/NodeIndexer.java b/src/org/torproject/onionoo/server/NodeIndexer.java
index b76f4c1..22d8608 100644
--- a/src/org/torproject/onionoo/server/NodeIndexer.java
+++ b/src/org/torproject/onionoo/server/NodeIndexer.java
@@ -23,140 +23,6 @@ import org.torproject.onionoo.util.ApplicationFactory;
import org.torproject.onionoo.util.DateTimeHelper;
import org.torproject.onionoo.util.Time;
-class NodeIndex {
-
- private String relaysPublishedString;
- public void setRelaysPublishedString(String relaysPublishedString) {
- this.relaysPublishedString = relaysPublishedString;
- }
- public String getRelaysPublishedString() {
- return relaysPublishedString;
- }
-
- private String bridgesPublishedString;
- public void setBridgesPublishedString(String bridgesPublishedString) {
- this.bridgesPublishedString = bridgesPublishedString;
- }
- public String getBridgesPublishedString() {
- return bridgesPublishedString;
- }
-
- private List<String> relaysByConsensusWeight;
- public void setRelaysByConsensusWeight(
- List<String> relaysByConsensusWeight) {
- this.relaysByConsensusWeight = relaysByConsensusWeight;
- }
- public List<String> getRelaysByConsensusWeight() {
- return relaysByConsensusWeight;
- }
-
-
- private Map<String, SummaryDocument> relayFingerprintSummaryLines;
- public void setRelayFingerprintSummaryLines(
- Map<String, SummaryDocument> relayFingerprintSummaryLines) {
- this.relayFingerprintSummaryLines = relayFingerprintSummaryLines;
- }
- public Map<String, SummaryDocument> getRelayFingerprintSummaryLines() {
- return this.relayFingerprintSummaryLines;
- }
-
- private Map<String, SummaryDocument> bridgeFingerprintSummaryLines;
- public void setBridgeFingerprintSummaryLines(
- Map<String, SummaryDocument> bridgeFingerprintSummaryLines) {
- this.bridgeFingerprintSummaryLines = bridgeFingerprintSummaryLines;
- }
- public Map<String, SummaryDocument> getBridgeFingerprintSummaryLines() {
- return this.bridgeFingerprintSummaryLines;
- }
-
- private Map<String, Set<String>> relaysByCountryCode = null;
- public void setRelaysByCountryCode(
- Map<String, Set<String>> relaysByCountryCode) {
- this.relaysByCountryCode = relaysByCountryCode;
- }
- public Map<String, Set<String>> getRelaysByCountryCode() {
- return relaysByCountryCode;
- }
-
- private Map<String, Set<String>> relaysByASNumber = null;
- public void setRelaysByASNumber(
- Map<String, Set<String>> relaysByASNumber) {
- this.relaysByASNumber = relaysByASNumber;
- }
- public Map<String, Set<String>> getRelaysByASNumber() {
- return relaysByASNumber;
- }
-
- private Map<String, Set<String>> relaysByFlag = null;
- public void setRelaysByFlag(Map<String, Set<String>> relaysByFlag) {
- this.relaysByFlag = relaysByFlag;
- }
- public Map<String, Set<String>> getRelaysByFlag() {
- return relaysByFlag;
- }
-
- private Map<String, Set<String>> bridgesByFlag = null;
- public void setBridgesByFlag(Map<String, Set<String>> bridgesByFlag) {
- this.bridgesByFlag = bridgesByFlag;
- }
- public Map<String, Set<String>> getBridgesByFlag() {
- return bridgesByFlag;
- }
-
- private Map<String, Set<String>> relaysByContact = null;
- public void setRelaysByContact(
- Map<String, Set<String>> relaysByContact) {
- this.relaysByContact = relaysByContact;
- }
- public Map<String, Set<String>> getRelaysByContact() {
- return relaysByContact;
- }
-
- private Map<String, Set<String>> relaysByFamily = null;
- public void setRelaysByFamily(Map<String, Set<String>> relaysByFamily) {
- this.relaysByFamily = relaysByFamily;
- }
- public Map<String, Set<String>> getRelaysByFamily() {
- return this.relaysByFamily;
- }
-
- private SortedMap<Integer, Set<String>> relaysByFirstSeenDays;
- public void setRelaysByFirstSeenDays(
- SortedMap<Integer, Set<String>> relaysByFirstSeenDays) {
- this.relaysByFirstSeenDays = relaysByFirstSeenDays;
- }
- public SortedMap<Integer, Set<String>> getRelaysByFirstSeenDays() {
- return relaysByFirstSeenDays;
- }
-
- private SortedMap<Integer, Set<String>> bridgesByFirstSeenDays;
- public void setBridgesByFirstSeenDays(
- SortedMap<Integer, Set<String>> bridgesByFirstSeenDays) {
- this.bridgesByFirstSeenDays = bridgesByFirstSeenDays;
- }
- public SortedMap<Integer, Set<String>> getBridgesByFirstSeenDays() {
- return bridgesByFirstSeenDays;
- }
-
- private SortedMap<Integer, Set<String>> relaysByLastSeenDays;
- public void setRelaysByLastSeenDays(
- SortedMap<Integer, Set<String>> relaysByLastSeenDays) {
- this.relaysByLastSeenDays = relaysByLastSeenDays;
- }
- public SortedMap<Integer, Set<String>> getRelaysByLastSeenDays() {
- return relaysByLastSeenDays;
- }
-
- private SortedMap<Integer, Set<String>> bridgesByLastSeenDays;
- public void setBridgesByLastSeenDays(
- SortedMap<Integer, Set<String>> bridgesByLastSeenDays) {
- this.bridgesByLastSeenDays = bridgesByLastSeenDays;
- }
- public SortedMap<Integer, Set<String>> getBridgesByLastSeenDays() {
- return bridgesByLastSeenDays;
- }
-}
-
public class NodeIndexer implements ServletContextListener, Runnable {
public void contextInitialized(ServletContextEvent contextEvent) {
diff --git a/src/org/torproject/onionoo/server/ResourceServlet.java b/src/org/torproject/onionoo/server/ResourceServlet.java
index 1e05d12..6f01448 100644
--- a/src/org/torproject/onionoo/server/ResourceServlet.java
+++ b/src/org/torproject/onionoo/server/ResourceServlet.java
@@ -43,45 +43,6 @@ public class ResourceServlet extends HttpServlet {
}
}
- public static class HttpServletRequestWrapper {
- private HttpServletRequest request;
- protected HttpServletRequestWrapper(HttpServletRequest request) {
- this.request = request;
- }
- protected String getRequestURI() {
- return this.request.getRequestURI();
- }
- @SuppressWarnings("rawtypes")
- protected Map getParameterMap() {
- return this.request.getParameterMap();
- }
- protected String[] getParameterValues(String parameterKey) {
- return this.request.getParameterValues(parameterKey);
- }
- }
-
- public static class HttpServletResponseWrapper {
- private HttpServletResponse response = null;
- protected HttpServletResponseWrapper(HttpServletResponse response) {
- this.response = response;
- }
- protected void sendError(int errorStatusCode) throws IOException {
- this.response.sendError(errorStatusCode);
- }
- protected void setHeader(String headerName, String headerValue) {
- this.response.setHeader(headerName, headerValue);
- }
- protected void setContentType(String contentType) {
- this.response.setContentType(contentType);
- }
- protected void setCharacterEncoding(String characterEncoding) {
- this.response.setCharacterEncoding(characterEncoding);
- }
- protected PrintWriter getWriter() throws IOException {
- return this.response.getWriter();
- }
- }
-
public void doGet(HttpServletRequest request,
HttpServletResponse response) throws IOException, ServletException {
HttpServletRequestWrapper requestWrapper =
diff --git a/src/org/torproject/onionoo/updater/DescriptorDownloader.java b/src/org/torproject/onionoo/updater/DescriptorDownloader.java
new file mode 100644
index 0000000..60d8d45
--- /dev/null
+++ b/src/org/torproject/onionoo/updater/DescriptorDownloader.java
@@ -0,0 +1,178 @@
+package org.torproject.onionoo.updater;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.zip.GZIPInputStream;
+
+class DescriptorDownloader {
+
+ private final String protocolHostNameResourcePrefix =
+ "https://collector.torproject.org/recent/";;
+
+ private String directory;
+
+ private final File inDir = new File("in/recent");
+
+ public DescriptorDownloader(DescriptorType descriptorType) {
+ switch (descriptorType) {
+ case RELAY_CONSENSUSES:
+ this.directory = "relay-descriptors/consensuses/";
+ break;
+ case RELAY_SERVER_DESCRIPTORS:
+ this.directory = "relay-descriptors/server-descriptors/";
+ break;
+ case RELAY_EXTRA_INFOS:
+ this.directory = "relay-descriptors/extra-infos/";
+ break;
+ case EXIT_LISTS:
+ this.directory = "exit-lists/";
+ break;
+ case BRIDGE_STATUSES:
+ this.directory = "bridge-descriptors/statuses/";
+ break;
+ case BRIDGE_SERVER_DESCRIPTORS:
+ this.directory = "bridge-descriptors/server-descriptors/";
+ break;
+ case BRIDGE_EXTRA_INFOS:
+ this.directory = "bridge-descriptors/extra-infos/";
+ break;
+ case BRIDGE_POOL_ASSIGNMENTS:
+ this.directory = "bridge-pool-assignments/";
+ break;
+ default:
+ System.err.println("Unknown descriptor type.");
+ return;
+ }
+ }
+
+ private SortedSet<String> localFiles = new TreeSet<String>();
+
+ public int statLocalFiles() {
+ File localDirectory = new File(this.inDir, this.directory);
+ if (localDirectory.exists()) {
+ for (File file : localDirectory.listFiles()) {
+ this.localFiles.add(file.getName());
+ }
+ }
+ return this.localFiles.size();
+ }
+
+ private SortedSet<String> remoteFiles = new TreeSet<String>();
+
+ public int fetchRemoteDirectory() {
+ String directoryUrl = this.protocolHostNameResourcePrefix
+ + this.directory;
+ try {
+ URL u = new URL(directoryUrl);
+ HttpURLConnection huc = (HttpURLConnection) u.openConnection();
+ huc.setRequestMethod("GET");
+ huc.connect();
+ if (huc.getResponseCode() != 200) {
+ System.err.println("Could not fetch " + directoryUrl
+ + ": " + huc.getResponseCode() + " "
+ + huc.getResponseMessage() + ". Skipping.");
+ return 0;
+ }
+ BufferedReader br = new BufferedReader(new InputStreamReader(
+ huc.getInputStream()));
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (!line.trim().startsWith("<tr>") ||
+ !line.contains("<a href=\"")) {
+ continue;
+ }
+ String linePart = line.substring(
+ line.indexOf("<a href=\"") + "<a href=\"".length());
+ if (!linePart.contains("\"")) {
+ continue;
+ }
+ linePart = linePart.substring(0, linePart.indexOf("\""));
+ if (linePart.endsWith("/")) {
+ continue;
+ }
+ this.remoteFiles.add(linePart);
+ }
+ br.close();
+ } catch (IOException e) {
+ System.err.println("Could not fetch or parse " + directoryUrl
+ + ". Skipping.");
+ }
+ return this.remoteFiles.size();
+ }
+
+ public int fetchRemoteFiles() {
+ int fetchedFiles = 0;
+ for (String remoteFile : this.remoteFiles) {
+ if (this.localFiles.contains(remoteFile)) {
+ continue;
+ }
+ String fileUrl = this.protocolHostNameResourcePrefix
+ + this.directory + remoteFile;
+ File localTempFile = new File(this.inDir, this.directory
+ + remoteFile + ".tmp");
+ File localFile = new File(this.inDir, this.directory + remoteFile);
+ try {
+ localFile.getParentFile().mkdirs();
+ URL u = new URL(fileUrl);
+ HttpURLConnection huc = (HttpURLConnection) u.openConnection();
+ huc.setRequestMethod("GET");
+ huc.addRequestProperty("Accept-Encoding", "gzip");
+ huc.connect();
+ if (huc.getResponseCode() != 200) {
+ System.err.println("Could not fetch " + fileUrl
+ + ": " + huc.getResponseCode() + " "
+ + huc.getResponseMessage() + ". Skipping.");
+ continue;
+ }
+ long lastModified = huc.getHeaderFieldDate("Last-Modified", -1L);
+ InputStream is;
+ if (huc.getContentEncoding() != null &&
+ huc.getContentEncoding().equalsIgnoreCase("gzip")) {
+ is = new GZIPInputStream(huc.getInputStream());
+ } else {
+ is = huc.getInputStream();
+ }
+ BufferedInputStream bis = new BufferedInputStream(is);
+ BufferedOutputStream bos = new BufferedOutputStream(
+ new FileOutputStream(localTempFile));
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = bis.read(data, 0, 1024)) >= 0) {
+ bos.write(data, 0, len);
+ }
+ bis.close();
+ bos.close();
+ localTempFile.renameTo(localFile);
+ if (lastModified >= 0) {
+ localFile.setLastModified(lastModified);
+ }
+ fetchedFiles++;
+ } catch (IOException e) {
+ System.err.println("Could not fetch or store " + fileUrl
+ + ". Skipping.");
+ }
+ }
+ return fetchedFiles;
+ }
+
+ public int deleteOldLocalFiles() {
+ int deletedFiles = 0;
+ for (String localFile : this.localFiles) {
+ if (!this.remoteFiles.contains(localFile)) {
+ new File(this.inDir, this.directory + localFile).delete();
+ deletedFiles++;
+ }
+ }
+ return deletedFiles;
+ }
+}
\ No newline at end of file
diff --git a/src/org/torproject/onionoo/updater/DescriptorHistory.java b/src/org/torproject/onionoo/updater/DescriptorHistory.java
new file mode 100644
index 0000000..0f6f578
--- /dev/null
+++ b/src/org/torproject/onionoo/updater/DescriptorHistory.java
@@ -0,0 +1,12 @@
+package org.torproject.onionoo.updater;
+
+enum DescriptorHistory {
+ RELAY_CONSENSUS_HISTORY,
+ RELAY_SERVER_HISTORY,
+ RELAY_EXTRAINFO_HISTORY,
+ EXIT_LIST_HISTORY,
+ BRIDGE_STATUS_HISTORY,
+ BRIDGE_SERVER_HISTORY,
+ BRIDGE_EXTRAINFO_HISTORY,
+ BRIDGE_POOLASSIGN_HISTORY,
+}
\ No newline at end of file
diff --git a/src/org/torproject/onionoo/updater/DescriptorQueue.java b/src/org/torproject/onionoo/updater/DescriptorQueue.java
new file mode 100644
index 0000000..96362c5
--- /dev/null
+++ b/src/org/torproject/onionoo/updater/DescriptorQueue.java
@@ -0,0 +1,221 @@
+package org.torproject.onionoo.updater;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.DescriptorFile;
+import org.torproject.descriptor.DescriptorReader;
+import org.torproject.descriptor.DescriptorSourceFactory;
+
+class DescriptorQueue {
+
+ private File inDir;
+
+ private File statusDir;
+
+ private DescriptorReader descriptorReader;
+
+ private File historyFile;
+
+ private Iterator<DescriptorFile> descriptorFiles;
+
+ private List<Descriptor> descriptors;
+
+ private int historySizeBefore;
+ public int getHistorySizeBefore() {
+ return this.historySizeBefore;
+ }
+
+ private int historySizeAfter;
+ public int getHistorySizeAfter() {
+ return this.historySizeAfter;
+ }
+
+ private long returnedDescriptors = 0L;
+ public long getReturnedDescriptors() {
+ return this.returnedDescriptors;
+ }
+
+ private long returnedBytes = 0L;
+ public long getReturnedBytes() {
+ return this.returnedBytes;
+ }
+
+ public DescriptorQueue(File inDir, File statusDir) {
+ this.inDir = inDir;
+ this.statusDir = statusDir;
+ this.descriptorReader =
+ DescriptorSourceFactory.createDescriptorReader();
+ }
+
+ public void addDirectory(DescriptorType descriptorType) {
+ String directoryName = null;
+ switch (descriptorType) {
+ case RELAY_CONSENSUSES:
+ directoryName = "relay-descriptors/consensuses";
+ break;
+ case RELAY_SERVER_DESCRIPTORS:
+ directoryName = "relay-descriptors/server-descriptors";
+ break;
+ case RELAY_EXTRA_INFOS:
+ directoryName = "relay-descriptors/extra-infos";
+ break;
+ case BRIDGE_STATUSES:
+ directoryName = "bridge-descriptors/statuses";
+ break;
+ case BRIDGE_SERVER_DESCRIPTORS:
+ directoryName = "bridge-descriptors/server-descriptors";
+ break;
+ case BRIDGE_EXTRA_INFOS:
+ directoryName = "bridge-descriptors/extra-infos";
+ break;
+ case BRIDGE_POOL_ASSIGNMENTS:
+ directoryName = "bridge-pool-assignments";
+ break;
+ case EXIT_LISTS:
+ directoryName = "exit-lists";
+ break;
+ default:
+ System.err.println("Unknown descriptor type. Not adding directory "
+ + "to descriptor reader.");
+ return;
+ }
+ File directory = new File(this.inDir, directoryName);
+ if (directory.exists() && directory.isDirectory()) {
+ this.descriptorReader.addDirectory(directory);
+ this.descriptorReader.setMaxDescriptorFilesInQueue(1);
+ } else {
+ System.err.println("Directory " + directory.getAbsolutePath()
+ + " either does not exist or is not a directory. Not adding "
+ + "to descriptor reader.");
+ }
+ }
+
+ public void readHistoryFile(DescriptorHistory descriptorHistory) {
+ String historyFileName = null;
+ switch (descriptorHistory) {
+ case RELAY_EXTRAINFO_HISTORY:
+ historyFileName = "relay-extrainfo-history";
+ break;
+ case BRIDGE_EXTRAINFO_HISTORY:
+ historyFileName = "bridge-extrainfo-history";
+ break;
+ case EXIT_LIST_HISTORY:
+ historyFileName = "exit-list-history";
+ break;
+ case BRIDGE_POOLASSIGN_HISTORY:
+ historyFileName = "bridge-poolassign-history";
+ break;
+ case RELAY_CONSENSUS_HISTORY:
+ historyFileName = "relay-consensus-history";
+ break;
+ case BRIDGE_STATUS_HISTORY:
+ historyFileName = "bridge-status-history";
+ break;
+ case RELAY_SERVER_HISTORY:
+ historyFileName = "relay-server-history";
+ break;
+ case BRIDGE_SERVER_HISTORY:
+ historyFileName = "bridge-server-history";
+ break;
+ default:
+ System.err.println("Unknown descriptor history. Not excluding "
+ + "files.");
+ return;
+ }
+ this.historyFile = new File(this.statusDir, historyFileName);
+ if (this.historyFile.exists() && this.historyFile.isFile()) {
+ SortedMap<String, Long> excludedFiles = new TreeMap<String, Long>();
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(
+ this.historyFile));
+ String line;
+ while ((line = br.readLine()) != null) {
+ try {
+ String[] parts = line.split(" ", 2);
+ excludedFiles.put(parts[1], Long.parseLong(parts[0]));
+ } catch (NumberFormatException e) {
+ System.err.println("Illegal line '" + line + "' in parse "
+ + "history. Skipping line.");
+ }
+ }
+ br.close();
+ } catch (IOException e) {
+ System.err.println("Could not read history file '"
+ + this.historyFile.getAbsolutePath() + "'. Not excluding "
+ + "descriptors in this execution.");
+ e.printStackTrace();
+ return;
+ }
+ this.historySizeBefore = excludedFiles.size();
+ this.descriptorReader.setExcludedFiles(excludedFiles);
+ }
+ }
+
+ public void writeHistoryFile() {
+ if (this.historyFile == null) {
+ return;
+ }
+ SortedMap<String, Long> excludedAndParsedFiles =
+ new TreeMap<String, Long>();
+ excludedAndParsedFiles.putAll(
+ this.descriptorReader.getExcludedFiles());
+ excludedAndParsedFiles.putAll(this.descriptorReader.getParsedFiles());
+ this.historySizeAfter = excludedAndParsedFiles.size();
+ try {
+ this.historyFile.getParentFile().mkdirs();
+ BufferedWriter bw = new BufferedWriter(new FileWriter(
+ this.historyFile));
+ for (Map.Entry<String, Long> e : excludedAndParsedFiles.entrySet()) {
+ String absolutePath = e.getKey();
+ long lastModifiedMillis = e.getValue();
+ bw.write(String.valueOf(lastModifiedMillis) + " " + absolutePath
+ + "\n");
+ }
+ bw.close();
+ } catch (IOException e) {
+ System.err.println("Could not write history file '"
+ + this.historyFile.getAbsolutePath() + "'. Not excluding "
+ + "descriptors in next execution.");
+ return;
+ }
+ }
+
+ public Descriptor nextDescriptor() {
+ Descriptor nextDescriptor = null;
+ if (this.descriptorFiles == null) {
+ this.descriptorFiles = this.descriptorReader.readDescriptors();
+ }
+ while (this.descriptors == null && this.descriptorFiles.hasNext()) {
+ DescriptorFile descriptorFile = this.descriptorFiles.next();
+ if (descriptorFile.getException() != null) {
+ System.err.println("Could not parse "
+ + descriptorFile.getFileName());
+ descriptorFile.getException().printStackTrace();
+ }
+ if (descriptorFile.getDescriptors() != null &&
+ !descriptorFile.getDescriptors().isEmpty()) {
+ this.descriptors = descriptorFile.getDescriptors();
+ }
+ }
+ if (this.descriptors != null) {
+ nextDescriptor = this.descriptors.remove(0);
+ this.returnedDescriptors++;
+ this.returnedBytes += nextDescriptor.getRawDescriptorBytes().length;
+ if (this.descriptors.isEmpty()) {
+ this.descriptors = null;
+ }
+ }
+ return nextDescriptor;
+ }
+}
\ No newline at end of file
diff --git a/src/org/torproject/onionoo/updater/DescriptorSource.java b/src/org/torproject/onionoo/updater/DescriptorSource.java
index 32fbd2a..ea1474f 100644
--- a/src/org/torproject/onionoo/updater/DescriptorSource.java
+++ b/src/org/torproject/onionoo/updater/DescriptorSource.java
@@ -2,38 +2,19 @@
* See LICENSE for licensing information */
package org.torproject.onionoo.updater;
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileOutputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.HttpURLConnection;
-import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.SortedMap;
import java.util.SortedSet;
-import java.util.TreeMap;
import java.util.TreeSet;
-import java.util.zip.GZIPInputStream;
import org.torproject.descriptor.BridgeNetworkStatus;
import org.torproject.descriptor.BridgePoolAssignment;
import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
-import org.torproject.descriptor.DescriptorReader;
-import org.torproject.descriptor.DescriptorSourceFactory;
import org.torproject.descriptor.ExitList;
import org.torproject.descriptor.ExitListEntry;
import org.torproject.descriptor.ExtraInfoDescriptor;
@@ -41,383 +22,6 @@ import org.torproject.descriptor.RelayNetworkStatusConsensus;
import org.torproject.descriptor.ServerDescriptor;
import org.torproject.onionoo.util.Logger;
-enum DescriptorHistory {
- RELAY_CONSENSUS_HISTORY,
- RELAY_SERVER_HISTORY,
- RELAY_EXTRAINFO_HISTORY,
- EXIT_LIST_HISTORY,
- BRIDGE_STATUS_HISTORY,
- BRIDGE_SERVER_HISTORY,
- BRIDGE_EXTRAINFO_HISTORY,
- BRIDGE_POOLASSIGN_HISTORY,
-}
-
-class DescriptorDownloader {
-
- private final String protocolHostNameResourcePrefix =
- "https://collector.torproject.org/recent/";;
-
- private String directory;
-
- private final File inDir = new File("in/recent");
-
- public DescriptorDownloader(DescriptorType descriptorType) {
- switch (descriptorType) {
- case RELAY_CONSENSUSES:
- this.directory = "relay-descriptors/consensuses/";
- break;
- case RELAY_SERVER_DESCRIPTORS:
- this.directory = "relay-descriptors/server-descriptors/";
- break;
- case RELAY_EXTRA_INFOS:
- this.directory = "relay-descriptors/extra-infos/";
- break;
- case EXIT_LISTS:
- this.directory = "exit-lists/";
- break;
- case BRIDGE_STATUSES:
- this.directory = "bridge-descriptors/statuses/";
- break;
- case BRIDGE_SERVER_DESCRIPTORS:
- this.directory = "bridge-descriptors/server-descriptors/";
- break;
- case BRIDGE_EXTRA_INFOS:
- this.directory = "bridge-descriptors/extra-infos/";
- break;
- case BRIDGE_POOL_ASSIGNMENTS:
- this.directory = "bridge-pool-assignments/";
- break;
- default:
- System.err.println("Unknown descriptor type.");
- return;
- }
- }
-
- private SortedSet<String> localFiles = new TreeSet<String>();
-
- public int statLocalFiles() {
- File localDirectory = new File(this.inDir, this.directory);
- if (localDirectory.exists()) {
- for (File file : localDirectory.listFiles()) {
- this.localFiles.add(file.getName());
- }
- }
- return this.localFiles.size();
- }
-
- private SortedSet<String> remoteFiles = new TreeSet<String>();
-
- public int fetchRemoteDirectory() {
- String directoryUrl = this.protocolHostNameResourcePrefix
- + this.directory;
- try {
- URL u = new URL(directoryUrl);
- HttpURLConnection huc = (HttpURLConnection) u.openConnection();
- huc.setRequestMethod("GET");
- huc.connect();
- if (huc.getResponseCode() != 200) {
- System.err.println("Could not fetch " + directoryUrl
- + ": " + huc.getResponseCode() + " "
- + huc.getResponseMessage() + ". Skipping.");
- return 0;
- }
- BufferedReader br = new BufferedReader(new InputStreamReader(
- huc.getInputStream()));
- String line;
- while ((line = br.readLine()) != null) {
- if (!line.trim().startsWith("<tr>") ||
- !line.contains("<a href=\"")) {
- continue;
- }
- String linePart = line.substring(
- line.indexOf("<a href=\"") + "<a href=\"".length());
- if (!linePart.contains("\"")) {
- continue;
- }
- linePart = linePart.substring(0, linePart.indexOf("\""));
- if (linePart.endsWith("/")) {
- continue;
- }
- this.remoteFiles.add(linePart);
- }
- br.close();
- } catch (IOException e) {
- System.err.println("Could not fetch or parse " + directoryUrl
- + ". Skipping.");
- }
- return this.remoteFiles.size();
- }
-
- public int fetchRemoteFiles() {
- int fetchedFiles = 0;
- for (String remoteFile : this.remoteFiles) {
- if (this.localFiles.contains(remoteFile)) {
- continue;
- }
- String fileUrl = this.protocolHostNameResourcePrefix
- + this.directory + remoteFile;
- File localTempFile = new File(this.inDir, this.directory
- + remoteFile + ".tmp");
- File localFile = new File(this.inDir, this.directory + remoteFile);
- try {
- localFile.getParentFile().mkdirs();
- URL u = new URL(fileUrl);
- HttpURLConnection huc = (HttpURLConnection) u.openConnection();
- huc.setRequestMethod("GET");
- huc.addRequestProperty("Accept-Encoding", "gzip");
- huc.connect();
- if (huc.getResponseCode() != 200) {
- System.err.println("Could not fetch " + fileUrl
- + ": " + huc.getResponseCode() + " "
- + huc.getResponseMessage() + ". Skipping.");
- continue;
- }
- long lastModified = huc.getHeaderFieldDate("Last-Modified", -1L);
- InputStream is;
- if (huc.getContentEncoding() != null &&
- huc.getContentEncoding().equalsIgnoreCase("gzip")) {
- is = new GZIPInputStream(huc.getInputStream());
- } else {
- is = huc.getInputStream();
- }
- BufferedInputStream bis = new BufferedInputStream(is);
- BufferedOutputStream bos = new BufferedOutputStream(
- new FileOutputStream(localTempFile));
- int len;
- byte[] data = new byte[1024];
- while ((len = bis.read(data, 0, 1024)) >= 0) {
- bos.write(data, 0, len);
- }
- bis.close();
- bos.close();
- localTempFile.renameTo(localFile);
- if (lastModified >= 0) {
- localFile.setLastModified(lastModified);
- }
- fetchedFiles++;
- } catch (IOException e) {
- System.err.println("Could not fetch or store " + fileUrl
- + ". Skipping.");
- }
- }
- return fetchedFiles;
- }
-
- public int deleteOldLocalFiles() {
- int deletedFiles = 0;
- for (String localFile : this.localFiles) {
- if (!this.remoteFiles.contains(localFile)) {
- new File(this.inDir, this.directory + localFile).delete();
- deletedFiles++;
- }
- }
- return deletedFiles;
- }
-}
-
-class DescriptorQueue {
-
- private File inDir;
-
- private File statusDir;
-
- private DescriptorReader descriptorReader;
-
- private File historyFile;
-
- private Iterator<DescriptorFile> descriptorFiles;
-
- private List<Descriptor> descriptors;
-
- private int historySizeBefore;
- public int getHistorySizeBefore() {
- return this.historySizeBefore;
- }
-
- private int historySizeAfter;
- public int getHistorySizeAfter() {
- return this.historySizeAfter;
- }
-
- private long returnedDescriptors = 0L;
- public long getReturnedDescriptors() {
- return this.returnedDescriptors;
- }
-
- private long returnedBytes = 0L;
- public long getReturnedBytes() {
- return this.returnedBytes;
- }
-
- public DescriptorQueue(File inDir, File statusDir) {
- this.inDir = inDir;
- this.statusDir = statusDir;
- this.descriptorReader =
- DescriptorSourceFactory.createDescriptorReader();
- }
-
- public void addDirectory(DescriptorType descriptorType) {
- String directoryName = null;
- switch (descriptorType) {
- case RELAY_CONSENSUSES:
- directoryName = "relay-descriptors/consensuses";
- break;
- case RELAY_SERVER_DESCRIPTORS:
- directoryName = "relay-descriptors/server-descriptors";
- break;
- case RELAY_EXTRA_INFOS:
- directoryName = "relay-descriptors/extra-infos";
- break;
- case BRIDGE_STATUSES:
- directoryName = "bridge-descriptors/statuses";
- break;
- case BRIDGE_SERVER_DESCRIPTORS:
- directoryName = "bridge-descriptors/server-descriptors";
- break;
- case BRIDGE_EXTRA_INFOS:
- directoryName = "bridge-descriptors/extra-infos";
- break;
- case BRIDGE_POOL_ASSIGNMENTS:
- directoryName = "bridge-pool-assignments";
- break;
- case EXIT_LISTS:
- directoryName = "exit-lists";
- break;
- default:
- System.err.println("Unknown descriptor type. Not adding directory "
- + "to descriptor reader.");
- return;
- }
- File directory = new File(this.inDir, directoryName);
- if (directory.exists() && directory.isDirectory()) {
- this.descriptorReader.addDirectory(directory);
- this.descriptorReader.setMaxDescriptorFilesInQueue(1);
- } else {
- System.err.println("Directory " + directory.getAbsolutePath()
- + " either does not exist or is not a directory. Not adding "
- + "to descriptor reader.");
- }
- }
-
- public void readHistoryFile(DescriptorHistory descriptorHistory) {
- String historyFileName = null;
- switch (descriptorHistory) {
- case RELAY_EXTRAINFO_HISTORY:
- historyFileName = "relay-extrainfo-history";
- break;
- case BRIDGE_EXTRAINFO_HISTORY:
- historyFileName = "bridge-extrainfo-history";
- break;
- case EXIT_LIST_HISTORY:
- historyFileName = "exit-list-history";
- break;
- case BRIDGE_POOLASSIGN_HISTORY:
- historyFileName = "bridge-poolassign-history";
- break;
- case RELAY_CONSENSUS_HISTORY:
- historyFileName = "relay-consensus-history";
- break;
- case BRIDGE_STATUS_HISTORY:
- historyFileName = "bridge-status-history";
- break;
- case RELAY_SERVER_HISTORY:
- historyFileName = "relay-server-history";
- break;
- case BRIDGE_SERVER_HISTORY:
- historyFileName = "bridge-server-history";
- break;
- default:
- System.err.println("Unknown descriptor history. Not excluding "
- + "files.");
- return;
- }
- this.historyFile = new File(this.statusDir, historyFileName);
- if (this.historyFile.exists() && this.historyFile.isFile()) {
- SortedMap<String, Long> excludedFiles = new TreeMap<String, Long>();
- try {
- BufferedReader br = new BufferedReader(new FileReader(
- this.historyFile));
- String line;
- while ((line = br.readLine()) != null) {
- try {
- String[] parts = line.split(" ", 2);
- excludedFiles.put(parts[1], Long.parseLong(parts[0]));
- } catch (NumberFormatException e) {
- System.err.println("Illegal line '" + line + "' in parse "
- + "history. Skipping line.");
- }
- }
- br.close();
- } catch (IOException e) {
- System.err.println("Could not read history file '"
- + this.historyFile.getAbsolutePath() + "'. Not excluding "
- + "descriptors in this execution.");
- e.printStackTrace();
- return;
- }
- this.historySizeBefore = excludedFiles.size();
- this.descriptorReader.setExcludedFiles(excludedFiles);
- }
- }
-
- public void writeHistoryFile() {
- if (this.historyFile == null) {
- return;
- }
- SortedMap<String, Long> excludedAndParsedFiles =
- new TreeMap<String, Long>();
- excludedAndParsedFiles.putAll(
- this.descriptorReader.getExcludedFiles());
- excludedAndParsedFiles.putAll(this.descriptorReader.getParsedFiles());
- this.historySizeAfter = excludedAndParsedFiles.size();
- try {
- this.historyFile.getParentFile().mkdirs();
- BufferedWriter bw = new BufferedWriter(new FileWriter(
- this.historyFile));
- for (Map.Entry<String, Long> e : excludedAndParsedFiles.entrySet()) {
- String absolutePath = e.getKey();
- long lastModifiedMillis = e.getValue();
- bw.write(String.valueOf(lastModifiedMillis) + " " + absolutePath
- + "\n");
- }
- bw.close();
- } catch (IOException e) {
- System.err.println("Could not write history file '"
- + this.historyFile.getAbsolutePath() + "'. Not excluding "
- + "descriptors in next execution.");
- return;
- }
- }
-
- public Descriptor nextDescriptor() {
- Descriptor nextDescriptor = null;
- if (this.descriptorFiles == null) {
- this.descriptorFiles = this.descriptorReader.readDescriptors();
- }
- while (this.descriptors == null && this.descriptorFiles.hasNext()) {
- DescriptorFile descriptorFile = this.descriptorFiles.next();
- if (descriptorFile.getException() != null) {
- System.err.println("Could not parse "
- + descriptorFile.getFileName());
- descriptorFile.getException().printStackTrace();
- }
- if (descriptorFile.getDescriptors() != null &&
- !descriptorFile.getDescriptors().isEmpty()) {
- this.descriptors = descriptorFile.getDescriptors();
- }
- }
- if (this.descriptors != null) {
- nextDescriptor = this.descriptors.remove(0);
- this.returnedDescriptors++;
- this.returnedBytes += nextDescriptor.getRawDescriptorBytes().length;
- if (this.descriptors.isEmpty()) {
- this.descriptors = null;
- }
- }
- return nextDescriptor;
- }
-}
-
public class DescriptorSource {
private final File inDir = new File("in/recent");
diff --git a/src/org/torproject/onionoo/updater/RdnsLookupRequest.java b/src/org/torproject/onionoo/updater/RdnsLookupRequest.java
new file mode 100644
index 0000000..4a06d20
--- /dev/null
+++ b/src/org/torproject/onionoo/updater/RdnsLookupRequest.java
@@ -0,0 +1,43 @@
+/* Copyright 2013 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.updater;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+class RdnsLookupRequest extends Thread {
+
+ private final ReverseDomainNameResolver reverseDomainNameResolver;
+ private RdnsLookupWorker parent;
+ private String address, hostName;
+ private long lookupStartedMillis = -1L, lookupCompletedMillis = -1L;
+ public RdnsLookupRequest(
+ ReverseDomainNameResolver reverseDomainNameResolver,
+ RdnsLookupWorker parent, String address) {
+ this.reverseDomainNameResolver = reverseDomainNameResolver;
+ this.parent = parent;
+ this.address = address;
+ }
+ public void run() {
+ this.lookupStartedMillis =
+ this.reverseDomainNameResolver.time.currentTimeMillis();
+ try {
+ String result = InetAddress.getByName(this.address).getHostName();
+ synchronized (this) {
+ this.hostName = result;
+ }
+ } catch (UnknownHostException e) {
+ /* We'll try again the next time. */
+ }
+ this.lookupCompletedMillis =
+ this.reverseDomainNameResolver.time.currentTimeMillis();
+ this.parent.interrupt();
+ }
+ public synchronized String getHostName() {
+ return hostName;
+ }
+ public synchronized long getLookupMillis() {
+ return this.lookupCompletedMillis - this.lookupStartedMillis;
+ }
+}
+
diff --git a/src/org/torproject/onionoo/updater/RdnsLookupWorker.java b/src/org/torproject/onionoo/updater/RdnsLookupWorker.java
new file mode 100644
index 0000000..cf7d580
--- /dev/null
+++ b/src/org/torproject/onionoo/updater/RdnsLookupWorker.java
@@ -0,0 +1,55 @@
+/* Copyright 2013 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo.updater;
+
+class RdnsLookupWorker extends Thread {
+
+ private final ReverseDomainNameResolver reverseDomainNameResolver;
+
+ RdnsLookupWorker(ReverseDomainNameResolver reverseDomainNameResolver) {
+ this.reverseDomainNameResolver = reverseDomainNameResolver;
+ }
+
+ public void run() {
+ while (this.reverseDomainNameResolver.time.currentTimeMillis() -
+ ReverseDomainNameResolver.RDNS_LOOKUP_MAX_DURATION_MILLIS
+ <= this.reverseDomainNameResolver.startedRdnsLookups) {
+ String rdnsLookupJob = null;
+ synchronized (this.reverseDomainNameResolver.rdnsLookupJobs) {
+ for (String job : this.reverseDomainNameResolver.rdnsLookupJobs) {
+ rdnsLookupJob = job;
+ this.reverseDomainNameResolver.rdnsLookupJobs.remove(job);
+ break;
+ }
+ }
+ if (rdnsLookupJob == null) {
+ break;
+ }
+ RdnsLookupRequest request = new RdnsLookupRequest(
+ this.reverseDomainNameResolver, this, rdnsLookupJob);
+ request.setDaemon(true);
+ request.start();
+ try {
+ Thread.sleep(
+ ReverseDomainNameResolver.RDNS_LOOKUP_MAX_REQUEST_MILLIS);
+ } catch (InterruptedException e) {
+ /* Getting interrupted should be the default case. */
+ }
+ String hostName = request.getHostName();
+ if (hostName != null) {
+ synchronized (this.reverseDomainNameResolver.rdnsLookupResults) {
+ this.reverseDomainNameResolver.rdnsLookupResults.put(
+ rdnsLookupJob, hostName);
+ }
+ }
+ long lookupMillis = request.getLookupMillis();
+ if (lookupMillis >= 0L) {
+ synchronized (this.reverseDomainNameResolver.rdnsLookupMillis) {
+ this.reverseDomainNameResolver.rdnsLookupMillis.add(
+ lookupMillis);
+ }
+ }
+ }
+ }
+}
+
diff --git a/src/org/torproject/onionoo/updater/ReverseDomainNameResolver.java b/src/org/torproject/onionoo/updater/ReverseDomainNameResolver.java
index 8ca7eb4..8694155 100644
--- a/src/org/torproject/onionoo/updater/ReverseDomainNameResolver.java
+++ b/src/org/torproject/onionoo/updater/ReverseDomainNameResolver.java
@@ -2,8 +2,6 @@
* See LICENSE for licensing information */
package org.torproject.onionoo.updater;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -19,84 +17,15 @@ import org.torproject.onionoo.util.Time;
public class ReverseDomainNameResolver {
- private class RdnsLookupWorker extends Thread {
- public void run() {
- while (time.currentTimeMillis() - RDNS_LOOKUP_MAX_DURATION_MILLIS
- <= startedRdnsLookups) {
- String rdnsLookupJob = null;
- synchronized (rdnsLookupJobs) {
- for (String job : rdnsLookupJobs) {
- rdnsLookupJob = job;
- rdnsLookupJobs.remove(job);
- break;
- }
- }
- if (rdnsLookupJob == null) {
- break;
- }
- RdnsLookupRequest request = new RdnsLookupRequest(this,
- rdnsLookupJob);
- request.setDaemon(true);
- request.start();
- try {
- Thread.sleep(RDNS_LOOKUP_MAX_REQUEST_MILLIS);
- } catch (InterruptedException e) {
- /* Getting interrupted should be the default case. */
- }
- String hostName = request.getHostName();
- if (hostName != null) {
- synchronized (rdnsLookupResults) {
- rdnsLookupResults.put(rdnsLookupJob, hostName);
- }
- }
- long lookupMillis = request.getLookupMillis();
- if (lookupMillis >= 0L) {
- synchronized (rdnsLookupMillis) {
- rdnsLookupMillis.add(lookupMillis);
- }
- }
- }
- }
- }
-
- private class RdnsLookupRequest extends Thread {
- private RdnsLookupWorker parent;
- private String address, hostName;
- private long lookupStartedMillis = -1L, lookupCompletedMillis = -1L;
- public RdnsLookupRequest(RdnsLookupWorker parent, String address) {
- this.parent = parent;
- this.address = address;
- }
- public void run() {
- this.lookupStartedMillis = time.currentTimeMillis();
- try {
- String result = InetAddress.getByName(this.address).getHostName();
- synchronized (this) {
- this.hostName = result;
- }
- } catch (UnknownHostException e) {
- /* We'll try again the next time. */
- }
- this.lookupCompletedMillis = time.currentTimeMillis();
- this.parent.interrupt();
- }
- public synchronized String getHostName() {
- return hostName;
- }
- public synchronized long getLookupMillis() {
- return this.lookupCompletedMillis - this.lookupStartedMillis;
- }
- }
-
- private Time time;
+ Time time;
public ReverseDomainNameResolver() {
this.time = ApplicationFactory.getTime();
}
- private static final long RDNS_LOOKUP_MAX_REQUEST_MILLIS =
+ static final long RDNS_LOOKUP_MAX_REQUEST_MILLIS =
DateTimeHelper.TEN_SECONDS;
- private static final long RDNS_LOOKUP_MAX_DURATION_MILLIS =
+ static final long RDNS_LOOKUP_MAX_DURATION_MILLIS =
DateTimeHelper.FIVE_MINUTES;
private static final long RDNS_LOOKUP_MAX_AGE_MILLIS =
DateTimeHelper.TWELVE_HOURS;
@@ -104,13 +33,13 @@ public class ReverseDomainNameResolver {
private Map<String, Long> addressLastLookupTimes;
- private Set<String> rdnsLookupJobs;
+ Set<String> rdnsLookupJobs;
- private Map<String, String> rdnsLookupResults;
+ Map<String, String> rdnsLookupResults;
- private List<Long> rdnsLookupMillis;
+ List<Long> rdnsLookupMillis;
- private long startedRdnsLookups;
+ long startedRdnsLookups;
private List<RdnsLookupWorker> rdnsLookupWorkers;
@@ -132,7 +61,7 @@ public class ReverseDomainNameResolver {
this.rdnsLookupMillis = new ArrayList<Long>();
this.rdnsLookupWorkers = new ArrayList<RdnsLookupWorker>();
for (int i = 0; i < RDNS_LOOKUP_WORKERS_NUM; i++) {
- RdnsLookupWorker rdnsLookupWorker = new RdnsLookupWorker();
+ RdnsLookupWorker rdnsLookupWorker = new RdnsLookupWorker(this);
this.rdnsLookupWorkers.add(rdnsLookupWorker);
rdnsLookupWorker.setDaemon(true);
rdnsLookupWorker.start();
diff --git a/test/org/torproject/onionoo/ResourceServletTest.java b/test/org/torproject/onionoo/ResourceServletTest.java
index 8cd3752..d27f499 100644
--- a/test/org/torproject/onionoo/ResourceServletTest.java
+++ b/test/org/torproject/onionoo/ResourceServletTest.java
@@ -22,10 +22,10 @@ import java.util.TreeSet;
import org.junit.Before;
import org.junit.Test;
import org.torproject.onionoo.docs.UpdateStatus;
+import org.torproject.onionoo.server.HttpServletRequestWrapper;
+import org.torproject.onionoo.server.HttpServletResponseWrapper;
import org.torproject.onionoo.server.NodeIndexer;
import org.torproject.onionoo.server.ResourceServlet;
-import org.torproject.onionoo.server.ResourceServlet.HttpServletRequestWrapper;
-import org.torproject.onionoo.server.ResourceServlet.HttpServletResponseWrapper;
import org.torproject.onionoo.util.ApplicationFactory;
import org.torproject.onionoo.util.DateTimeHelper;
import org.torproject.onionoo.util.Time;
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits