[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [doctor/master] Use the new metrics-lib for downloading consensuses and votes.
commit 0c96c1a6410dd93545692de70f66be2357f0dcbd
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Mon Dec 12 11:29:03 2011 +0100
Use the new metrics-lib for downloading consensuses and votes.
Don't use the lib for parsing consensuses and votes. That part is not
implemented yet.
---
build.xml | 1 +
src/org/torproject/doctor/Downloader.java | 261 ++++++-----------------------
2 files changed, 52 insertions(+), 210 deletions(-)
diff --git a/build.xml b/build.xml
index 2db0db4..feedb12 100755
--- a/build.xml
+++ b/build.xml
@@ -5,6 +5,7 @@
<path id="classpath">
<pathelement path="${classes}"/>
<pathelement location="${libs}/commons-codec-1.4.jar"/>
+ <pathelement location="${libs}/descriptor.jar"/>
</path>
<target name="init">
<mkdir dir="${classes}"/>
diff --git a/src/org/torproject/doctor/Downloader.java b/src/org/torproject/doctor/Downloader.java
index a2e6db5..a82ac8d 100755
--- a/src/org/torproject/doctor/Downloader.java
+++ b/src/org/torproject/doctor/Downloader.java
@@ -7,241 +7,82 @@ import java.net.*;
import java.text.*;
import java.util.*;
import java.util.zip.*;
+import org.torproject.descriptor.*;
/* Download the latest network status consensus and corresponding
* votes. */
public class Downloader {
- /* List of directory authorities to download consensuses and votes
- * from. */
- private SortedMap<String, String> authorities =
- new TreeMap<String, String>();
- public Downloader() {
- this.authorities.put("gabelmoo", "212.112.245.170");
- this.authorities.put("tor26", "86.59.21.38");
- this.authorities.put("ides", "216.224.124.114:9030");
- this.authorities.put("maatuska", "213.115.239.118:443");
- this.authorities.put("dannenberg", "193.23.244.244");
- this.authorities.put("urras", "208.83.223.34:443");
- this.authorities.put("moria1", "128.31.0.34:9131");
- this.authorities.put("dizum", "194.109.206.212");
- }
-
/* Download a new consensus and corresponding votes. */
public void downloadFromAuthorities() {
- this.downloadConsensus();
- if (!this.downloadedConsensuses.isEmpty()) {
- this.parseConsensusToFindReferencedVotes();
- this.downloadReferencedVotes();
- }
- }
- /* Download the most recent consensus from all authorities. */
- private List<Download> downloadedConsensuses =
- new ArrayList<Download>();
- private void downloadConsensus() {
- Map<String, String> urls = new HashMap<String, String>();
- for (Map.Entry<String, String> e : this.authorities.entrySet()) {
- String nickname = e.getKey();
- String address = e.getValue();
- String resource = "/tor/status-vote/current/consensus.z";
- String fullUrl = "http://"; + address + resource;
- urls.put(fullUrl, nickname);
- }
- this.downloadedConsensuses = this.downloadFromAuthority(urls);
- if (this.downloadedConsensuses.isEmpty()) {
- System.err.println("Could not download consensus from any of the "
- + "directory authorities. Ignoring.");
- }
- }
+ RelayDescriptorDownloader downloader =
+ DescriptorSourceFactory.createRelayDescriptorDownloader();
- /* Downloads a consensus or vote in a separate thread that can be
- * interrupted after a timeout. */
- private static class DownloadRunnable implements Runnable {
- Thread mainThread;
- String nickname;
- String url;
- String response;
- boolean finished = false;
- long requestStart = 0L, requestEnd = 0L;
- public DownloadRunnable(String nickname, String url) {
- this.mainThread = Thread.currentThread();
- this.nickname = nickname;
- this.url = url;
- }
- public void run() {
- this.requestStart = System.currentTimeMillis();
- try {
- URL u = new URL(this.url);
- HttpURLConnection huc = (HttpURLConnection) u.openConnection();
- huc.setRequestMethod("GET");
- huc.connect();
- int responseCode = huc.getResponseCode();
- if (responseCode == 200) {
- BufferedInputStream in = new BufferedInputStream(
- new InflaterInputStream(huc.getInputStream()));
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- int len;
- byte[] data = new byte[1024];
- while (!this.finished &&
- (len = in.read(data, 0, 1024)) >= 0) {
- baos.write(data, 0, len);
- }
- if (this.finished) {
- return;
- }
- in.close();
- byte[] allData = baos.toByteArray();
- this.response = new String(allData);
- this.finished = true;
- this.mainThread.interrupt();
- }
- } catch (IOException e) {
- /* Can't do much except leaving this.response at null. */
- }
- this.finished = true;
- this.requestEnd = System.currentTimeMillis();
- }
- public Download getDownload() {
- Download result = null;
- if (this.response != null) {
- result = new Download(this.nickname, this.url, this.response,
- this.requestStart, this.requestEnd - this.requestStart);
- }
- return result;
- }
- }
+ downloader.addDirectoryAuthority("gabelmoo", "212.112.245.170", 80);
+ downloader.addDirectoryAuthority("tor26", "86.59.21.38", 80);
+ downloader.addDirectoryAuthority("ides", "216.224.124.114", 9030);
+ downloader.addDirectoryAuthority("maatuska", "213.115.239.118", 443);
+ downloader.addDirectoryAuthority("dannenberg", "193.23.244.244", 80);
+ downloader.addDirectoryAuthority("urras", "208.83.223.34", 443);
+ downloader.addDirectoryAuthority("moria1", "128.31.0.34", 9131);
+ downloader.addDirectoryAuthority("dizum", "194.109.206.212", 80);
- /* Download one or more consensuses or votes from one or more directory
- * authorities using a timeout of 60 seconds. */
- private List<Download> downloadFromAuthority(Map<String, String> urls) {
- Set<DownloadRunnable> downloadRunnables =
- new HashSet<DownloadRunnable>();
- for (Map.Entry<String, String> e : urls.entrySet()) {
- String url = e.getKey();
- String nickname = e.getValue();
- DownloadRunnable downloadRunnable = new DownloadRunnable(nickname,
- url);
- downloadRunnables.add(downloadRunnable);
- new Thread(downloadRunnable).start();
- }
- long started = System.currentTimeMillis(), sleep;
- while ((sleep = started + 60L * 1000L - System.currentTimeMillis())
- > 0L) {
- try {
- Thread.sleep(sleep);
- } catch (InterruptedException e) {
- /* Do nothing. */
- }
- boolean unfinished = false;
- for (DownloadRunnable downloadRunnable : downloadRunnables) {
- if (!downloadRunnable.finished) {
- unfinished = true;
- break;
- }
- }
- if (!unfinished) {
- break;
- }
- }
- List<Download> responses = new ArrayList<Download>();
- for (DownloadRunnable downloadRunnable : downloadRunnables) {
- Download download = downloadRunnable.getDownload();
- if (download != null) {
- responses.add(download);
- }
- downloadRunnable.finished = true;
- }
- return responses;
- }
+ downloader.setIncludeCurrentConsensusFromAllDirectoryAuthorities();
+ downloader.setIncludeCurrentReferencedVotes();
- /* Date-time formats to parse and format timestamps. */
- private static SimpleDateFormat dateTimeFormat;
- static {
- dateTimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- }
+ downloader.setRequestTimeout(60L * 1000L);
- /* Parse the downloaded consensus to find fingerprints of directory
- * authorities publishing the corresponding votes. */
- private SortedSet<String> fingerprints = new TreeSet<String>();
- private void parseConsensusToFindReferencedVotes() {
- for (Download download : this.downloadedConsensuses) {
- String downloadedConsensus = download.getResponseString();
- try {
- BufferedReader br = new BufferedReader(new StringReader(
- downloadedConsensus));
- String line;
- while ((line = br.readLine()) != null) {
- if (line.startsWith("valid-after ")) {
- try {
- long validAfterMillis = dateTimeFormat.parse(line.substring(
- "valid-after ".length())).getTime();
- if (validAfterMillis + 60L * 60L * 1000L <
- System.currentTimeMillis()) {
- /* Consensus is more than 1 hour old. We won't be able to
- * download the corresponding votes anymore. */
- break;
- }
- } catch (ParseException e) {
- System.err.println("Could not parse valid-after timestamp "
- + "in line '" + line + "' of a downloaded consensus. "
- + "Not downloading votes.");
- break;
- }
- } else if (line.startsWith("dir-source ")) {
- String[] parts = line.split(" ");
- if (parts.length < 3) {
- System.err.println("Bad dir-source line '" + line
- + "' in downloaded consensus. Skipping.");
- continue;
- }
- String nickname = parts[1];
- if (nickname.endsWith("-legacy")) {
- continue;
- }
- String fingerprint = parts[2];
- this.fingerprints.add(fingerprint);
- }
+ Iterator<DescriptorRequest> descriptorRequests =
+ downloader.downloadDescriptors();
+ while (descriptorRequests.hasNext()) {
+ DescriptorRequest request = descriptorRequests.next();
+ String authority = request.getDirectoryNickname();
+ String requestUrl = request.getRequestUrl();
+ long requestStart = request.getRequestStart();
+ long fetchTime = request.getRequestEnd()
+ - request.getRequestStart();
+ if (request.globalTimeoutHasExpired()) {
+ System.err.println("Global timeout has expired. Exiting.");
+ System.exit(1);
+ } else if (!request.requestTimeoutHasExpired()) {
+ if (request.getDescriptors().isEmpty()) {
+ /* No response. We'll realize later on if we're missing a
+ * consensus or vote. */
+ continue;
+ } else if (request.getDescriptors().size() > 1) {
+ System.out.println("Response contains more than 1 "
+ + "descriptor. Considering only the first.");
}
- br.close();
- } catch (IOException e) {
- System.err.println("Could not parse consensus to find referenced "
- + "votes in it. Skipping.");
- }
- }
- }
-
- /* Download the votes published by directory authorities listed in the
- * consensus. */
- private List<Download> downloadedVotes = new ArrayList<Download>();
- private void downloadReferencedVotes() {
- for (String fingerprint : this.fingerprints) {
- String downloadedVote = null;
- List<String> authorities = new ArrayList<String>(
- this.authorities.values());
- Collections.shuffle(authorities);
- for (String authority : authorities) {
- if (downloadedVote != null) {
- break;
+ Descriptor downloadedDescriptor = request.getDescriptors().get(0);
+ String response = new String(request.getDescriptors().get(0).
+ getRawDescriptorBytes());
+ Download download = new Download(authority, requestUrl, response,
+ requestStart, fetchTime);
+ if (downloadedDescriptor instanceof
+ RelayNetworkStatusConsensus) {
+ this.downloadedConsensuses.add(download);
+ } else if (downloadedDescriptor instanceof
+ RelayNetworkStatusVote) {
+ this.downloadedVotes.add(download);
+ } else {
+ System.err.println("Did not expect a descriptor of type "
+ + downloadedDescriptor.getClass() + ". Ignoring.");
}
- String resource = "/tor/status-vote/current/" + fingerprint
- + ".z";
- String fullUrl = "http://"; + authority + resource;
- Map<String, String> urls = new HashMap<String, String>();
- urls.put(fullUrl, authority);
- this.downloadedVotes.addAll(this.downloadFromAuthority(urls));
}
}
}
/* Return the previously downloaded (unparsed) consensus string by
* authority nickname. */
+ private List<Download> downloadedConsensuses =
+ new ArrayList<Download>();
public List<Download> getConsensuses() {
return this.downloadedConsensuses;
}
/* Return the previously downloaded (unparsed) vote strings. */
+ private List<Download> downloadedVotes = new ArrayList<Download>();
public List<Download> getVotes() {
return this.downloadedVotes;
}
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits