[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [onionoo/master] Add new document with fractional uptimes.
commit 7c911367930d6aa4183090c13f8c2a9de4842f56
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Mon Mar 10 14:24:48 2014 +0100
Add new document with fractional uptimes.
Prepares for providing monthly relay and bridge statistics (#11041).
---
etc/web.xml.template | 4 +
src/org/torproject/onionoo/DocumentStore.java | 31 +-
src/org/torproject/onionoo/Main.java | 4 +-
src/org/torproject/onionoo/ResourceServlet.java | 2 +
src/org/torproject/onionoo/ResponseBuilder.java | 28 ++
src/org/torproject/onionoo/UptimeDataWriter.java | 533 ++++++++++++++++++++++
src/org/torproject/onionoo/UptimeDocument.java | 8 +
src/org/torproject/onionoo/UptimeStatus.java | 5 +
web/index.html | 192 ++++++++
9 files changed, 800 insertions(+), 7 deletions(-)
diff --git a/etc/web.xml.template b/etc/web.xml.template
index 53a1878..f69314f 100644
--- a/etc/web.xml.template
+++ b/etc/web.xml.template
@@ -40,6 +40,10 @@
<servlet-name>Resource</servlet-name>
<url-pattern>/clients</url-pattern>
</servlet-mapping>
+ <servlet-mapping>
+ <servlet-name>Resource</servlet-name>
+ <url-pattern>/uptime</url-pattern>
+ </servlet-mapping>
</web-app>
diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java
index 1092e25..519b4f2 100644
--- a/src/org/torproject/onionoo/DocumentStore.java
+++ b/src/org/torproject/onionoo/DocumentStore.java
@@ -124,6 +124,9 @@ public class DocumentStore {
} else if (documentType.equals(ClientsStatus.class)) {
directory = this.statusDir;
subdirectory = "clients";
+ } else if (documentType.equals(UptimeStatus.class)) {
+ directory = this.statusDir;
+ subdirectory = "uptimes";
} else if (documentType.equals(DetailsDocument.class)) {
directory = this.outDir;
subdirectory = "details";
@@ -136,6 +139,9 @@ public class DocumentStore {
} else if (documentType.equals(ClientsDocument.class)) {
directory = this.outDir;
subdirectory = "clients";
+ } else if (documentType.equals(UptimeDocument.class)) {
+ directory = this.outDir;
+ subdirectory = "uptimes";
}
if (directory != null && subdirectory != null) {
Stack<File> files = new Stack<File>();
@@ -186,7 +192,8 @@ public class DocumentStore {
} else if (document instanceof DetailsDocument ||
document instanceof BandwidthDocument ||
document instanceof WeightsDocument ||
- document instanceof ClientsDocument) {
+ document instanceof ClientsDocument ||
+ document instanceof UptimeDocument) {
Gson gson = new Gson();
documentString = gson.toJson(this);
} else {
@@ -275,7 +282,8 @@ public class DocumentStore {
} else if (documentType.equals(DetailsDocument.class) ||
documentType.equals(BandwidthDocument.class) ||
documentType.equals(WeightsDocument.class) ||
- documentType.equals(ClientsDocument.class)) {
+ documentType.equals(ClientsDocument.class) ||
+ documentType.equals(UptimeDocument.class)) {
return this.retrieveParsedDocumentFile(documentType,
documentString);
} else {
@@ -351,12 +359,11 @@ public class DocumentStore {
private <T extends Document> File getDocumentFile(Class<T> documentType,
String fingerprint) {
File documentFile = null;
- if (fingerprint == null &&
- !documentType.equals(UpdateStatus.class)) {
+ if (fingerprint == null && !documentType.equals(UpdateStatus.class) &&
+ !documentType.equals(UptimeStatus.class)) {
// TODO Instead of using the update file workaround, add new method
// lastModified(Class<T> documentType) that serves a similar
- // purpose. Once that's implemented, make fingerprint mandatory for
- // all methods.
+ // purpose.
return null;
}
File directory = null;
@@ -381,6 +388,15 @@ public class DocumentStore {
fileName = String.format("clients/%s/%s/%s",
fingerprint.substring(0, 1), fingerprint.substring(1, 2),
fingerprint);
+ } else if (documentType.equals(UptimeStatus.class)) {
+ directory = this.statusDir;
+ if (fingerprint == null) {
+ fileName = "uptime";
+ } else {
+ fileName = String.format("uptimes/%s/%s/%s",
+ fingerprint.substring(0, 1), fingerprint.substring(1, 2),
+ fingerprint);
+ }
} else if (documentType.equals(UpdateStatus.class)) {
directory = this.outDir;
fileName = "update";
@@ -401,6 +417,9 @@ public class DocumentStore {
} else if (documentType.equals(ClientsDocument.class)) {
directory = this.outDir;
fileName = String.format("clients/%s", fingerprint);
+ } else if (documentType.equals(UptimeDocument.class)) {
+ directory = this.outDir;
+ fileName = String.format("uptimes/%s", fingerprint);
}
if (directory != null && fileName != null) {
documentFile = new File(directory, fileName);
diff --git a/src/org/torproject/onionoo/Main.java b/src/org/torproject/onionoo/Main.java
index 87e7c9a..5631a5e 100644
--- a/src/org/torproject/onionoo/Main.java
+++ b/src/org/torproject/onionoo/Main.java
@@ -38,7 +38,9 @@ public class Main {
Logger.printStatusTime("Initialized weights data writer");
ClientsDataWriter cdw = new ClientsDataWriter(dso, ds, t);
Logger.printStatusTime("Initialized clients data writer");
- DataWriter[] dws = new DataWriter[] { ndw, bdw, wdw, cdw };
+ UptimeDataWriter udw = new UptimeDataWriter(dso, ds, t);
+ Logger.printStatusTime("Initialized uptime data writer");
+ DataWriter[] dws = new DataWriter[] { ndw, bdw, wdw, cdw, udw };
Logger.printStatus("Reading descriptors.");
dso.readRelayNetworkConsensuses();
diff --git a/src/org/torproject/onionoo/ResourceServlet.java b/src/org/torproject/onionoo/ResourceServlet.java
index d04828f..de47d44 100644
--- a/src/org/torproject/onionoo/ResourceServlet.java
+++ b/src/org/torproject/onionoo/ResourceServlet.java
@@ -130,6 +130,8 @@ public class ResourceServlet extends HttpServlet {
resourceType = "weights";
} else if (uri.startsWith("/clients")) {
resourceType = "clients";
+ } else if (uri.startsWith("/uptime")) {
+ resourceType = "uptime";
} else {
response.sendError(HttpServletResponse.SC_BAD_REQUEST);
return;
diff --git a/src/org/torproject/onionoo/ResponseBuilder.java b/src/org/torproject/onionoo/ResponseBuilder.java
index b7231db..b0775b1 100644
--- a/src/org/torproject/onionoo/ResponseBuilder.java
+++ b/src/org/torproject/onionoo/ResponseBuilder.java
@@ -776,6 +776,8 @@ public class ResponseBuilder {
return this.writeWeightsLines(summaryLine);
} else if (this.resourceType.equals("clients")) {
return this.writeClientsLines(summaryLine);
+ } else if (this.resourceType.equals("uptime")) {
+ return this.writeUptimeLines(summaryLine);
} else {
return "";
}
@@ -928,4 +930,30 @@ public class ResponseBuilder {
return "";
}
}
+
+ private String writeUptimeLines(String summaryLine) {
+ String fingerprint = null;
+ if (summaryLine.contains("\"f\":\"")) {
+ fingerprint = summaryLine.substring(summaryLine.indexOf(
+ "\"f\":\"") + "\"f\":\"".length());
+ } else if (summaryLine.contains("\"h\":\"")) {
+ fingerprint = summaryLine.substring(summaryLine.indexOf(
+ "\"h\":\"") + "\"h\":\"".length());
+ } else {
+ return "";
+ }
+ fingerprint = fingerprint.substring(0, 40);
+ UptimeDocument uptimeDocument = documentStore.retrieve(
+ UptimeDocument.class, false, fingerprint);
+ if (uptimeDocument != null &&
+ uptimeDocument.documentString != null) {
+ String uptimeLines = uptimeDocument.documentString;
+ uptimeLines = uptimeLines.substring(0, uptimeLines.length() - 1);
+ return uptimeLines;
+ } else {
+ // TODO We should probably log that we didn't find an uptime
+ // document that we expected to exist.
+ return "";
+ }
+ }
}
diff --git a/src/org/torproject/onionoo/UptimeDataWriter.java b/src/org/torproject/onionoo/UptimeDataWriter.java
new file mode 100644
index 0000000..7a8b3af
--- /dev/null
+++ b/src/org/torproject/onionoo/UptimeDataWriter.java
@@ -0,0 +1,533 @@
+/* Copyright 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Scanner;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TimeZone;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.torproject.descriptor.BridgeNetworkStatus;
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.NetworkStatusEntry;
+import org.torproject.descriptor.RelayNetworkStatusConsensus;
+
+public class UptimeDataWriter implements DataWriter, DescriptorListener {
+
+ private DescriptorSource descriptorSource;
+
+ private DocumentStore documentStore;
+
+ private long now;
+
+ public UptimeDataWriter(DescriptorSource descriptorSource,
+ DocumentStore documentStore, Time time) {
+ this.descriptorSource = descriptorSource;
+ this.documentStore = documentStore;
+ this.now = time.currentTimeMillis();
+ this.registerDescriptorListeners();
+ }
+
+ private void registerDescriptorListeners() {
+ this.descriptorSource.registerListener(this,
+ DescriptorType.RELAY_CONSENSUSES);
+ this.descriptorSource.registerListener(this,
+ DescriptorType.BRIDGE_STATUSES);
+ }
+
+ public void processDescriptor(Descriptor descriptor, boolean relay) {
+ if (descriptor instanceof RelayNetworkStatusConsensus) {
+ this.processRelayNetworkStatusConsensus(
+ (RelayNetworkStatusConsensus) descriptor);
+ } else if (descriptor instanceof BridgeNetworkStatus) {
+ this.processBridgeNetworkStatus(
+ (BridgeNetworkStatus) descriptor);
+ }
+ }
+
+ private SortedSet<Long> newRelayStatuses = new TreeSet<Long>(),
+ newBridgeStatuses = new TreeSet<Long>();
+ private SortedMap<String, SortedSet<Long>>
+ newRunningRelays = new TreeMap<String, SortedSet<Long>>(),
+ newRunningBridges = new TreeMap<String, SortedSet<Long>>();
+
+ private static final long ONE_HOUR_MILLIS = 60L * 60L * 1000L;
+
+ private void processRelayNetworkStatusConsensus(
+ RelayNetworkStatusConsensus consensus) {
+ SortedSet<String> fingerprints = new TreeSet<String>();
+ for (NetworkStatusEntry entry :
+ consensus.getStatusEntries().values()) {
+ if (entry.getFlags().contains("Running")) {
+ fingerprints.add(entry.getFingerprint());
+ }
+ }
+ if (!fingerprints.isEmpty()) {
+ long dateHourMillis = (consensus.getValidAfterMillis()
+ / ONE_HOUR_MILLIS) * ONE_HOUR_MILLIS;
+ for (String fingerprint : fingerprints) {
+ if (!this.newRunningRelays.containsKey(fingerprint)) {
+ this.newRunningRelays.put(fingerprint, new TreeSet<Long>());
+ }
+ this.newRunningRelays.get(fingerprint).add(dateHourMillis);
+ }
+ this.newRelayStatuses.add(dateHourMillis);
+ }
+ }
+
+ private void processBridgeNetworkStatus(BridgeNetworkStatus status) {
+ SortedSet<String> fingerprints = new TreeSet<String>();
+ for (NetworkStatusEntry entry :
+ status.getStatusEntries().values()) {
+ if (entry.getFlags().contains("Running")) {
+ fingerprints.add(entry.getFingerprint());
+ }
+ }
+ if (!fingerprints.isEmpty()) {
+ long dateHourMillis = (status.getPublishedMillis()
+ / ONE_HOUR_MILLIS) * ONE_HOUR_MILLIS;
+ for (String fingerprint : fingerprints) {
+ if (!this.newRunningBridges.containsKey(fingerprint)) {
+ this.newRunningBridges.put(fingerprint, new TreeSet<Long>());
+ }
+ this.newRunningBridges.get(fingerprint).add(dateHourMillis);
+ }
+ this.newBridgeStatuses.add(dateHourMillis);
+ }
+ }
+
+ public void updateStatuses() {
+ for (Map.Entry<String, SortedSet<Long>> e :
+ this.newRunningRelays.entrySet()) {
+ this.updateStatus(true, e.getKey(), e.getValue());
+ }
+ this.updateStatus(true, null, this.newRelayStatuses);
+ for (Map.Entry<String, SortedSet<Long>> e :
+ this.newRunningBridges.entrySet()) {
+ this.updateStatus(false, e.getKey(), e.getValue());
+ }
+ this.updateStatus(false, null, this.newBridgeStatuses);
+ Logger.printStatusTime("Updated uptime status files");
+ }
+
+ private static class UptimeHistory
+ implements Comparable<UptimeHistory> {
+ private boolean relay;
+ private long startMillis;
+ private int uptimeHours;
+ private UptimeHistory(boolean relay, long startMillis,
+ int uptimeHours) {
+ this.relay = relay;
+ this.startMillis = startMillis;
+ this.uptimeHours = uptimeHours;
+ }
+ public static UptimeHistory fromString(String uptimeHistoryString) {
+ String[] parts = uptimeHistoryString.split(" ", 3);
+ if (parts.length != 3) {
+ return null;
+ }
+ boolean relay = false;
+ if (parts[0].equals("r")) {
+ relay = true;
+ } else if (!parts[0].equals("b")) {
+ return null;
+ }
+ long startMillis = -1L;
+ SimpleDateFormat dateHourFormat = new SimpleDateFormat(
+ "yyyy-MM-dd-HH");
+ dateHourFormat.setLenient(false);
+ dateHourFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ try {
+ startMillis = dateHourFormat.parse(parts[1]).getTime();
+ } catch (ParseException e) {
+ return null;
+ }
+ int uptimeHours = -1;
+ try {
+ uptimeHours = Integer.parseInt(parts[2]);
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ return new UptimeHistory(relay, startMillis, uptimeHours);
+ }
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ SimpleDateFormat dateHourFormat = new SimpleDateFormat(
+ "yyyy-MM-dd-HH");
+ dateHourFormat.setLenient(false);
+ dateHourFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ sb.append(this.relay ? "r" : "b");
+ sb.append(" " + dateHourFormat.format(this.startMillis));
+ sb.append(" " + String.format("%d", this.uptimeHours));
+ return sb.toString();
+ }
+ public void addUptime(UptimeHistory other) {
+ this.uptimeHours += other.uptimeHours;
+ if (this.startMillis > other.startMillis) {
+ this.startMillis = other.startMillis;
+ }
+ }
+ public int compareTo(UptimeHistory other) {
+ if (this.relay && !other.relay) {
+ return -1;
+ } else if (!this.relay && other.relay) {
+ return 1;
+ }
+ return this.startMillis < other.startMillis ? -1 :
+ this.startMillis > other.startMillis ? 1 : 0;
+ }
+ public boolean equals(Object other) {
+ return other instanceof UptimeHistory &&
+ this.relay == ((UptimeHistory) other).relay &&
+ this.startMillis == ((UptimeHistory) other).startMillis;
+ }
+ }
+
+ private void updateStatus(boolean relay, String fingerprint,
+ SortedSet<Long> newUptimeHours) {
+ SortedSet<UptimeHistory> history = this.readHistory(fingerprint);
+ this.addToHistory(history, relay, newUptimeHours);
+ history = this.compressHistory(history);
+ this.writeHistory(fingerprint, history);
+ }
+
+ private SortedSet<UptimeHistory> readHistory(String fingerprint) {
+ SortedSet<UptimeHistory> history = new TreeSet<UptimeHistory>();
+ UptimeStatus uptimeStatus = fingerprint == null ?
+ documentStore.retrieve(UptimeStatus.class, false) :
+ documentStore.retrieve(UptimeStatus.class, false, fingerprint);
+ if (uptimeStatus != null) {
+ Scanner s = new Scanner(uptimeStatus.documentString);
+ while (s.hasNextLine()) {
+ String line = s.nextLine();
+ UptimeHistory parsedLine = UptimeHistory.fromString(line);
+ if (parsedLine != null) {
+ history.add(parsedLine);
+ } else {
+ System.err.println("Could not parse uptime history line '"
+ + line + "' for fingerprint '" + fingerprint
+ + "'. Skipping.");
+ }
+ }
+ s.close();
+ }
+ return history;
+ }
+
+ private void addToHistory(SortedSet<UptimeHistory> history,
+ boolean relay, SortedSet<Long> newIntervals) {
+ for (long startMillis : newIntervals) {
+ UptimeHistory interval = new UptimeHistory(relay, startMillis, 1);
+ if (!history.headSet(interval).isEmpty()) {
+ UptimeHistory prev = history.headSet(interval).last();
+ if (prev.relay == interval.relay &&
+ prev.startMillis + ONE_HOUR_MILLIS * prev.uptimeHours >
+ interval.startMillis) {
+ continue;
+ }
+ }
+ if (!history.tailSet(interval).isEmpty()) {
+ UptimeHistory next = history.tailSet(interval).first();
+ if (next.relay == interval.relay &&
+ next.startMillis < interval.startMillis + ONE_HOUR_MILLIS) {
+ continue;
+ }
+ }
+ history.add(interval);
+ }
+ }
+
+ private SortedSet<UptimeHistory> compressHistory(
+ SortedSet<UptimeHistory> history) {
+ SortedSet<UptimeHistory> compressedHistory =
+ new TreeSet<UptimeHistory>();
+ UptimeHistory lastInterval = null;
+ for (UptimeHistory interval : history) {
+ if (lastInterval != null &&
+ lastInterval.startMillis + ONE_HOUR_MILLIS
+ * lastInterval.uptimeHours == interval.startMillis &&
+ lastInterval.relay == interval.relay) {
+ lastInterval.addUptime(interval);
+ } else {
+ if (lastInterval != null) {
+ compressedHistory.add(lastInterval);
+ }
+ lastInterval = interval;
+ }
+ }
+ if (lastInterval != null) {
+ compressedHistory.add(lastInterval);
+ }
+ return compressedHistory;
+ }
+
+ private void writeHistory(String fingerprint,
+ SortedSet<UptimeHistory> history) {
+ StringBuilder sb = new StringBuilder();
+ for (UptimeHistory interval : history) {
+ sb.append(interval.toString() + "\n");
+ }
+ UptimeStatus uptimeStatus = new UptimeStatus();
+ uptimeStatus.documentString = sb.toString();
+ if (fingerprint == null) {
+ this.documentStore.store(uptimeStatus);
+ } else {
+ this.documentStore.store(uptimeStatus, fingerprint);
+ }
+ }
+
+ public void updateDocuments() {
+ SortedSet<UptimeHistory>
+ knownRelayStatuses = new TreeSet<UptimeHistory>(),
+ knownBridgeStatuses = new TreeSet<UptimeHistory>();
+ SortedSet<UptimeHistory> knownStatuses = this.readHistory(null);
+ for (UptimeHistory status : knownStatuses) {
+ if (status.relay) {
+ knownRelayStatuses.add(status);
+ } else {
+ knownBridgeStatuses.add(status);
+ }
+ }
+ for (String fingerprint : this.newRunningRelays.keySet()) {
+ this.updateDocument(true, fingerprint, knownRelayStatuses);
+ }
+ for (String fingerprint : this.newRunningBridges.keySet()) {
+ this.updateDocument(false, fingerprint, knownBridgeStatuses);
+ }
+ Logger.printStatusTime("Wrote uptime document files");
+ }
+
+ private void updateDocument(boolean relay, String fingerprint,
+ SortedSet<UptimeHistory> knownStatuses) {
+ SortedSet<UptimeHistory> history = this.readHistory(fingerprint);
+ UptimeDocument uptimeDocument = new UptimeDocument();
+ uptimeDocument.documentString = this.formatHistoryString(relay,
+ fingerprint, history, knownStatuses);
+ this.documentStore.store(uptimeDocument, fingerprint);
+ }
+
+ private String[] graphNames = new String[] {
+ "1_week",
+ "1_month",
+ "3_months",
+ "1_year",
+ "5_years" };
+
+ private long[] graphIntervals = new long[] {
+ 7L * 24L * 60L * 60L * 1000L,
+ 31L * 24L * 60L * 60L * 1000L,
+ 92L * 24L * 60L * 60L * 1000L,
+ 366L * 24L * 60L * 60L * 1000L,
+ 5L * 366L * 24L * 60L * 60L * 1000L };
+
+ private long[] dataPointIntervals = new long[] {
+ 60L * 60L * 1000L,
+ 4L * 60L * 60L * 1000L,
+ 12L * 60L * 60L * 1000L,
+ 2L * 24L * 60L * 60L * 1000L,
+ 10L * 24L * 60L * 60L * 1000L };
+
+ private String formatHistoryString(boolean relay, String fingerprint,
+ SortedSet<UptimeHistory> history,
+ SortedSet<UptimeHistory> knownStatuses) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("{\"fingerprint\":\"" + fingerprint + "\"");
+ sb.append(",\n\"uptime\":{");
+ int graphIntervalsWritten = 0;
+ for (int graphIntervalIndex = 0; graphIntervalIndex <
+ this.graphIntervals.length; graphIntervalIndex++) {
+ String timeline = this.formatTimeline(graphIntervalIndex, relay,
+ history, knownStatuses);
+ if (timeline != null) {
+ sb.append((graphIntervalsWritten++ > 0 ? "," : "") + "\n"
+ + timeline);
+ }
+ }
+ sb.append("}");
+ sb.append("\n}\n");
+ return sb.toString();
+ }
+
+ private String formatTimeline(int graphIntervalIndex, boolean relay,
+ SortedSet<UptimeHistory> history,
+ SortedSet<UptimeHistory> knownStatuses) {
+ String graphName = this.graphNames[graphIntervalIndex];
+ long graphInterval = this.graphIntervals[graphIntervalIndex];
+ long dataPointInterval =
+ this.dataPointIntervals[graphIntervalIndex];
+ int dataPointIntervalHours = (int) (dataPointInterval
+ / ONE_HOUR_MILLIS);
+ List<Integer> statusDataPoints = new ArrayList<Integer>();
+ long intervalStartMillis = ((this.now - graphInterval)
+ / dataPointInterval) * dataPointInterval;
+ int statusHours = 0;
+ for (UptimeHistory hist : knownStatuses) {
+ if (hist.relay != relay) {
+ continue;
+ }
+ long histEndMillis = hist.startMillis + ONE_HOUR_MILLIS
+ * hist.uptimeHours;
+ if (histEndMillis < intervalStartMillis) {
+ continue;
+ }
+ while (hist.startMillis >= intervalStartMillis
+ + dataPointInterval) {
+ statusDataPoints.add(statusHours * 5 > dataPointIntervalHours
+ ? statusHours : -1);
+ statusHours = 0;
+ intervalStartMillis += dataPointInterval;
+ }
+ while (histEndMillis >= intervalStartMillis + dataPointInterval) {
+ statusHours += (int) ((intervalStartMillis + dataPointInterval
+ - Math.max(hist.startMillis, intervalStartMillis))
+ / ONE_HOUR_MILLIS);
+ statusDataPoints.add(statusHours * 5 > dataPointIntervalHours
+ ? statusHours : -1);
+ statusHours = 0;
+ intervalStartMillis += dataPointInterval;
+ }
+ statusHours += (int) ((histEndMillis - Math.max(hist.startMillis,
+ intervalStartMillis)) / ONE_HOUR_MILLIS);
+ }
+ statusDataPoints.add(statusHours * 5 > dataPointIntervalHours
+ ? statusHours : -1);
+ List<Integer> uptimeDataPoints = new ArrayList<Integer>();
+ intervalStartMillis = ((this.now - graphInterval)
+ / dataPointInterval) * dataPointInterval;
+ int uptimeHours = 0;
+ long firstStatusStartMillis = -1L;
+ for (UptimeHistory hist : history) {
+ if (hist.relay != relay) {
+ continue;
+ }
+ if (firstStatusStartMillis < 0L) {
+ firstStatusStartMillis = hist.startMillis;
+ }
+ long histEndMillis = hist.startMillis + ONE_HOUR_MILLIS
+ * hist.uptimeHours;
+ if (histEndMillis < intervalStartMillis) {
+ continue;
+ }
+ while (hist.startMillis >= intervalStartMillis
+ + dataPointInterval) {
+ if (firstStatusStartMillis < intervalStartMillis
+ + dataPointInterval) {
+ uptimeDataPoints.add(uptimeHours);
+ } else {
+ uptimeDataPoints.add(-1);
+ }
+ uptimeHours = 0;
+ intervalStartMillis += dataPointInterval;
+ }
+ while (histEndMillis >= intervalStartMillis + dataPointInterval) {
+ uptimeHours += (int) ((intervalStartMillis + dataPointInterval
+ - Math.max(hist.startMillis, intervalStartMillis))
+ / ONE_HOUR_MILLIS);
+ uptimeDataPoints.add(uptimeHours);
+ uptimeHours = 0;
+ intervalStartMillis += dataPointInterval;
+ }
+ uptimeHours += (int) ((histEndMillis - Math.max(hist.startMillis,
+ intervalStartMillis)) / ONE_HOUR_MILLIS);
+ }
+ uptimeDataPoints.add(uptimeHours);
+ List<Double> dataPoints = new ArrayList<Double>();
+ for (int dataPointIndex = 0; dataPointIndex < statusDataPoints.size();
+ dataPointIndex++) {
+ if (dataPointIndex >= uptimeDataPoints.size()) {
+ dataPoints.add(0.0);
+ } else if (uptimeDataPoints.get(dataPointIndex) >= 0 &&
+ statusDataPoints.get(dataPointIndex) > 0) {
+ dataPoints.add(((double) uptimeDataPoints.get(dataPointIndex))
+ / ((double) statusDataPoints.get(dataPointIndex)));
+ } else {
+ dataPoints.add(-1.0);
+ }
+ }
+ int firstNonNullIndex = -1, lastNonNullIndex = -1;
+ for (int dataPointIndex = 0; dataPointIndex < dataPoints.size();
+ dataPointIndex++) {
+ double dataPoint = dataPoints.get(dataPointIndex);
+ if (dataPoint >= 0.0) {
+ if (firstNonNullIndex < 0) {
+ firstNonNullIndex = dataPointIndex;
+ }
+ lastNonNullIndex = dataPointIndex;
+ }
+ }
+ if (firstNonNullIndex < 0) {
+ return null;
+ }
+ long firstDataPointMillis = (((this.now - graphInterval)
+ / dataPointInterval) + firstNonNullIndex)
+ * dataPointInterval + dataPointInterval / 2L;
+ if (graphIntervalIndex > 0 && firstDataPointMillis >=
+ this.now - graphIntervals[graphIntervalIndex - 1]) {
+ /* Skip uptime history object, because it doesn't contain
+ * anything new that wasn't already contained in the last
+ * uptime history object(s). */
+ return null;
+ }
+ long lastDataPointMillis = firstDataPointMillis
+ + (lastNonNullIndex - firstNonNullIndex) * dataPointInterval;
+ double factor = 1.0 / 999.0;
+ int count = lastNonNullIndex - firstNonNullIndex + 1;
+ StringBuilder sb = new StringBuilder();
+ SimpleDateFormat dateTimeFormat = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+ dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ sb.append("\"" + graphName + "\":{"
+ + "\"first\":\"" + dateTimeFormat.format(firstDataPointMillis)
+ + "\",\"last\":\"" + dateTimeFormat.format(lastDataPointMillis)
+ + "\",\"interval\":" + String.valueOf(dataPointInterval / 1000L)
+ + ",\"factor\":" + String.format(Locale.US, "%.9f", factor)
+ + ",\"count\":" + String.valueOf(count) + ",\"values\":[");
+ int dataPointsWritten = 0, previousNonNullIndex = -2;
+ boolean foundTwoAdjacentDataPoints = false;
+ for (int dataPointIndex = firstNonNullIndex; dataPointIndex <=
+ lastNonNullIndex; dataPointIndex++) {
+ double dataPoint = dataPoints.get(dataPointIndex);
+ if (dataPoint >= 0.0) {
+ if (dataPointIndex - previousNonNullIndex == 1) {
+ foundTwoAdjacentDataPoints = true;
+ }
+ previousNonNullIndex = dataPointIndex;
+ }
+ sb.append((dataPointsWritten++ > 0 ? "," : "")
+ + (dataPoint < -0.5 ? "null" :
+ String.valueOf((long) (dataPoint * 999.0))));
+ }
+ sb.append("]}");
+ if (foundTwoAdjacentDataPoints) {
+ return sb.toString();
+ } else {
+ return null;
+ }
+ }
+
+ public String getStatsString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(" " + Logger.formatDecimalNumber(
+ this.newRelayStatuses.size()) + " hours of relay uptimes "
+ + "processed\n");
+ sb.append(" " + Logger.formatDecimalNumber(
+ this.newBridgeStatuses.size()) + " hours of bridge uptimes "
+ + "processed\n");
+ sb.append(" " + Logger.formatDecimalNumber(
+ this.newRunningRelays.size() + this.newRunningBridges.size())
+ + " uptime status files updated\n");
+ sb.append(" " + Logger.formatDecimalNumber(
+ this.newRunningRelays.size() + this.newRunningBridges.size())
+ + " uptime document files updated\n");
+ return sb.toString();
+ }
+}
+
diff --git a/src/org/torproject/onionoo/UptimeDocument.java b/src/org/torproject/onionoo/UptimeDocument.java
new file mode 100644
index 0000000..f71cb87
--- /dev/null
+++ b/src/org/torproject/onionoo/UptimeDocument.java
@@ -0,0 +1,8 @@
+/* Copyright 2014 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.onionoo;
+
+class UptimeDocument extends Document {
+
+}
+
diff --git a/src/org/torproject/onionoo/UptimeStatus.java b/src/org/torproject/onionoo/UptimeStatus.java
new file mode 100644
index 0000000..d16c8fc
--- /dev/null
+++ b/src/org/torproject/onionoo/UptimeStatus.java
@@ -0,0 +1,5 @@
+package org.torproject.onionoo;
+
+class UptimeStatus extends Document {
+}
+
diff --git a/web/index.html b/web/index.html
index c07ca86..732ff8f 100644
--- a/web/index.html
+++ b/web/index.html
@@ -58,6 +58,7 @@ h3 .request-response { padding: 0 !important; }
<li><a href="#bandwidth">Bandwidth documents</a></li>
<li><a href="#weights">Weights documents</a></li>
<li><a href="#clients">Clients documents</a></li>
+ <li><a href="#uptime">Uptime documents</a></li>
</ul>
</div>
@@ -1955,6 +1956,197 @@ It might be removed in the future without notice.</font>
</div> <!-- box -->
+<div class="box">
+<a name="uptime"></a>
+<h3>Uptime documents <a href="#uptime">#</a>
+<span class="request-response">
+<a href="uptime?limit=4">example request</a>
+</span>
+</h3>
+
+<p>
+<font color="blue">Added on March 10, 2014.</font>
+Uptime documents contain fractional uptimes of relays and bridges.
+Uptime documents contain different time intervals and are available for
+all relays and bridges that have been running in the past week.
+Uptime documents contain the following fields:
+</p>
+
+<ul class="properties">
+
+<li>
+<b>relays_published</b>
+<code class="typeof">string</code>
+<span class="required-true">required</span>
+<p>
+UTC timestamp (YYYY-MM-DD hh:mm:ss) when
+the last known relay network status consensus started being valid.
+Indicates how recent the relay uptime documents in this document are.
+</p>
+</li>
+
+<li>
+<b>relays</b>
+<code class="typeof">array of objects</code>
+<span class="required-true">required</span>
+<p>
+Array of objects representing relay uptime documents.
+Each array object contains the following key-value pairs:
+</p>
+
+<ul class="properties">
+
+<li>
+<b>fingerprint</b>
+<code class="typeof">string</code>
+<span class="required-true">required</span>
+<p>
+Relay fingerprint consisting of 40 upper-case
+hexadecimal characters.
+</p>
+</li>
+
+<li>
+<b>uptime</b>
+<code class="typeof">object</code>
+<span class="required-false">optional</span>
+<p>
+History object containing the fractional uptime of this relay.
+Keys are string representation of the time period covered by the uptime
+history object.
+Keys are fixed strings <strong>"1_week"</strong>,
+<strong>"1_month"</strong>, <strong>"3_months"</strong>,
+<strong>"1_year"</strong>, and <strong>"5_years"</strong>.
+Keys refer to the last known uptime history of a relay, not to the time
+when the uptime document was published.
+An uptime history object is only contained if the time period it covers
+is not already contained in another uptime history object with shorter
+time period and higher data resolution.
+Each uptime history object contains the following key-value pairs:
+</p>
+
+<ul class="properties">
+
+<li>
+<b>first</b>
+<code class="typeof">string</code>
+<span class="required-true">required</span>
+<p>
+UTC timestamp (YYYY-MM-DD hh:mm:ss) of the first data
+data point in the uptime history.
+</p>
+</li>
+
+<li>
+<b>last</b>
+<code class="typeof">string</code>
+<span class="required-true">required</span>
+<p>
+UTC timestamp (YYYY-MM-DD hh:mm:ss) of the last data
+data point in the uptime history.
+</p>
+</li>
+
+<li>
+<b>interval</b>
+<code class="typeof">number</code>
+<span class="required-true">required</span>
+<p>
+Time interval between two data points in seconds.
+</p>
+</li>
+
+<li>
+<b>factor</b>
+<code class="typeof">number</code>
+<span class="required-true">required</span>
+<p>
+Factor by which subsequent uptime values need to
+be multiplied to get the fractional uptime.
+This is only done for compatibility reasons with the other document types.
+</p>
+</li>
+
+<li>
+<b>count</b>
+<code class="typeof">number</code>
+<span class="required-false">optional</span>
+<p>
+Number of provided data points, included mostly for debugging purposes.
+Can also be derived from the number of elements in the subsequent array.
+</p>
+</li>
+
+<li>
+<b>values</b>
+<code class="typeof">array of numbers</code>
+<span class="required-true">required</span>
+<p>
+Array of normalized uptime values.
+May contain null values if less than 20% of network statuses have been
+processed for a given time period.
+Contains at least two subsequent non-null values to enable drawing of line
+graphs.
+</p>
+</li>
+
+</ul>
+
+</li>
+
+</ul>
+
+</li>
+
+<li>
+<b>bridges_published</b>
+<code class="typeof">string</code>
+<span class="required-true">required</span>
+<p>
+UTC timestamp (YYYY-MM-DD hh:mm:ss) when
+the last known bridge network status was published.
+Indicates how recent the bridge uptime documents in this document are.
+</p>
+</li>
+
+<li>
+<b>bridges</b>
+<code class="typeof">array of objects</code>
+<span class="required-true">required</span>
+<p>
+Array of objects representing bridge uptime documents.
+Each array object contains the following key-value pairs:
+</p>
+
+<ul class="properties">
+
+<li>
+<b>fingerprint</b>
+<code class="typeof">string</code>
+<span class="required-true">required</span>
+<p>
+SHA-1 hash of the bridge fingerprint consisting
+of 40 upper-case hexadecimal characters.
+</p>
+</li>
+
+<li>
+<b>uptime</b>
+<code class="typeof">object</code>
+<span class="required-true">required</span>
+<p>
+Object containing uptime history objects for different time periods.
+The specification of uptime history objects is similar to those in the
+<strong>uptime</strong> field of <strong>relays</strong>.
+</p>
+</li>
+
+</li>
+
+</ul>
+
+</div> <!-- box -->
+
</body>
</html>
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits