[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [onionoo/master] Use Gson to format JSON details status files.
commit 677c5a3af88f8b7862c47d2148df6f0f9523062e
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Sun Apr 20 19:02:24 2014 +0200
Use Gson to format JSON details status files.
---
src/org/torproject/onionoo/DetailsStatus.java | 130 +++++++++++++-
src/org/torproject/onionoo/DocumentStore.java | 29 +++-
.../onionoo/NodeDetailsStatusUpdater.java | 179 +++++---------------
3 files changed, 201 insertions(+), 137 deletions(-)
diff --git a/src/org/torproject/onionoo/DetailsStatus.java b/src/org/torproject/onionoo/DetailsStatus.java
index 1ebb100..1a497e4 100644
--- a/src/org/torproject/onionoo/DetailsStatus.java
+++ b/src/org/torproject/onionoo/DetailsStatus.java
@@ -1,7 +1,133 @@
-/* Copyright 2013 The Tor Project
+/* Copyright 2013--2014 The Tor Project
* See LICENSE for licensing information */
package org.torproject.onionoo;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang.StringEscapeUtils;
+
class DetailsStatus extends Document {
-}
+ /* We must ensure that details files only contain ASCII characters
+ * and no UTF-8 characters. While UTF-8 characters are perfectly
+ * valid in JSON, this would break compatibility with existing files
+ * pretty badly. We do this by escaping non-ASCII characters, e.g.,
+ * \u00F2. Gson won't treat this as UTF-8, but will think that we want
+ * to write six characters '\', 'u', '0', '0', 'F', '2'. The only thing
+ * we'll have to do is to change back the '\\' that Gson writes for the
+ * '\'. */
+ private static String escapeJSON(String s) {
+ return s == null ? null :
+ StringEscapeUtils.escapeJavaScript(s).replaceAll("\\\\'", "'");
+ }
+ private static String unescapeJSON(String s) {
+ return s == null ? null :
+ StringEscapeUtils.unescapeJavaScript(s.replaceAll("'", "\\'"));
+ }
+
+ private String desc_published;
+ public void setDescPublished(String descPublished) {
+ this.desc_published = descPublished;
+ }
+ public String getDescPublished() {
+ return this.desc_published;
+ }
+
+ private String last_restarted;
+ public void setLastRestarted(String lastRestarted) {
+ this.last_restarted = lastRestarted;
+ }
+ public String getLastRestarted() {
+ return this.last_restarted;
+ }
+
+ private Integer bandwidth_rate;
+ public void setBandwidthRate(Integer bandwidthRate) {
+ this.bandwidth_rate = bandwidthRate;
+ }
+ public Integer getBandwidthRate() {
+ return this.bandwidth_rate;
+ }
+
+ private Integer bandwidth_burst;
+ public void setBandwidthBurst(Integer bandwidthBurst) {
+ this.bandwidth_burst = bandwidthBurst;
+ }
+ public Integer getBandwidthBurst() {
+ return this.bandwidth_burst;
+ }
+
+ private Integer observed_bandwidth;
+ public void setObservedBandwidth(Integer observedBandwidth) {
+ this.observed_bandwidth = observedBandwidth;
+ }
+ public Integer getObservedBandwidth() {
+ return this.observed_bandwidth;
+ }
+
+ private Integer advertised_bandwidth;
+ public void setAdvertisedBandwidth(Integer advertisedBandwidth) {
+ this.advertised_bandwidth = advertisedBandwidth;
+ }
+ public Integer getAdvertisedBandwidth() {
+ return this.advertised_bandwidth;
+ }
+
+ private List<String> exit_policy;
+ public void setExitPolicy(List<String> exitPolicy) {
+ this.exit_policy = exitPolicy;
+ }
+ public List<String> getExitPolicy() {
+ return this.exit_policy;
+ }
+
+ private String contact;
+ public void setContact(String contact) {
+ this.contact = escapeJSON(contact);
+ }
+ public String getContact() {
+ return unescapeJSON(this.contact);
+ }
+
+ private String platform;
+ public void setPlatform(String platform) {
+ this.platform = escapeJSON(platform);
+ }
+ public String getPlatform() {
+ return unescapeJSON(this.platform);
+ }
+
+ private List<String> family;
+ public void setFamily(List<String> family) {
+ this.family = family;
+ }
+ public List<String> getFamily() {
+ return this.family;
+ }
+
+ private Map<String, List<String>> exit_policy_v6_summary;
+ public void setExitPolicyV6Summary(
+ Map<String, List<String>> exitPolicyV6Summary) {
+ this.exit_policy_v6_summary = exitPolicyV6Summary;
+ }
+ public Map<String, List<String>> getExitPolicyV6Summary() {
+ return this.exit_policy_v6_summary;
+ }
+
+ private Boolean hibernating;
+ public void setHibernating(Boolean hibernating) {
+ this.hibernating = hibernating;
+ }
+ public Boolean getHibernating() {
+ return this.hibernating;
+ }
+
+ private String pool_assignment;
+ public void setPoolAssignment(String poolAssignment) {
+ this.pool_assignment = poolAssignment;
+ }
+ public String getPoolAssignment() {
+ return this.pool_assignment;
+ }
+}
diff --git a/src/org/torproject/onionoo/DocumentStore.java b/src/org/torproject/onionoo/DocumentStore.java
index e98fb4e..d80999e 100644
--- a/src/org/torproject/onionoo/DocumentStore.java
+++ b/src/org/torproject/onionoo/DocumentStore.java
@@ -21,6 +21,7 @@ import java.util.TreeMap;
import java.util.TreeSet;
import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
import com.google.gson.JsonParseException;
// TODO For later migration from disk to database, do the following:
@@ -187,13 +188,34 @@ public class DocumentStore {
String documentString;
if (document.getDocumentString() != null) {
documentString = document.getDocumentString();
- } else if (document instanceof DetailsDocument ||
- document instanceof BandwidthDocument ||
+ } else if (document instanceof BandwidthDocument ||
document instanceof WeightsDocument ||
document instanceof ClientsDocument ||
document instanceof UptimeDocument) {
Gson gson = new Gson();
documentString = gson.toJson(document);
+ } else if (document instanceof DetailsStatus ||
+ document instanceof DetailsDocument) {
+ /* Don't escape HTML characters, like < and >, contained in
+ * strings. */
+ Gson gson = new GsonBuilder().disableHtmlEscaping().create();
+ /* We must ensure that details files only contain ASCII characters
+ * and no UTF-8 characters. While UTF-8 characters are perfectly
+ * valid in JSON, this would break compatibility with existing files
+ * pretty badly. We already make sure that all strings in details
+ * objects are escaped JSON, e.g., \u00F2. When Gson serlializes
+ * this string, it escapes the \ to \\, hence writes \\u00F2. We
+ * need to undo this and change \\u00F2 back to \u00F2. */
+ documentString = gson.toJson(document).replaceAll("\\\\\\\\u",
+ "\\\\u");
+ /* Existing details statuses don't contain opening and closing curly
+ * brackets, so we should remove them from new details statuses,
+ * too. */
+ if (document instanceof DetailsStatus) {
+ documentString = documentString.substring(
+ documentString.indexOf("{") + 1,
+ documentString.lastIndexOf("}"));
+ }
} else if (document instanceof BandwidthStatus ||
document instanceof WeightsStatus ||
document instanceof ClientsStatus ||
@@ -380,6 +402,9 @@ public class DocumentStore {
documentType.equals(ClientsStatus.class) ||
documentType.equals(UptimeStatus.class)) {
return this.retrieveParsedStatusFile(documentType, documentString);
+ } else if (documentType.equals(DetailsStatus.class)) {
+ return this.retrieveParsedDocumentFile(documentType, "{"
+ + documentString + "}");
} else {
System.err.println("Parsing is not supported for type "
+ documentType.getName() + ".");
diff --git a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java
index 4cfdd33..030a8ec 100644
--- a/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java
+++ b/src/org/torproject/onionoo/NodeDetailsStatusUpdater.java
@@ -5,15 +5,14 @@ package org.torproject.onionoo;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
-import java.util.Scanner;
import java.util.Set;
import java.util.SortedMap;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
-import org.apache.commons.lang.StringEscapeUtils;
import org.torproject.descriptor.BridgeNetworkStatus;
import org.torproject.descriptor.BridgePoolAssignment;
import org.torproject.descriptor.Descriptor;
@@ -291,27 +290,16 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
ServerDescriptor descriptor) {
String fingerprint = descriptor.getFingerprint();
DetailsStatus detailsStatus = this.documentStore.retrieve(
- DetailsStatus.class, false, fingerprint);
+ DetailsStatus.class, true, fingerprint);
String publishedDateTime =
DateTimeHelper.format(descriptor.getPublishedMillis());
- if (detailsStatus != null) {
- String detailsString = detailsStatus.getDocumentString();
- String descPublishedLine = "\"desc_published\":\""
- + publishedDateTime + "\",";
- Scanner s = new Scanner(detailsString);
- while (s.hasNextLine()) {
- String line = s.nextLine();
- if (line.startsWith("\"desc_published\":\"")) {
- if (descPublishedLine.compareTo(line) < 0) {
- return;
- } else {
- break;
- }
- }
- }
- s.close();
+ if (detailsStatus == null) {
+ detailsStatus = new DetailsStatus();
+ } else if (detailsStatus.getDescPublished() != null &&
+ publishedDateTime.compareTo(
+ detailsStatus.getDescPublished()) < 0) {
+ return;
}
- StringBuilder sb = new StringBuilder();
String lastRestartedString = DateTimeHelper.format(
descriptor.getPublishedMillis() - descriptor.getUptime()
* DateTimeHelper.ONE_SECOND);
@@ -320,55 +308,31 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
int observedBandwidth = descriptor.getBandwidthObserved();
int advertisedBandwidth = Math.min(bandwidthRate,
Math.min(bandwidthBurst, observedBandwidth));
- sb.append("\"desc_published\":\"" + publishedDateTime + "\",\n"
- + "\"last_restarted\":\"" + lastRestartedString + "\",\n"
- + "\"bandwidth_rate\":" + bandwidthRate + ",\n"
- + "\"bandwidth_burst\":" + bandwidthBurst + ",\n"
- + "\"observed_bandwidth\":" + observedBandwidth + ",\n"
- + "\"advertised_bandwidth\":" + advertisedBandwidth + ",\n"
- + "\"exit_policy\":[");
- int written = 0;
- for (String exitPolicyLine : descriptor.getExitPolicyLines()) {
- sb.append((written++ > 0 ? "," : "") + "\n \"" + exitPolicyLine
- + "\"");
- }
- sb.append("\n]");
- if (descriptor.getContact() != null) {
- sb.append(",\n\"contact\":\""
- + escapeJSON(descriptor.getContact()) + "\"");
- }
- if (descriptor.getPlatform() != null) {
- sb.append(",\n\"platform\":\""
- + escapeJSON(descriptor.getPlatform()) + "\"");
- }
- if (descriptor.getFamilyEntries() != null) {
- sb.append(",\n\"family\":[");
- written = 0;
- for (String familyEntry : descriptor.getFamilyEntries()) {
- sb.append((written++ > 0 ? "," : "") + "\n \"" + familyEntry
- + "\"");
- }
- sb.append("\n]");
- }
+ detailsStatus.setDescPublished(publishedDateTime);
+ detailsStatus.setLastRestarted(lastRestartedString);
+ detailsStatus.setBandwidthRate(bandwidthRate);
+ detailsStatus.setBandwidthBurst(bandwidthBurst);
+ detailsStatus.setObservedBandwidth(observedBandwidth);
+ detailsStatus.setAdvertisedBandwidth(advertisedBandwidth);
+ detailsStatus.setExitPolicy(descriptor.getExitPolicyLines());
+ detailsStatus.setContact(descriptor.getContact());
+ detailsStatus.setPlatform(descriptor.getPlatform());
+ detailsStatus.setFamily(descriptor.getFamilyEntries());
if (descriptor.getIpv6DefaultPolicy() != null &&
(descriptor.getIpv6DefaultPolicy().equals("accept") ||
descriptor.getIpv6DefaultPolicy().equals("reject")) &&
descriptor.getIpv6PortList() != null) {
- sb.append(",\n\"exit_policy_v6_summary\":{\""
- + descriptor.getIpv6DefaultPolicy() + "\":[");
- int portsWritten = 0;
- for (String portOrPortRange :
- descriptor.getIpv6PortList().split(",")) {
- sb.append((portsWritten++ > 0 ? "," : "") + "\"" + portOrPortRange
- + "\"");
- }
- sb.append("]}");
+ Map<String, List<String>> exitPolicyV6Summary =
+ new HashMap<String, List<String>>();
+ List<String> portsOrPortRanges = Arrays.asList(
+ descriptor.getIpv6PortList().split(","));
+ exitPolicyV6Summary.put(descriptor.getIpv6DefaultPolicy(),
+ portsOrPortRanges);
+ detailsStatus.setExitPolicyV6Summary(exitPolicyV6Summary);
}
if (descriptor.isHibernating()) {
- sb.append(",\n\"hibernating\":true");
+ detailsStatus.setHibernating(true);
}
- detailsStatus = new DetailsStatus();
- detailsStatus.setDocumentString(sb.toString());
this.documentStore.store(detailsStatus, fingerprint);
if (descriptor.getContact() != null) {
this.contacts.put(fingerprint, descriptor.getContact());
@@ -379,79 +343,43 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
ServerDescriptor descriptor) {
String fingerprint = descriptor.getFingerprint();
DetailsStatus detailsStatus = this.documentStore.retrieve(
- DetailsStatus.class, false, fingerprint);
+ DetailsStatus.class, true, fingerprint);
String publishedDateTime =
DateTimeHelper.format(descriptor.getPublishedMillis());
- String poolAssignmentLine = null;
- if (detailsStatus != null) {
- String detailsString = detailsStatus.getDocumentString();
- String descPublishedLine = "\"desc_published\":\""
- + publishedDateTime + "\",";
- Scanner s = new Scanner(detailsString);
- while (s.hasNextLine()) {
- String line = s.nextLine();
- if (line.startsWith("\"pool_assignment\":")) {
- poolAssignmentLine = line;
- } else if (line.startsWith("\"desc_published\":") &&
- descPublishedLine.compareTo(line) < 0) {
- return;
- }
- }
- s.close();
+ if (detailsStatus == null) {
+ detailsStatus = new DetailsStatus();
+ } else if (detailsStatus.getDescPublished() != null &&
+ publishedDateTime.compareTo(
+ detailsStatus.getDescPublished()) < 0) {
+ return;
}
- StringBuilder sb = new StringBuilder();
String lastRestartedString = DateTimeHelper.format(
descriptor.getPublishedMillis() - descriptor.getUptime()
* DateTimeHelper.ONE_SECOND);
int advertisedBandwidth = Math.min(descriptor.getBandwidthRate(),
Math.min(descriptor.getBandwidthBurst(),
descriptor.getBandwidthObserved()));
- sb.append("\"desc_published\":\"" + publishedDateTime + "\",\n"
- + "\"last_restarted\":\"" + lastRestartedString + "\",\n"
- + "\"advertised_bandwidth\":" + advertisedBandwidth + ",\n"
- + "\"platform\":\"" + escapeJSON(descriptor.getPlatform())
- + "\"");
- if (poolAssignmentLine != null) {
- sb.append(",\n" + poolAssignmentLine);
- }
- detailsStatus = new DetailsStatus();
- detailsStatus.setDocumentString(sb.toString());
+ detailsStatus.setDescPublished(publishedDateTime);
+ detailsStatus.setLastRestarted(lastRestartedString);
+ detailsStatus.setAdvertisedBandwidth(advertisedBandwidth);
+ detailsStatus.setPlatform(descriptor.getPlatform());
this.documentStore.store(detailsStatus, fingerprint);
}
- private static String escapeJSON(String s) {
- return StringEscapeUtils.escapeJavaScript(s).replaceAll("\\\\'", "'");
- }
-
private void processBridgePoolAssignment(
BridgePoolAssignment bridgePoolAssignment) {
for (Map.Entry<String, String> e :
bridgePoolAssignment.getEntries().entrySet()) {
String fingerprint = e.getKey();
String details = e.getValue();
- StringBuilder sb = new StringBuilder();
DetailsStatus detailsStatus = this.documentStore.retrieve(
- DetailsStatus.class, false, fingerprint);
- if (detailsStatus != null) {
- String detailsString = detailsStatus.getDocumentString();
- Scanner s = new Scanner(detailsString);
- int linesWritten = 0;
- boolean endsWithComma = false;
- while (s.hasNextLine()) {
- String line = s.nextLine();
- if (!line.startsWith("\"pool_assignment\":")) {
- sb.append((linesWritten++ > 0 ? "\n" : "") + line);
- endsWithComma = line.endsWith(",");
- }
- }
- s.close();
- if (sb.length() > 0) {
- sb.append((endsWithComma ? "" : ",") + "\n");
- }
+ DetailsStatus.class, true, fingerprint);
+ if (detailsStatus == null) {
+ detailsStatus = new DetailsStatus();
+ } else if (details.equals(detailsStatus.getPoolAssignment())) {
+ return;
}
- sb.append("\"pool_assignment\":\"" + details + "\"");
- detailsStatus = new DetailsStatus();
- detailsStatus.setDocumentString(sb.toString());
+ detailsStatus.setPoolAssignment(details);
this.documentStore.store(detailsStatus, fingerprint);
}
}
@@ -540,25 +468,10 @@ public class NodeDetailsStatusUpdater implements DescriptorListener,
!relay.getRelayFlags().contains("BadExit");
boolean isGuard = relay.getRelayFlags().contains("Guard");
DetailsStatus detailsStatus = this.documentStore.retrieve(
- DetailsStatus.class, false, fingerprint);
+ DetailsStatus.class, true, fingerprint);
if (detailsStatus != null) {
- double advertisedBandwidth = -1.0;
- String detailsString = detailsStatus.getDocumentString();
- Scanner s = new Scanner(detailsString);
- while (s.hasNextLine()) {
- String line = s.nextLine();
- if (!line.startsWith("\"advertised_bandwidth\":")) {
- continue;
- }
- try {
- advertisedBandwidth = (double) Integer.parseInt(
- line.split(":")[1].replaceAll(",", ""));
- } catch (NumberFormatException ex) {
- /* Handle below. */
- }
- break;
- }
- s.close();
+ double advertisedBandwidth =
+ detailsStatus.getAdvertisedBandwidth();
if (advertisedBandwidth >= 0.0) {
advertisedBandwidths.put(fingerprint, advertisedBandwidth);
totalAdvertisedBandwidth += advertisedBandwidth;
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits