[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [metrics-lib/master] Parse new .tpf Torperf data format.
commit 05a1cf7e7d79bbbad4ece6dabe601cd37d4dfd1d
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Wed May 30 10:59:09 2012 +0200
Parse new .tpf Torperf data format.
---
src/org/torproject/descriptor/TorperfResult.java | 93 ++++
.../torproject/descriptor/impl/DescriptorImpl.java | 12 +-
.../descriptor/impl/DescriptorParserImpl.java | 3 +-
.../descriptor/impl/TorperfResultImpl.java | 466 ++++++++++++++++++++
4 files changed, 570 insertions(+), 4 deletions(-)
diff --git a/src/org/torproject/descriptor/TorperfResult.java b/src/org/torproject/descriptor/TorperfResult.java
new file mode 100644
index 0000000..f1b7a9f
--- /dev/null
+++ b/src/org/torproject/descriptor/TorperfResult.java
@@ -0,0 +1,93 @@
+/* Copyright 2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.descriptor;
+
+import java.util.List;
+import java.util.SortedMap;
+
+public interface TorperfResult extends Descriptor {
+
+ /* Return the configured name of the data source. */
+ public String getSource();
+
+ /* Return the configured file size in bytes. */
+ public int getFileSize();
+
+ /* Return the time when the connection process starts. */
+ public long getStartMillis();
+
+ /* Return the time when the socket was created. */
+ public long getSocketMillis();
+
+ /* Return the time when the socket was connected. */
+ public long getConnectMillis();
+
+ /* Return the time when SOCKS 5 authentication methods have been
+ * negotiated. */
+ public long getNegotiateMillis();
+
+ /* Return the time when the SOCKS request was sent. */
+ public long getRequestMillis();
+
+ /* Return the time when the SOCKS response was received. */
+ public long getResponseMillis();
+
+ /* Return the time when the HTTP request was written. */
+ public long getDataRequestMillis();
+
+ /* Return the time when the first response was received. */
+ public long getDataResponseMillis();
+
+ /* Return the time when the payload was complete. */
+ public long getDataCompleteMillis();
+
+ /* Return the total number of bytes written. */
+ public int getWriteBytes();
+
+ /* Return the total number of bytes read. */
+ public int getReadBytes();
+
+ /* Return whether the request timed out (as opposed to failing), or null
+ * if the torperf line didn't contain that information. */
+ public Boolean didTimeout();
+
+ /* Return the times when x% of expected bytes were read for x = { 10,
+ * 20, 30, 40, 50, 60, 70, 80, 90 }, or null if the torperf line didn't
+ * contain that information. */
+ public SortedMap<Integer, Long> getDataPercentiles();
+
+ /* Return the time when the circuit was launched, or -1 if the torperf
+ * line didn't contain that information. */
+ public long getLaunchMillis();
+
+ /* Return the time when the circuit was used, or -1 if the torperf line
+ * didn't contain that information. */
+ public long getUsedAtMillis();
+
+ /* Return a list of fingerprints of the relays in the circuit, or null
+ * if the torperf line didn't contain that information. */
+ public List<String> getPath();
+
+ /* Return a list of times in millis when circuit hops were built, or
+ * null if the torperf line didn't contain that information. */
+ public List<Long> getBuildTimes();
+
+ /* Return the circuit build timeout that the Tor client used when
+ * building this circuit, or -1 if the torperf line didn't contain that
+ * information. */
+ public long getTimeout();
+
+ /* Return the circuit build time quantile that the Tor client uses to
+ * determine its circuit-build timeout, or -1.0 if the torperf line
+ * didn't contain that information. */
+ public double getQuantile();
+
+ /* Return the identifier of the circuit used for this measurement, or -1
+ * if the torperf line didn't contain that information. */
+ public int getCircId();
+
+ /* Return the identifier of the stream used for this measurement, or -1
+ * if the torperf line didn't contain that information. */
+ public int getUsedBy();
+}
+
diff --git a/src/org/torproject/descriptor/impl/DescriptorImpl.java b/src/org/torproject/descriptor/impl/DescriptorImpl.java
index 27e5153..a4f1613 100644
--- a/src/org/torproject/descriptor/impl/DescriptorImpl.java
+++ b/src/org/torproject/descriptor/impl/DescriptorImpl.java
@@ -13,7 +13,7 @@ import org.torproject.descriptor.Descriptor;
public abstract class DescriptorImpl implements Descriptor {
- protected static List<Descriptor> parseRelayOrBridgeDescriptors(
+ protected static List<Descriptor> parseDescriptors(
byte[] rawDescriptorBytes, String fileName,
boolean failUnrecognizedDescriptorLines)
throws DescriptorParseException {
@@ -83,6 +83,9 @@ public abstract class DescriptorImpl implements Descriptor {
firstLines.contains("\nsigned-directory\n")) {
parsedDescriptors.add(new RelayDirectoryImpl(rawDescriptorBytes,
failUnrecognizedDescriptorLines));
+ } else if (firstLines.startsWith("@type torperf 1.0\n")) {
+ parsedDescriptors.addAll(TorperfResultImpl.parseTorperfResults(
+ rawDescriptorBytes, failUnrecognizedDescriptorLines));
} else {
throw new DescriptorParseException("Could not detect descriptor "
+ "type in descriptor starting with '" + firstLines + "'.");
@@ -156,12 +159,17 @@ public abstract class DescriptorImpl implements Descriptor {
/* Parse annotation lines from the descriptor bytes. */
private List<String> annotations = new ArrayList<String>();
- private void cutOffAnnotations(byte[] rawDescriptorBytes) {
+ private void cutOffAnnotations(byte[] rawDescriptorBytes)
+ throws DescriptorParseException {
String ascii = new String(rawDescriptorBytes);
int start = 0;
while ((start == 0 && ascii.startsWith("@")) ||
(start > 0 && ascii.indexOf("\n@", start - 1) >= 0)) {
int end = ascii.indexOf("\n", start);
+ if (end < 0) {
+ throw new DescriptorParseException("Annotation line does not "
+ + "contain a newline.");
+ }
this.annotations.add(ascii.substring(start, end));
start = end + 1;
}
diff --git a/src/org/torproject/descriptor/impl/DescriptorParserImpl.java b/src/org/torproject/descriptor/impl/DescriptorParserImpl.java
index 2ee6f40..c1d2ae7 100644
--- a/src/org/torproject/descriptor/impl/DescriptorParserImpl.java
+++ b/src/org/torproject/descriptor/impl/DescriptorParserImpl.java
@@ -19,8 +19,7 @@ public class DescriptorParserImpl implements DescriptorParser {
public List<Descriptor> parseDescriptors(byte[] rawDescriptorBytes,
String fileName) throws DescriptorParseException {
- return DescriptorImpl.parseRelayOrBridgeDescriptors(
- rawDescriptorBytes, fileName,
+ return DescriptorImpl.parseDescriptors(rawDescriptorBytes, fileName,
this.failUnrecognizedDescriptorLines);
}
}
diff --git a/src/org/torproject/descriptor/impl/TorperfResultImpl.java b/src/org/torproject/descriptor/impl/TorperfResultImpl.java
new file mode 100644
index 0000000..6a8c1c7
--- /dev/null
+++ b/src/org/torproject/descriptor/impl/TorperfResultImpl.java
@@ -0,0 +1,466 @@
+/* Copyright 2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.descriptor.impl;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Scanner;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.TorperfResult;
+
+public class TorperfResultImpl extends DescriptorImpl
+ implements TorperfResult {
+
+ public static List<Descriptor> parseTorperfResults(
+ byte[] rawDescriptorBytes, boolean failUnrecognizedDescriptorLines)
+ throws DescriptorParseException {
+ if (rawDescriptorBytes.length == 0) {
+ throw new DescriptorParseException("Descriptor is empty.");
+ }
+ List<Descriptor> parsedDescriptors = new ArrayList<Descriptor>();
+ String descriptorString = new String(rawDescriptorBytes);
+ Scanner s = new Scanner(descriptorString).useDelimiter("\n");
+ while (s.hasNext()) {
+ String line = s.next();
+ if (line.startsWith("@type torperf ")) {
+ String[] parts = line.split(" ");
+ if (parts.length != 3) {
+ throw new DescriptorParseException("Illegal line '" + line
+ + "'.");
+ }
+ String version = parts[2];
+ if (!version.startsWith("1.")) {
+ throw new DescriptorParseException("Unsupported version in "
+ + " line '" + line + "'.");
+ }
+ } else {
+ parsedDescriptors.add(new TorperfResultImpl(line.getBytes(),
+ failUnrecognizedDescriptorLines));
+ }
+ }
+ return parsedDescriptors;
+ }
+
+ protected TorperfResultImpl(byte[] rawDescriptorBytes,
+ boolean failUnrecognizedDescriptorLines)
+ throws DescriptorParseException {
+ super(rawDescriptorBytes, failUnrecognizedDescriptorLines, false);
+ this.parseTorperfResultLine(new String(rawDescriptorBytes));
+ }
+
+ private void parseTorperfResultLine(String line)
+ throws DescriptorParseException {
+ if (line.isEmpty()) {
+ throw new DescriptorParseException("Blank lines are not allowed.");
+ }
+ String[] parts = line.split(" ");
+ for (int i = 0; i < parts.length; i++) {
+ String keyValue = parts[i];
+ String[] keyValueParts = keyValue.split("=");
+ if (keyValueParts.length != 2) {
+ throw new DescriptorParseException("Illegal key-value pair in "
+ + "line '" + line + "'.");
+ }
+ String key = keyValueParts[0];
+ this.markKeyAsParsed(key, line);
+ String value = keyValueParts[1];
+ if (key.equals("SOURCE")) {
+ this.parseSource(value, keyValue, line);
+ } else if (key.equals("FILESIZE")) {
+ this.parseFileSize(value, keyValue, line);
+ } else if (key.equals("START")) {
+ this.parseStart(value, keyValue, line);
+ } else if (key.equals("SOCKET")) {
+ this.parseSocket(value, keyValue, line);
+ } else if (key.equals("CONNECT")) {
+ this.parseConnect(value, keyValue, line);
+ } else if (key.equals("NEGOTIATE")) {
+ this.parseNegotiate(value, keyValue, line);
+ } else if (key.equals("REQUEST")) {
+ this.parseRequest(value, keyValue, line);
+ } else if (key.equals("RESPONSE")) {
+ this.parseResponse(value, keyValue, line);
+ } else if (key.equals("DATAREQUEST")) {
+ this.parseDataRequest(value, keyValue, line);
+ } else if (key.equals("DATARESPONSE")) {
+ this.parseDataResponse(value, keyValue, line);
+ } else if (key.equals("DATACOMPLETE")) {
+ this.parseDataComplete(value, keyValue, line);
+ } else if (key.equals("WRITEBYTES")) {
+ this.parseWriteBytes(value, keyValue, line);
+ } else if (key.equals("READBYTES")) {
+ this.parseReadBytes(value, keyValue, line);
+ } else if (key.equals("DIDTIMEOUT")) {
+ this.parseDidTimeout(value, keyValue, line);
+ } else if (key.startsWith("DATAPERC")) {
+ this.parseDataPercentile(value, keyValue, line);
+ } else if (key.equals("LAUNCH")) {
+ this.parseLaunch(value, keyValue, line);
+ } else if (key.equals("USED_AT")) {
+ this.parseUsedAt(value, keyValue, line);
+ } else if (key.equals("PATH")) {
+ this.parsePath(value, keyValue, line);
+ } else if (key.equals("BUILDTIMES")) {
+ this.parseBuildTimes(value, keyValue, line);
+ } else if (key.equals("TIMEOUT")) {
+ this.parseTimeout(value, keyValue, line);
+ } else if (key.equals("QUANTILE")) {
+ this.parseQuantile(value, keyValue, line);
+ } else if (key.equals("CIRC_ID")) {
+ this.parseCircId(value, keyValue, line);
+ } else if (key.equals("USED_BY")) {
+ this.parseUsedBy(value, keyValue, line);
+ } else if (this.failUnrecognizedDescriptorLines) {
+ throw new DescriptorParseException("Unrecognized key '" + key
+ + "' in line '" + line + "'.");
+ } else {
+ if (this.unrecognizedLines == null) {
+ this.unrecognizedLines = new ArrayList<String>();
+ }
+ this.unrecognizedLines.add(line);
+ }
+ }
+ this.checkAllRequiredKeysParsed(line);
+ }
+
+ private Set<String> parsedKeys = new HashSet<String>();
+ private Set<String> requiredKeys = new HashSet<String>(Arrays.asList(
+ ("SOURCE,FILESIZE,START,SOCKET,CONNECT,NEGOTIATE,REQUEST,RESPONSE,"
+ + "DATAREQUEST,DATARESPONSE,DATACOMPLETE,WRITEBYTES,READBYTES").
+ split(",")));
+ private void markKeyAsParsed(String key, String line)
+ throws DescriptorParseException {
+ if (this.parsedKeys.contains(key)) {
+ throw new DescriptorParseException("Key '" + key + "' is contained "
+ + "at least twice in line '" + line + "', but must be "
+ + "contained at most once.");
+ }
+ this.parsedKeys.add(key);
+ this.requiredKeys.remove(key);
+ }
+ private void checkAllRequiredKeysParsed(String line)
+ throws DescriptorParseException {
+ for (String key : this.requiredKeys) {
+ throw new DescriptorParseException("Key '" + key + "' is contained "
+ + "contained 0 times in line '" + line + "', but must be "
+ + "contained exactly once.");
+ }
+ }
+
+ private void parseSource(String value, String keyValue, String line)
+ throws DescriptorParseException {
+ this.source = value;
+ }
+
+ private void parseFileSize(String value, String keyValue, String line)
+ throws DescriptorParseException {
+ try {
+ this.fileSize = Integer.parseInt(value);
+ } catch (NumberFormatException e) {
+ throw new DescriptorParseException("Illegal value in '" + keyValue
+ + "' in line '" + line + "'.");
+ }
+ }
+
+ private void parseStart(String value, String keyValue, String line)
+ throws DescriptorParseException {
+ this.startMillis = this.parseTimestamp(value, keyValue, line);
+ }
+
+ private void parseSocket(String value, String keyValue, String line)
+ throws DescriptorParseException {
+ this.socketMillis = this.parseTimestamp(value, keyValue, line);
+ }
+
+ private void parseConnect(String value, String keyValue, String line)
+ throws DescriptorParseException {
+ this.connectMillis = this.parseTimestamp(value, keyValue, line);
+ }
+
+ private void parseNegotiate(String value, String keyValue, String line)
+ throws DescriptorParseException {
+ this.negotiateMillis = this.parseTimestamp(value, keyValue, line);
+ }
+
+ private void parseRequest(String value, String keyValue, String line)
+ throws DescriptorParseException {
+ this.requestMillis = this.parseTimestamp(value, keyValue, line);
+ }
+
+ private void parseResponse(String value, String keyValue, String line)
+ throws DescriptorParseException {
+ this.responseMillis = this.parseTimestamp(value, keyValue, line);
+ }
+
+ private void parseDataRequest(String value, String keyValue,
+ String line) throws DescriptorParseException {
+ this.dataRequestMillis = this.parseTimestamp(value, keyValue, line);
+ }
+
+ private void parseDataResponse(String value, String keyValue,
+ String line) throws DescriptorParseException {
+ this.dataResponseMillis = this.parseTimestamp(value, keyValue, line);
+ }
+
+ private void parseDataComplete(String value, String keyValue,
+ String line) throws DescriptorParseException {
+ this.dataCompleteMillis = this.parseTimestamp(value, keyValue, line);
+ }
+
+ private void parseWriteBytes(String value, String keyValue, String line)
+ throws DescriptorParseException {
+ this.writeBytes = parseInt(value, keyValue, line);
+ }
+
+ private void parseReadBytes(String value, String keyValue, String line)
+ throws DescriptorParseException {
+ this.readBytes = parseInt(value, keyValue, line);
+ }
+
+ private void parseDidTimeout(String value, String keyValue, String line)
+ throws DescriptorParseException {
+ if (value.equals("1")) {
+ this.didTimeout = true;
+ } else if (value.equals("0")) {
+ this.didTimeout = false;
+ } else {
+ throw new DescriptorParseException("Illegal value in '" + keyValue
+ + "' in line '" + line + "'.");
+ }
+ }
+
+ private Set<String> unparsedPercentiles = new HashSet<String>(
+ Arrays.asList("10,20,30,40,50,60,70,80,90".split(",")));
+ private void parseDataPercentile(String value, String keyValue,
+ String line) throws DescriptorParseException {
+ String percentileString = keyValue.substring("DATAPERC".length(),
+ keyValue.indexOf("="));
+ if (!unparsedPercentiles.contains(percentileString)) {
+ throw new DescriptorParseException("Illegal value in '" + keyValue
+ + "' in line '" + line + "'.");
+ }
+ unparsedPercentiles.remove(percentileString);
+ if (this.dataPercentiles == null) {
+ this.dataPercentiles = new TreeMap<Integer, Long>();
+ }
+ int percentile = Integer.parseInt(percentileString);
+ long timestamp = this.parseTimestamp(value, keyValue, line);
+ this.dataPercentiles.put(percentile, timestamp);
+ }
+
+ private void parseLaunch(String value, String keyValue, String line)
+ throws DescriptorParseException {
+ this.launchMillis = this.parseTimestamp(value, keyValue, line);
+ }
+
+ private void parseUsedAt(String value, String keyValue, String line)
+ throws DescriptorParseException {
+ this.usedAtMillis = this.parseTimestamp(value, keyValue, line);
+ }
+
+ private void parsePath(String value, String keyValue, String line)
+ throws DescriptorParseException {
+ this.path = new ArrayList<String>();
+ for (String fingerprint : value.split(",")) {
+ if (fingerprint.length() != 41) {
+ throw new DescriptorParseException("Illegal value in '" + keyValue
+ + "' in line '" + line + "'.");
+ }
+ this.path.add(ParseHelper.parseTwentyByteHexString(line,
+ fingerprint.substring(1)));
+ }
+ }
+
+ private void parseBuildTimes(String value, String keyValue, String line)
+ throws DescriptorParseException {
+ this.buildTimes = new ArrayList<Long>();
+ for (String buildTimeString : value.split(",")) {
+ this.buildTimes.add(this.parseTimestamp(buildTimeString, keyValue,
+ line));
+ }
+ }
+
+ private void parseTimeout(String value, String keyValue, String line)
+ throws DescriptorParseException {
+ this.timeout = this.parseInt(value, keyValue, line);
+ }
+
+ private void parseQuantile(String value, String keyValue, String line)
+ throws DescriptorParseException {
+ this.quantile = this.parseDouble(value, keyValue, line);
+ }
+
+ private void parseCircId(String value, String keyValue, String line)
+ throws DescriptorParseException {
+ this.circId = this.parseInt(value, keyValue, line);
+ }
+
+ private void parseUsedBy(String value, String keyValue, String line)
+ throws DescriptorParseException {
+ this.usedBy = this.parseInt(value, keyValue, line);
+ }
+
+ private long parseTimestamp(String value, String keyValue, String line)
+ throws DescriptorParseException {
+ long timestamp = -1L;
+ if (value.contains(".") && value.split("\\.").length == 2) {
+ String zeroPaddedValue = (value + "000");
+ String threeDecimalPlaces = zeroPaddedValue.substring(0,
+ zeroPaddedValue.indexOf(".") + 4);
+ String millisString = threeDecimalPlaces.replaceAll("\\.", "");
+ try {
+ timestamp = Long.parseLong(millisString);
+ } catch (NumberFormatException e) {
+ /* Handle below. */
+ }
+ }
+ if (timestamp < 0L) {
+ throw new DescriptorParseException("Illegal timestamp '" + value + "' in '"
+ + keyValue + "' in line '" + line + "'.");
+ }
+ return timestamp;
+ }
+
+ private int parseInt(String value, String keyValue, String line)
+ throws DescriptorParseException {
+ try {
+ return Integer.parseInt(value);
+ } catch (NumberFormatException e) {
+ throw new DescriptorParseException("Illegal value in '" + keyValue
+ + " in line '" + line + "'.");
+ }
+ }
+
+ private double parseDouble(String value, String keyValue, String line)
+ throws DescriptorParseException {
+ try {
+ return Double.parseDouble(value);
+ } catch (NumberFormatException e) {
+ throw new DescriptorParseException("Illegal value in '" + keyValue
+ + "' in line '" + line + "'.");
+ }
+ }
+
+ private String source;
+ public String getSource() {
+ return this.source;
+ }
+
+ private int fileSize;
+ public int getFileSize() {
+ return this.fileSize;
+ }
+
+ private long startMillis;
+ public long getStartMillis() {
+ return this.startMillis;
+ }
+
+ private long socketMillis;
+ public long getSocketMillis() {
+ return this.socketMillis;
+ }
+
+ private long connectMillis;
+ public long getConnectMillis() {
+ return this.connectMillis;
+ }
+
+ private long negotiateMillis;
+ public long getNegotiateMillis() {
+ return this.negotiateMillis;
+ }
+
+ private long requestMillis;
+ public long getRequestMillis() {
+ return this.requestMillis;
+ }
+
+ private long responseMillis;
+ public long getResponseMillis() {
+ return this.responseMillis;
+ }
+
+ private long dataRequestMillis;
+ public long getDataRequestMillis() {
+ return this.dataRequestMillis;
+ }
+
+ private long dataResponseMillis;
+ public long getDataResponseMillis() {
+ return this.dataResponseMillis;
+ }
+
+ private long dataCompleteMillis;
+ public long getDataCompleteMillis() {
+ return this.dataCompleteMillis;
+ }
+
+ private int writeBytes;
+ public int getWriteBytes() {
+ return this.writeBytes;
+ }
+
+ private int readBytes;
+ public int getReadBytes() {
+ return this.readBytes;
+ }
+
+ private boolean didTimeout;
+ public Boolean didTimeout() {
+ return this.didTimeout;
+ }
+
+ private SortedMap<Integer, Long> dataPercentiles;
+ public SortedMap<Integer, Long> getDataPercentiles() {
+ return this.dataPercentiles == null ? null :
+ new TreeMap<Integer, Long>(this.dataPercentiles);
+ }
+
+ private long launchMillis = -1L;
+ public long getLaunchMillis() {
+ return this.launchMillis;
+ }
+
+ private long usedAtMillis = -1L;
+ public long getUsedAtMillis() {
+ return this.usedAtMillis;
+ }
+
+ private List<String> path;
+ public List<String> getPath() {
+ return new ArrayList<String>(this.path);
+ }
+
+ private List<Long> buildTimes;
+ public List<Long> getBuildTimes() {
+ return new ArrayList<Long>(this.buildTimes);
+ }
+
+ private long timeout = -1L;
+ public long getTimeout() {
+ return this.timeout;
+ }
+
+ private double quantile = -1.0;
+ public double getQuantile() {
+ return this.quantile;
+ }
+
+ private int circId = -1;
+ public int getCircId() {
+ return this.circId;
+ }
+
+ private int usedBy = -1;
+ public int getUsedBy() {
+ return this.usedBy;
+ }
+}
+
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits