[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [metrics-lib/master] Parse OnionPerf analysis results format v3.0.



commit b44c6bccb46ca1eb9c861a84dfcd7d5d2047dee4
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Thu Jul 16 21:37:19 2020 +0200

    Parse OnionPerf analysis results format v3.0.
    
    Implements tpo/metrics/library#40001.
---
 CHANGELOG.md                                       |   1 +
 .../onionperf/OnionPerfAnalysisConverter.java      | 235 ++++++++++++++++-----
 .../onionperf/ParsedOnionPerfAnalysis.java         | 206 ++++++++++++++++++
 .../onionperf/TorperfResultsBuilder.java           |  38 ++++
 .../onionperf/OnionPerfAnalysisConverterTest.java  |  80 ++++++-
 ...20-07-13.op-nl-test1.onionperf.analysis.json.xz | Bin 0 -> 1736 bytes
 6 files changed, 499 insertions(+), 61 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7bb22c4..4679688 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,7 @@
 
  * Medium changes
    - Extend Torperf results to provide error codes.
+   - Parse OnionPerf analysis results format version 3.0.
 
 
 # Changes in version 2.13.0 - 2020-05-16
diff --git a/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java b/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java
index 61fd173..f1b9d11 100644
--- a/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java
+++ b/src/main/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverter.java
@@ -69,25 +69,32 @@ public class OnionPerfAnalysisConverter {
    *     Torperf results.
    */
   public List<Descriptor> asTorperfResults() throws DescriptorParseException {
-    ParsedOnionPerfAnalysis parsedOnionPerfAnalysis;
+    ParsedOnionPerfAnalysis parsedOnionPerfAnalysis
+        = this.parseOnionPerfAnalysis();
+    this.verifyDocumentTypeAndVersion(parsedOnionPerfAnalysis);
+    StringBuilder formattedTorperfResults
+        = this.formatTorperfResults(parsedOnionPerfAnalysis);
+    this.parseFormattedTorperfResults(formattedTorperfResults);
+    return this.convertedTorperfResults;
+  }
+
+  /**
+   * Parse the OnionPerf analysis JSON document.
+   */
+  private ParsedOnionPerfAnalysis parseOnionPerfAnalysis()
+      throws DescriptorParseException {
     try {
       InputStream compressedInputStream = new ByteArrayInputStream(
           this.rawDescriptorBytes);
       InputStream decompressedInputStream = new XZCompressorInputStream(
           compressedInputStream);
       byte[] decompressedBytes = IOUtils.toByteArray(decompressedInputStream);
-      parsedOnionPerfAnalysis = ParsedOnionPerfAnalysis.fromBytes(
-          decompressedBytes);
+      return ParsedOnionPerfAnalysis.fromBytes(decompressedBytes);
     } catch (IOException ioException) {
       throw new DescriptorParseException("Ran into an I/O error while "
           + "attempting to parse an OnionPerf analysis document.",
           ioException);
     }
-    this.verifyDocumentTypeAndVersion(parsedOnionPerfAnalysis);
-    StringBuilder formattedTorperfResults
-        = this.formatTorperfResults(parsedOnionPerfAnalysis);
-    this.parseFormattedTorperfResults(formattedTorperfResults);
-    return this.convertedTorperfResults;
   }
 
   /**
@@ -109,9 +116,9 @@ public class OnionPerfAnalysisConverter {
       throw new DescriptorParseException("Parsed OnionPerf analysis file does "
           + "not contain version information.");
     } else if ((parsedOnionPerfAnalysis.version instanceof Double
-        && (double) parsedOnionPerfAnalysis.version > 2.999)
+        && (double) parsedOnionPerfAnalysis.version > 3.999)
         || (parsedOnionPerfAnalysis.version instanceof String
-        && ((String) parsedOnionPerfAnalysis.version).compareTo("3.") >= 0)) {
+        && ((String) parsedOnionPerfAnalysis.version).compareTo("4.") >= 0)) {
       throw new DescriptorParseException("Parsed OnionPerf analysis file "
           + "contains unsupported version " + parsedOnionPerfAnalysis.version
           + ".");
@@ -131,8 +138,7 @@ public class OnionPerfAnalysisConverter {
         : parsedOnionPerfAnalysis.data.entrySet()) {
       String nickname = data.getKey();
       ParsedOnionPerfAnalysis.MeasurementData measurements = data.getValue();
-      if (null == measurements.measurementIp || null == measurements.tgen
-          || null == measurements.tgen.transfers) {
+      if (null == measurements.tgen) {
         continue;
       }
       String measurementIp = measurements.measurementIp;
@@ -153,57 +159,69 @@ public class OnionPerfAnalysisConverter {
           }
         }
       }
-      for (ParsedOnionPerfAnalysis.Transfer transfer
-          : measurements.tgen.transfers.values()) {
-        if (null == transfer.endpointLocal) {
-          continue;
-        }
-        String[] endpointLocalParts = transfer.endpointLocal.split(":");
-        if (endpointLocalParts.length < 3) {
-          continue;
-        }
-        TorperfResultsBuilder torperfResultsBuilder
-            = new TorperfResultsBuilder();
-
-        torperfResultsBuilder.addString("SOURCE", nickname);
-        torperfResultsBuilder.addString("SOURCEADDRESS", measurementIp);
-        this.formatTransferParts(torperfResultsBuilder, transfer);
-        List<String> errorCodeParts = null;
-        if (transfer.isError) {
-          errorCodeParts = new ArrayList<>();
-          if ("PROXY".equals(transfer.errorCode)) {
-            errorCodeParts.add("TOR");
-          } else {
-            errorCodeParts.add("TGEN");
-            errorCodeParts.add(transfer.errorCode);
+      if (null != measurements.tgen.transfers) {
+        for (ParsedOnionPerfAnalysis.Transfer transfer
+            : measurements.tgen.transfers.values()) {
+          TorperfResultsBuilder torperfResultsBuilder
+              = new TorperfResultsBuilder();
+          torperfResultsBuilder.addString("SOURCE", nickname);
+          torperfResultsBuilder.addString("SOURCEADDRESS", measurementIp);
+          this.formatTransferParts(torperfResultsBuilder, transfer);
+          if (null != transfer.endpointLocal) {
+            String[] endpointLocalParts = transfer.endpointLocal.split(":");
+            if (endpointLocalParts.length >= 3) {
+              String sourcePort = endpointLocalParts[2];
+              if (streamsBySourcePort.containsKey(sourcePort)) {
+                for (ParsedOnionPerfAnalysis.Stream stream
+                    : streamsBySourcePort.get(sourcePort)) {
+                  if (Math.abs(transfer.unixTsEnd - stream.unixTsEnd) < 150.0) {
+                    this.formatStreamParts(torperfResultsBuilder, stream);
+                    if (null != stream.circuitId
+                        && circuitsByCircuitId.containsKey(stream.circuitId)) {
+                      ParsedOnionPerfAnalysis.Circuit circuit
+                          = circuitsByCircuitId.get(stream.circuitId);
+                      this.formatCircuitParts(torperfResultsBuilder, circuit);
+                    }
+                  }
+                }
+              }
+            }
           }
+          formattedTorperfResults.append(torperfResultsBuilder.build());
         }
-        String sourcePort = endpointLocalParts[2];
-        if (streamsBySourcePort.containsKey(sourcePort)) {
-          for (ParsedOnionPerfAnalysis.Stream stream
-              : streamsBySourcePort.get(sourcePort)) {
-            if (Math.abs(transfer.unixTsEnd - stream.unixTsEnd) < 150.0) {
-              if (null != errorCodeParts && null != stream.failureReasonLocal) {
-                errorCodeParts.add(stream.failureReasonLocal);
-                if (null != stream.failureReasonRemote) {
-                  errorCodeParts.add(stream.failureReasonRemote);
+      }
+      if (null != measurements.tgen.streams) {
+        for (ParsedOnionPerfAnalysis.TgenStream stream
+             : measurements.tgen.streams.values()) {
+          TorperfResultsBuilder torperfResultsBuilder
+              = new TorperfResultsBuilder();
+          torperfResultsBuilder.addString("SOURCE", nickname);
+          torperfResultsBuilder.addString("SOURCEADDRESS", measurementIp);
+          this.formatTgenStreamParts(torperfResultsBuilder, stream);
+          if (null != stream.transportInfo
+              && null != stream.transportInfo.local) {
+            String[] endpointLocalParts = stream.transportInfo.local.split(":");
+            if (endpointLocalParts.length >= 3) {
+              String sourcePort = endpointLocalParts[2];
+              if (streamsBySourcePort.containsKey(sourcePort)) {
+                for (ParsedOnionPerfAnalysis.Stream torStream
+                    : streamsBySourcePort.get(sourcePort)) {
+                  if (Math.abs(stream.unixTsEnd
+                      - torStream.unixTsEnd) < 150.0) {
+                    this.formatStreamParts(torperfResultsBuilder, torStream);
+                    if (null != torStream.circuitId && circuitsByCircuitId
+                        .containsKey(torStream.circuitId)) {
+                      ParsedOnionPerfAnalysis.Circuit circuit
+                          = circuitsByCircuitId.get(torStream.circuitId);
+                      this.formatCircuitParts(torperfResultsBuilder, circuit);
+                    }
+                  }
                 }
               }
-              if (null != stream.circuitId
-                  && circuitsByCircuitId.containsKey(stream.circuitId)) {
-                ParsedOnionPerfAnalysis.Circuit circuit
-                    = circuitsByCircuitId.get(stream.circuitId);
-                this.formatStreamParts(torperfResultsBuilder, stream);
-                this.formatCircuitParts(torperfResultsBuilder, circuit);
-              }
             }
           }
+          formattedTorperfResults.append(torperfResultsBuilder.build());
         }
-        if (null != errorCodeParts) {
-          String errorCode = String.join("/", errorCodeParts);
-          torperfResultsBuilder.addString("ERRORCODE", errorCode);
-        }
-        formattedTorperfResults.append(torperfResultsBuilder.build());
       }
     }
     return formattedTorperfResults;
@@ -288,6 +306,105 @@ public class OnionPerfAnalysisConverter {
           transfer.elapsedSeconds.lastByte);
       if (transfer.isError) {
         torperfResultsBuilder.addInteger("DIDTIMEOUT", 1);
+        if ("PROXY".equals(transfer.errorCode)) {
+          torperfResultsBuilder.addErrorCodePart("TOR");
+        } else {
+          torperfResultsBuilder.addErrorCodePart("TGEN");
+          torperfResultsBuilder.addErrorCodePart(transfer.errorCode);
+        }
+      }
+    }
+  }
+
+  /**
+   * Format relevant tgen stream data as Torperf result key-value pairs.
+   *
+   * @param torperfResultsBuilder Torperf results builder to add key-value pairs
+   *     to.
+   * @param stream Stream data obtained from the parsed OnionPerf analysis file.
+   */
+  private void formatTgenStreamParts(
+      TorperfResultsBuilder torperfResultsBuilder,
+      ParsedOnionPerfAnalysis.TgenStream stream) {
+    torperfResultsBuilder.addString("READBYTES", "0");
+    torperfResultsBuilder.addString("WRITEBYTES", "0");
+    if (null != stream.byteInfo) {
+      torperfResultsBuilder.addString("READBYTES",
+          stream.byteInfo.totalBytesRecv);
+      torperfResultsBuilder.addString("WRITEBYTES",
+          stream.byteInfo.totalBytesSend);
+    }
+    if (null != stream.streamInfo) {
+      torperfResultsBuilder.addString("FILESIZE", stream.streamInfo.recvsize);
+      torperfResultsBuilder.addString("HOSTNAMELOCAL", stream.streamInfo.name);
+      torperfResultsBuilder.addString("HOSTNAMEREMOTE",
+          stream.streamInfo.peername);
+    }
+    if (null != stream.transportInfo) {
+      torperfResultsBuilder.addString("ENDPOINTLOCAL",
+          stream.transportInfo.local);
+      torperfResultsBuilder.addString("ENDPOINTPROXY",
+          stream.transportInfo.proxy);
+      torperfResultsBuilder.addString("ENDPOINTREMOTE",
+          stream.transportInfo.remote);
+    }
+    torperfResultsBuilder.addInteger("DIDTIMEOUT", 0);
+
+    for (String key : new String[] { "START", "SOCKET", "CONNECT", "NEGOTIATE",
+        "REQUEST", "RESPONSE", "DATAREQUEST", "DATARESPONSE",
+        "DATACOMPLETE" }) {
+      torperfResultsBuilder.addString(key, "0.0");
+    }
+    torperfResultsBuilder.addTimestamp("START", stream.unixTsStart, 0.0);
+    if (null != stream.unixTsStart) {
+      if (null != stream.timeInfo) {
+        torperfResultsBuilder.addTimestamp("SOCKET", stream.unixTsStart,
+            stream.timeInfo.usecsToSocketCreate);
+        torperfResultsBuilder.addTimestamp("CONNECT", stream.unixTsStart,
+            stream.timeInfo.usecsToSocketConnect);
+        torperfResultsBuilder.addTimestamp("NEGOTIATE", stream.unixTsStart,
+            stream.timeInfo.usecsToProxyChoice);
+        torperfResultsBuilder.addTimestamp("REQUEST", stream.unixTsStart,
+            stream.timeInfo.usecsToProxyRequest);
+        torperfResultsBuilder.addTimestamp("RESPONSE", stream.unixTsStart,
+            stream.timeInfo.usecsToProxyResponse);
+        torperfResultsBuilder.addTimestamp("DATAREQUEST", stream.unixTsStart,
+            stream.timeInfo.usecsToCommand);
+        torperfResultsBuilder.addTimestamp("DATARESPONSE", stream.unixTsStart,
+            stream.timeInfo.usecsToResponse);
+        torperfResultsBuilder.addTimestamp("DATACOMPLETE", stream.unixTsStart,
+            stream.timeInfo.usecsToLastByteRecv);
+      }
+      if (null != stream.elapsedSeconds) {
+        if (null != stream.elapsedSeconds.payloadBytesRecv) {
+          for (Map.Entry<String, Double> payloadBytesRecvEntry
+              : stream.elapsedSeconds.payloadBytesRecv.entrySet()) {
+            String key = String.format("PARTIAL%s",
+                payloadBytesRecvEntry.getKey());
+            Double elapsedSeconds = payloadBytesRecvEntry.getValue();
+            torperfResultsBuilder.addTimestamp(key, stream.unixTsStart,
+                elapsedSeconds);
+          }
+        }
+        if (null != stream.elapsedSeconds.payloadProgressRecv) {
+          for (Map.Entry<String, Double> payloadProgressRecvEntry
+              : stream.elapsedSeconds.payloadProgressRecv.entrySet()) {
+            String key = String.format("DATAPERC%.0f",
+                Double.parseDouble(payloadProgressRecvEntry.getKey()) * 100.0);
+            Double elapsedSeconds = payloadProgressRecvEntry.getValue();
+            torperfResultsBuilder.addTimestamp(key, stream.unixTsStart,
+                elapsedSeconds);
+          }
+        }
+      }
+      if (null != stream.isError && stream.isError) {
+        torperfResultsBuilder.addInteger("DIDTIMEOUT", 1);
+        if ("PROXY".equals(stream.streamInfo.error)) {
+          torperfResultsBuilder.addErrorCodePart("TOR");
+        } else {
+          torperfResultsBuilder.addErrorCodePart("TGEN");
+          torperfResultsBuilder.addErrorCodePart(stream.streamInfo.error);
+        }
       }
     }
   }
@@ -301,6 +418,12 @@ public class OnionPerfAnalysisConverter {
    */
   private void formatStreamParts(TorperfResultsBuilder torperfResultsBuilder,
       ParsedOnionPerfAnalysis.Stream stream) {
+    if (null != stream.failureReasonLocal) {
+      torperfResultsBuilder.addErrorCodePart(stream.failureReasonLocal);
+      if (null != stream.failureReasonRemote) {
+        torperfResultsBuilder.addErrorCodePart(stream.failureReasonRemote);
+      }
+    }
     torperfResultsBuilder.addTimestamp("USED_AT", stream.unixTsEnd, 0.0);
     torperfResultsBuilder.addInteger("USED_BY", stream.streamId);
   }
diff --git a/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java b/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java
index 4eca0ff..e94fdb4 100644
--- a/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java
+++ b/src/main/java/org/torproject/descriptor/onionperf/ParsedOnionPerfAnalysis.java
@@ -4,6 +4,7 @@
 package org.torproject.descriptor.onionperf;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.PropertyAccessor;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -95,6 +96,11 @@ public class ParsedOnionPerfAnalysis {
      * Measurement data by transfer identifier.
      */
     Map<String, Transfer> transfers;
+
+    /**
+     * Measurement data by stream identifier.
+     */
+    Map<String, TgenStream> streams;
   }
 
   /**
@@ -240,6 +246,206 @@ public class ParsedOnionPerfAnalysis {
     Double socketCreate;
   }
 
+  /**
+   * Measurement data related to a single stream obtained from client-side
+   * {@code tgen} logs.
+   */
+  static class TgenStream {
+
+    /**
+     * Information on sent and received bytes.
+     */
+    ByteInfo byteInfo;
+
+    /**
+     * Elapsed seconds until a given number or fraction of payload bytes have
+     * been received or sent, obtained from {@code [stream-status]},
+     * {@code [stream-success]}, and {@code [stream-error]} log messages, only
+     * included if the measurement was a success.
+     */
+    ElapsedSecondsPayload elapsedSeconds;
+
+    /**
+     * Whether an error occurred.
+     */
+    Boolean isError;
+
+    /**
+     * Information about the TGen stream.
+     */
+    StreamInfo streamInfo;
+
+    /**
+     * Elapsed time until reaching given substeps in a measurement.
+     */
+    TimeInfo timeInfo;
+
+    /**
+     * Information about the TGen transport.
+     */
+    TransportInfo transportInfo;
+
+    /**
+     * Initial start time of the measurement, obtained by subtracting the
+     * largest number of elapsed microseconds in {@code time_info} from
+     * {@code unix_ts_end}, given in seconds since the epoch.
+     */
+    Double unixTsStart;
+
+    /**
+     * Final end time of the measurement, obtained from the log time of the
+     * {@code [stream-success]} or {@code [stream-error]} log message, given in
+     * seconds since the epoch.
+     */
+    Double unixTsEnd;
+  }
+
+  /**
+   * Information on sent and received bytes.
+   */
+  static class ByteInfo {
+
+    /**
+     * Total number of bytes received.
+     */
+    @JsonProperty("total-bytes-recv")
+    String totalBytesRecv;
+
+    /**
+     * Total number of bytes sent.
+     */
+    @JsonProperty("total-bytes-send")
+    String totalBytesSend;
+  }
+
+  /**
+   * Elapsed seconds until a given number or fraction of payload bytes have been
+   * received or sent, obtained from {@code [stream-status]},
+   * {@code [stream-success]}, and {@code [stream-error]} log messages, only
+   * included if the measurement was a success.
+   */
+  static class ElapsedSecondsPayload {
+
+    /**
+     * Number of received payload bytes.
+     */
+    Map<String, Double> payloadBytesRecv;
+
+    /**
+     * Fraction of received payload bytes.
+     */
+    Map<String, Double> payloadProgressRecv;
+  }
+
+  /**
+   * Information about the TGen stream.
+   */
+  static class StreamInfo {
+
+    /**
+     * Error code, or {@code NONE} if no error occurred.
+     */
+    String error;
+
+    /**
+     * Hostname of the TGen client.
+     */
+    String name;
+
+    /**
+     * Hostname of the TGen server.
+     */
+    String peername;
+
+    /**
+     * Number of expected payload bytes in the response.
+     */
+    String recvsize;
+  }
+
+  /**
+   * Elapsed time until reaching given substeps in a measurement.
+   */
+  static class TimeInfo {
+
+    /**
+     * Elapsed microseconds until the TGen client has sent the command to the
+     * TGen server, or -1 if missing (step 7).
+     */
+    @JsonProperty("usecs-to-command")
+    String usecsToCommand;
+
+    /**
+     * Elapsed microseconds until the TGen client has received the last payload
+     * byte, or -1 if missing (step 10).
+     */
+    @JsonProperty("usecs-to-last-byte-recv")
+    String usecsToLastByteRecv;
+
+    /**
+     * Elapsed microseconds until the TGen client has received the SOCKS choice
+     * from the Tor client, or -1 if missing (step 4).
+     */
+    @JsonProperty("usecs-to-proxy-choice")
+    String usecsToProxyChoice;
+
+    /**
+     * Elapsed microseconds until the TGen client has sent the SOCKS request to
+     * the Tor client, or -1 if missing (step 5).
+     */
+    @JsonProperty("usecs-to-proxy-request")
+    String usecsToProxyRequest;
+
+    /**
+     * Elapsed microseconds until the TGen client has received the SOCKS
+     * response from the Tor client, or -1 if missing (step 6).
+     */
+    @JsonProperty("usecs-to-proxy-response")
+    String usecsToProxyResponse;
+
+    /**
+     * Elapsed microseconds until the TGen client has received the command from
+     * the TGen server, or -1 if missing (step 8).
+     */
+    @JsonProperty("usecs-to-response")
+    String usecsToResponse;
+
+    /**
+     * Elapsed microseconds until the TGen client has connected to the Tor
+     * client's SOCKS port, or -1 if missing (step 2).
+     */
+    @JsonProperty("usecs-to-socket-connect")
+    String usecsToSocketConnect;
+
+    /**
+     * Elapsed microseconds until the TGen client has opened a TCP connection
+     * to the Tor client's SOCKS port, or -1 if missing (step 1).
+     */
+    @JsonProperty("usecs-to-socket-create")
+    String usecsToSocketCreate;
+  }
+
+  /**
+   * Information about the TGen transport.
+   */
+  static class TransportInfo {
+
+    /**
+     * Local host name, IP address, and TCP port.
+     */
+    String local;
+
+    /**
+     * Proxy host name, IP address, and TCP port.
+     */
+    String proxy;
+
+    /**
+     * Remote host name, IP address, and TCP port.
+     */
+    String remote;
+  }
+
   /**
    * Measurement data obtained from client-side {@code tor} controller event
    * logs.
diff --git a/src/main/java/org/torproject/descriptor/onionperf/TorperfResultsBuilder.java b/src/main/java/org/torproject/descriptor/onionperf/TorperfResultsBuilder.java
index a99257a..766f7ad 100644
--- a/src/main/java/org/torproject/descriptor/onionperf/TorperfResultsBuilder.java
+++ b/src/main/java/org/torproject/descriptor/onionperf/TorperfResultsBuilder.java
@@ -15,6 +15,19 @@ import java.util.TreeMap;
  */
 public class TorperfResultsBuilder {
 
+  /**
+   * Error code parts, to be formatted as
+   * {@code ERRORCODE=part_1/part_2/.../part_n}.
+   */
+  private List<String> errorCodeParts = null;
+
+  void addErrorCodePart(String errorCodePart) {
+    if (null == errorCodeParts) {
+      this.errorCodeParts = new ArrayList<>();
+    }
+    this.errorCodeParts.add(errorCodePart);
+  }
+
   /**
    * Key-value pairs to be formatted as Torperf results line.
    */
@@ -77,6 +90,27 @@ public class TorperfResultsBuilder {
     }
   }
 
+  /**
+   * Add a timestamp value as the sum of a double value, formatted as seconds
+   * since the epoch with two decimal places, and a string value, formatted as
+   * microseconds since the first value, unless either of the two summands is
+   * {@code null} or negative.
+   *
+   * @param key Key.
+   * @param unixTsStart First summand representing seconds since the epoch.
+   * @param elapsedMicroseconds Second summand representing microseconds
+   *     elapsed since the first summand.
+   */
+  void addTimestamp(String key, Double unixTsStart,
+      String elapsedMicroseconds) {
+    if (null != unixTsStart && unixTsStart >= 0.0 && null != elapsedMicroseconds
+        && !elapsedMicroseconds.startsWith("-")) {
+      double elapsedSeconds = Double.parseDouble(elapsedMicroseconds)
+          / 1000000.0;
+      this.addTimestamp(key, unixTsStart, elapsedSeconds);
+    }
+  }
+
   /**
    * Build the Torperf results line by putting together all key-value pairs as
    * {@code "key=value"}, separated by spaces, prefixed by an annotation line
@@ -89,6 +123,10 @@ public class TorperfResultsBuilder {
     StringBuilder result = new StringBuilder();
     result.append("@type torperf 1.1\r\n");
     List<String> torperfResultsParts = new ArrayList<>();
+    if (null != this.errorCodeParts) {
+      String errorCode = String.join("/", errorCodeParts);
+      this.addString("ERRORCODE", errorCode);
+    }
     for (Map.Entry<String, String> keyValuePairsEntry
         : this.keyValuePairs.entrySet()) {
       torperfResultsParts.add(String.format("%s=%s",
diff --git a/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java b/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java
index 51e0896..f8d9f90 100644
--- a/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java
+++ b/src/test/java/org/torproject/descriptor/onionperf/OnionPerfAnalysisConverterTest.java
@@ -3,6 +3,7 @@
 
 package org.torproject.descriptor.onionperf;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
@@ -16,6 +17,8 @@ import org.junit.Test;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
 
 public class OnionPerfAnalysisConverterTest {
 
@@ -89,6 +92,10 @@ public class OnionPerfAnalysisConverterTest {
   @Test
   public void testAsTorperfResults() throws IOException,
       DescriptorParseException {
+    List<String> expectedTorperfResults = new ArrayList<>();
+    expectedTorperfResults.add(torperfResultTransfer1m1);
+    expectedTorperfResults.add(torperfResultTransfer1m3);
+    expectedTorperfResults.add(torperfResultTransfer50k2);
     URL resouce = getClass().getClassLoader().getResource(
         "onionperf/onionperf.analysis.json.xz");
     assertNotNull(resouce);
@@ -103,12 +110,75 @@ public class OnionPerfAnalysisConverterTest {
       String formattedTorperfResult
           = new String(descriptor.getRawDescriptorBytes()).trim();
       assertNotNull(formattedTorperfResult);
-      assertTrue(String.format("Unrecognized formatted Torperf result: %s",
-          formattedTorperfResult),
-          formattedTorperfResult.equals(torperfResultTransfer1m1)
-          || formattedTorperfResult.equals(torperfResultTransfer1m3)
-          || formattedTorperfResult.equals(torperfResultTransfer50k2));
+      String expectedTorperfResult = expectedTorperfResults.remove(0);
+      assertEquals(expectedTorperfResult, formattedTorperfResult);
     }
+    assertTrue(expectedTorperfResults.isEmpty());
+  }
+
+  private final String torperfResultStream155
+      = "BUILDTIMES=0.62,0.90,1.26 CIRC_ID=1008 CONNECT=1594633118.41 "
+      + "DATACOMPLETE=1594633145.46 DATAPERC0=1594633123.64 "
+      + "DATAPERC10=1594633125.57 DATAPERC100=1594633145.46 "
+      + "DATAPERC20=1594633127.56 DATAPERC30=1594633129.56 "
+      + "DATAPERC40=1594633132.20 DATAPERC50=1594633134.29 "
+      + "DATAPERC60=1594633136.41 DATAPERC70=1594633138.84 "
+      + "DATAPERC80=1594633140.76 DATAPERC90=1594633142.88 "
+      + "DATAREQUEST=1594633123.01 DATARESPONSE=1594633123.64 DIDTIMEOUT=0 "
+      + "ENDPOINTLOCAL=localhost:127.0.0.1:46222 "
+      + "ENDPOINTPROXY=localhost:127.0.0.1:29849 "
+      + "ENDPOINTREMOTE=jjsvldkkjd3lxy6gljy6xmhrn5mnirzgj2mrftrx2bf3n4bbx53fxza"
+      + "d.onion:0.0.0.0:8080 FILESIZE=5242880 HOSTNAMELOCAL=op-nl-test1 "
+      + "HOSTNAMEREMOTE=op-nl-test1 LAUNCH=1594633119.06 "
+      + "NEGOTIATE=1594633118.41 PARTIAL10240=1594633123.80 "
+      + "PARTIAL102400=1594633124.11 PARTIAL1048576=1594633127.56 "
+      + "PARTIAL20480=1594633123.85 PARTIAL204800=1594633124.26 "
+      + "PARTIAL2097152=1594633132.20 PARTIAL51200=1594633123.96 "
+      + "PARTIAL512000=1594633125.50 PARTIAL5242880=1594633145.46 "
+      + "PATH=$65888719E2F619E6198F1045A93AF0176C05354D,"
+      + "$3043C1A6DF23AFEDC8FD3C0671FADCEDFF6D3429,"
+      + "$8E5F4EE45E0631A60E59CAA42E1464FD7120459D QUANTILE=0.8 "
+      + "READBYTES=5242990 REQUEST=1594633118.42 RESPONSE=1594633123.01 "
+      + "SOCKET=1594633118.41 SOURCE=op-nl-test1 SOURCEADDRESS=unknown "
+      + "START=1594633118.41 TIMEOUT=1500 USED_AT=1594633145.46 USED_BY=1498 "
+      + "WRITEBYTES=2174";
+
+  private final String torperfResultStream156
+      = "CONNECT=1594633539.83 DATACOMPLETE=0.0 DATAREQUEST=0.0 "
+      + "DATARESPONSE=0.0 DIDTIMEOUT=1 ENDPOINTLOCAL=localhost:127.0.0.1:46246 "
+      + "ENDPOINTPROXY=localhost:127.0.0.1:29849 "
+      + "ENDPOINTREMOTE=jjsvldkkjd3lxy6gljy6xmhrn5mnirzgj2mrftrx2bf3n4bbx53fxza"
+      + "d.onion:0.0.0.0:8080 ERRORCODE=TOR/TIMEOUT FILESIZE=5242880 "
+      + "HOSTNAMELOCAL=op-nl-test1 HOSTNAMEREMOTE=(null) "
+      + "NEGOTIATE=1594633539.83 READBYTES=0 REQUEST=1594633539.83 "
+      + "RESPONSE=0.0 SOCKET=1594633539.83 SOURCE=op-nl-test1 "
+      + "SOURCEADDRESS=unknown START=1594633539.83 USED_AT=1594633539.83 "
+      + "USED_BY=1510 WRITEBYTES=0";
+
+  @Test
+  public void testAsTorperfResultsVersion3() throws IOException,
+      DescriptorParseException {
+    List<String> expectedTorperfResults = new ArrayList<>();
+    expectedTorperfResults.add(torperfResultStream155);
+    expectedTorperfResults.add(torperfResultStream156);
+    URL resouce = getClass().getClassLoader().getResource(
+        "onionperf/2020-07-13.op-nl-test1.onionperf.analysis.json.xz");
+    assertNotNull(resouce);
+    InputStream compressedInputStream = resouce.openStream();
+    assertNotNull(compressedInputStream);
+    byte[] rawDescriptorBytes = IOUtils.toByteArray(compressedInputStream);
+    OnionPerfAnalysisConverter onionPerfAnalysisConverter
+        = new OnionPerfAnalysisConverter(rawDescriptorBytes, null);
+    for (Descriptor descriptor
+        : onionPerfAnalysisConverter.asTorperfResults()) {
+      assertTrue(descriptor instanceof TorperfResult);
+      String formattedTorperfResult
+          = new String(descriptor.getRawDescriptorBytes()).trim();
+      assertNotNull(formattedTorperfResult);
+      String expectedTorperfResult = expectedTorperfResults.remove(0);
+      assertEquals(expectedTorperfResult, formattedTorperfResult);
+    }
+    assertTrue(expectedTorperfResults.isEmpty());
   }
 }
 
diff --git a/src/test/resources/onionperf/2020-07-13.op-nl-test1.onionperf.analysis.json.xz b/src/test/resources/onionperf/2020-07-13.op-nl-test1.onionperf.analysis.json.xz
new file mode 100644
index 0000000..78be8c5
Binary files /dev/null and b/src/test/resources/onionperf/2020-07-13.op-nl-test1.onionperf.analysis.json.xz differ



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits