[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [collector/master] Archive bandwidth files in relaydescs module.



commit dcbac68b48ae31b1bfbabab7a9c32f5577b78571
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Thu May 2 22:02:23 2019 +0200

    Archive bandwidth files in relaydescs module.
    
    Also update to metrics-lib 2.6.1.
    
    Implements #30218.
---
 CHANGELOG.md                                       |  7 ++++
 build.xml                                          |  4 +--
 .../metrics/collector/conf/Annotation.java         |  1 +
 .../collector/relaydescs/ArchiveWriter.java        | 42 +++++++++++++++++++++-
 .../relaydescs/RelayDescriptorDownloader.java      | 24 ++++++++++---
 .../relaydescs/RelayDescriptorParser.java          | 42 ++++++++++++++++++++++
 src/main/resources/create-tarballs.sh              |  7 ++++
 7 files changed, 120 insertions(+), 7 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0307748..0e592ae 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,10 @@
+# Changes in version 1.9.0 - 2019-05-??
+
+ * Medium changes
+   - Archive bandwidth files in relaydescs module.
+   - Update to metrics-lib 2.6.1.
+
+
 # Changes in version 1.8.0 - 2018-10-11
 
  * Medium changes
diff --git a/build.xml b/build.xml
index 5cea51a..5874013 100644
--- a/build.xml
+++ b/build.xml
@@ -11,7 +11,7 @@
   <property name="release.version" value="1.8.0-dev" />
   <property name="project-main-class" value="org.torproject.metrics.collector.Main" />
   <property name="name" value="collector"/>
-  <property name="metricslibversion" value="2.4.0" />
+  <property name="metricslibversion" value="2.6.1" />
   <property name="jarincludes" value="collector.properties logback.xml" />
 
   <patternset id="runtime" >
@@ -21,7 +21,7 @@
       <include name="jackson-core-2.8.6.jar"/>
       <include name="jackson-databind-2.8.6.jar"/>
       <include name="xz-1.6.jar"/>
-      <include name="metrics-lib-${metricslibversion}.jar"/>
+      <include name="metrics-lib-${metricslibversion}-thin.jar"/>
       <include name="logback-core-1.1.9.jar" />
       <include name="logback-classic-1.1.9.jar" />
       <include name="slf4j-api-1.7.22.jar" />
diff --git a/src/main/java/org/torproject/metrics/collector/conf/Annotation.java b/src/main/java/org/torproject/metrics/collector/conf/Annotation.java
index f90516b..2e47df0 100644
--- a/src/main/java/org/torproject/metrics/collector/conf/Annotation.java
+++ b/src/main/java/org/torproject/metrics/collector/conf/Annotation.java
@@ -6,6 +6,7 @@ package org.torproject.metrics.collector.conf;
 /** This enum contains all currently valid descriptor annotations. */
 public enum Annotation {
 
+  BandwidthFile("@type bandwidth-file 1.0\n"),
   BridgeExtraInfo("@type bridge-extra-info 1.3\n"),
   BridgeServer("@type bridge-server-descriptor 1.2\n"),
   Cert("@type dir-key-certificate-3 1.0\n"),
diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
index 966b649..e1279ee 100644
--- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
+++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
@@ -3,6 +3,7 @@
 
 package org.torproject.metrics.collector.relaydescs;
 
+import org.torproject.descriptor.BandwidthFile;
 import org.torproject.descriptor.Descriptor;
 import org.torproject.descriptor.DescriptorParser;
 import org.torproject.descriptor.DescriptorSourceFactory;
@@ -33,6 +34,10 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.HashMap;
@@ -44,6 +49,7 @@ import java.util.SortedSet;
 import java.util.Stack;
 import java.util.TimeZone;
 import java.util.TreeMap;
+import java.util.TreeSet;
 
 public class ArchiveWriter extends CollecTorMain {
 
@@ -51,12 +57,15 @@ public class ArchiveWriter extends CollecTorMain {
       ArchiveWriter.class);
 
   private long now = System.currentTimeMillis();
+  private LocalDateTime nowLocalDateTime
+      = LocalDateTime.ofInstant(Instant.ofEpochMilli(this.now), ZoneOffset.UTC);
   private String outputDirectory;
   private String rsyncCatString;
   private DescriptorParser descriptorParser;
   private int storedConsensusesCounter = 0;
   private int storedMicrodescConsensusesCounter = 0;
   private int storedVotesCounter = 0;
+  private int storedBandwidthsCounter = 0;
   private int storedCertsCounter = 0;
   private int storedServerDescriptorsCounter = 0;
   private int storedExtraInfoDescriptorsCounter = 0;
@@ -74,6 +83,8 @@ public class ArchiveWriter extends CollecTorMain {
   private SortedMap<Long, Set<String>> storedExtraInfoDescriptors =
       new TreeMap<>();
   private SortedMap<Long, Set<String>> storedMicrodescriptors = new TreeMap<>();
+  private SortedMap<LocalDateTime, Set<String>> storedBandwidths
+      = new TreeMap<>();
 
   private File storedServerDescriptorsFile;
   private File storedExtraInfoDescriptorsFile;
@@ -103,6 +114,8 @@ public class ArchiveWriter extends CollecTorMain {
         RelayServerDescriptor.class);
     this.mapPathDescriptors.put("recent/relay-descriptors/extra-infos",
         RelayExtraInfoDescriptor.class);
+    this.mapPathDescriptors.put("recent/relay-descriptors/bandwidths",
+        BandwidthFile.class);
   }
 
   @Override
@@ -203,6 +216,7 @@ public class ArchiveWriter extends CollecTorMain {
     this.storedConsensuses.clear();
     this.storedMicrodescConsensuses.clear();
     this.storedVotes.clear();
+    this.storedBandwidths.clear();
     this.storedServerDescriptors.clear();
     this.storedExtraInfoDescriptors.clear();
     this.storedMicrodescriptors.clear();
@@ -299,7 +313,8 @@ public class ArchiveWriter extends CollecTorMain {
         .append(this.storedConsensusesCounter).append(" consensus(es), ")
         .append(this.storedMicrodescConsensusesCounter).append(" microdesc ")
         .append("consensus(es), ").append(this.storedVotesCounter)
-        .append(" vote(s), ").append(this.storedCertsCounter)
+        .append(" vote(s), ").append(this.storedBandwidthsCounter)
+        .append(" bandwidth file(s), ").append(this.storedCertsCounter)
         .append(" certificate(s), ").append(this.storedServerDescriptorsCounter)
         .append(" server descriptor(s), ")
         .append(this.storedExtraInfoDescriptorsCounter).append(" extra-info ")
@@ -309,6 +324,7 @@ public class ArchiveWriter extends CollecTorMain {
     this.storedConsensusesCounter = 0;
     this.storedMicrodescConsensusesCounter = 0;
     this.storedVotesCounter = 0;
+    this.storedBandwidthsCounter = 0;
     this.storedCertsCounter = 0;
     this.storedServerDescriptorsCounter = 0;
     this.storedExtraInfoDescriptorsCounter = 0;
@@ -727,6 +743,30 @@ public class ArchiveWriter extends CollecTorMain {
     }
   }
 
+  /** Stores a bandwidth file to disk. */
+  void storeBandwidthFile(byte[] data, LocalDateTime fileCreatedOrTimestamp,
+      String bandwidthFileDigest) {
+    DateTimeFormatter printFormat = DateTimeFormatter
+        .ofPattern("uuuu/MM/dd/uuuu-MM-dd-HH-mm-ss").withZone(ZoneOffset.UTC);
+    File tarballFile = Paths.get(this.outputDirectory, "bandwidth",
+        fileCreatedOrTimestamp.format(printFormat) + "-bandwidth-"
+        + bandwidthFileDigest).toFile();
+    boolean tarballFileExistedBefore = tarballFile.exists();
+    File rsyncFile = Paths.get(recentPathName, RELAY_DESCRIPTORS, "bandwidths",
+        tarballFile.getName()).toFile();
+    File[] outputFiles = new File[] { tarballFile, rsyncFile };
+    if (this.store(Annotation.BandwidthFile.bytes(), data, outputFiles, null)) {
+      this.storedVotesCounter++;
+    }
+    if (!tarballFileExistedBefore
+        && this.nowLocalDateTime.isAfter(fileCreatedOrTimestamp.plusDays(3L))) {
+      this.storedBandwidths.putIfAbsent(fileCreatedOrTimestamp,
+          new TreeSet<>());
+      this.storedBandwidths.get(fileCreatedOrTimestamp)
+          .add(bandwidthFileDigest);
+    }
+  }
+
   /** Stores a key certificate to disk. */
   public void storeCertificate(byte[] data, String fingerprint,
       long published) {
diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java
index 4764a4b..5a241f4 100644
--- a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java
+++ b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java
@@ -258,6 +258,8 @@ public class RelayDescriptorDownloader {
 
   private int requestedVotes = 0;
 
+  private int requestedBandwidthFiles = 0;
+
   private int requestedMissingServerDescriptors = 0;
 
   private int requestedAllServerDescriptors = 0;
@@ -274,6 +276,8 @@ public class RelayDescriptorDownloader {
 
   private int downloadedVotes = 0;
 
+  private int downloadedBandwidthFiles = 0;
+
   private int downloadedMissingServerDescriptors = 0;
 
   private int downloadedAllServerDescriptors = 0;
@@ -729,6 +733,14 @@ public class RelayDescriptorDownloader {
           }
         }
 
+        /* Now try to download the bandwidth file, regardless of whether this
+         * authority might provide one or when we last downloaded a bandwidth
+         * file from it. */
+        this.requestedBandwidthFiles++;
+        this.downloadedBandwidthFiles +=
+            this.downloadResourceFromAuthority(authority,
+            "/tor/status-vote/next/bandwidth");
+
         /* Download either all server and extra-info descriptors or only
          * those that we're missing. Start with server descriptors, then
          * request extra-info descriptors. Finally, request missing
@@ -886,7 +898,7 @@ public class RelayDescriptorDownloader {
         allData == null ? 0 : allData.length);
     int receivedDescriptors = 0;
     if (allData != null) {
-      if (resource.startsWith("/tor/status-vote/current/")) {
+      if (resource.startsWith("/tor/status-vote/")) {
         this.rdp.parse(allData);
         receivedDescriptors = 1;
       } else if (resource.startsWith("/tor/server/")
@@ -1067,11 +1079,13 @@ public class RelayDescriptorDownloader {
         this.newMissingServerDescriptors, this.newMissingExtraInfoDescriptors,
         this.newMissingMicrodescriptors);
     logger.info("We requested {} consensus(es), {} microdesc consensus(es), "
-        + "{} vote(s), {} missing server descriptor(s), {} times all server "
+        + "{} vote(s), {} bandwidth file(s), {} missing server descriptor(s), "
+        + "{} times all server "
         + "descriptors, {} missing extra-info descriptor(s), {} times all "
         + "extra-info descriptors, and {} missing microdescriptor(s) from the "
         + "directory authorities.", this.requestedConsensuses,
         this.requestedMicrodescConsensuses, this.requestedVotes,
+        this.requestedBandwidthFiles,
         this.requestedMissingServerDescriptors,
         this.requestedAllServerDescriptors,
         this.requestedMissingExtraInfoDescriptors,
@@ -1085,12 +1099,14 @@ public class RelayDescriptorDownloader {
     logger.info("We sent these numbers of requests to the directory "
         + "authorities:{}", sb.toString());
     logger.info("We successfully downloaded {} consensus(es), {} microdesc "
-        + "consensus(es), {} vote(s), {} missing server descriptor(s), {} "
+        + "consensus(es), {} vote(s), {} bandwidth file(s), "
+        + "{} missing server descriptor(s), {} "
         + "server descriptor(s) when downloading all descriptors, {} missing "
         + "extra-info descriptor(s), {} extra-info descriptor(s) when "
         + "downloading all descriptors, and {} missing microdescriptor(s).",
         this.downloadedConsensuses, this.downloadedMicrodescConsensuses,
-        this.downloadedVotes, this.downloadedMissingServerDescriptors,
+        this.downloadedVotes, this.downloadedBandwidthFiles,
+        this.downloadedMissingServerDescriptors,
         this.downloadedAllServerDescriptors,
         this.downloadedMissingExtraInfoDescriptors,
         this.downloadedAllExtraInfoDescriptors,
diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java
index 5224a61..113ac77 100644
--- a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java
+++ b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java
@@ -14,6 +14,10 @@ import java.io.IOException;
 import java.io.StringReader;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeParseException;
 import java.util.SortedSet;
 import java.util.TimeZone;
 import java.util.TreeSet;
@@ -318,6 +322,44 @@ public class RelayDescriptorParser {
          * time(s) of microdesc consensuses containing them, because we
          * don't know which month directories to put them in.  Have to use
          * storeMicrodescriptor below. */
+      } else if (line.matches("[0-9]{10}")) {
+        /* The following code is a much more lenient version of the parser in
+         * metrics-lib that we need for storing a bandwidth file even if
+         * metrics-lib has trouble verifying its format. As in metrics-lib,
+         * identifying bandwidth files by a 10-digit timestamp in the first line
+         * breaks with files generated before 2002 or after 2286 and when the
+         * next descriptor identifier starts with just a timestamp in the first
+         * line rather than a document type identifier. */
+        String timestampLine = line;
+        LocalDateTime fileCreatedOrTimestamp = null;
+        try {
+          while ((line = br.readLine()) != null) {
+            if (line.startsWith("file_created=")) {
+              fileCreatedOrTimestamp = LocalDateTime.parse(
+                  line.substring("file_created=".length()));
+              break;
+            } else if (line.startsWith("bw=") || line.contains(" bw=")
+                || "====".equals(line) || "=====".equals(line)) {
+              break;
+            }
+          }
+        } catch (IOException | DateTimeParseException e) {
+          /* Fall back to using timestamp in first line. */
+        }
+        if (null == fileCreatedOrTimestamp) {
+          try {
+            fileCreatedOrTimestamp = LocalDateTime.ofInstant(
+                Instant.ofEpochSecond(Long.parseLong(timestampLine)),
+                ZoneOffset.UTC);
+          } catch (NumberFormatException | DateTimeParseException e) {
+            logger.warn("Could not parse timestamp or file_created time from "
+                + "bandwidth file. Storing with timestamp 2000-01-01 00:00:00");
+            fileCreatedOrTimestamp = LocalDateTime.of(2000, 1, 1, 0, 0, 0);
+          }
+        }
+        this.aw.storeBandwidthFile(data, fileCreatedOrTimestamp,
+            DigestUtils.sha256Hex(data).toUpperCase());
+        stored = true;
       }
       br.close();
     } catch (IOException | ParseException e) {
diff --git a/src/main/resources/create-tarballs.sh b/src/main/resources/create-tarballs.sh
index d247c52..7e4668a 100755
--- a/src/main/resources/create-tarballs.sh
+++ b/src/main/resources/create-tarballs.sh
@@ -47,6 +47,8 @@ TARBALLS=(
   consensuses-$YEARTWO-$MONTHTWO
   votes-$YEARONE-$MONTHONE
   votes-$YEARTWO-$MONTHTWO
+  bandwidths-$YEARONE-$MONTHONE
+  bandwidths-$YEARTWO-$MONTHTWO
   server-descriptors-$YEARONE-$MONTHONE
   server-descriptors-$YEARTWO-$MONTHTWO
   extra-infos-$YEARONE-$MONTHONE
@@ -72,6 +74,8 @@ DIRECTORIES=(
   $OUTDIR/relay-descriptors/consensus/$YEARTWO/$MONTHTWO
   $OUTDIR/relay-descriptors/vote/$YEARONE/$MONTHONE/
   $OUTDIR/relay-descriptors/vote/$YEARTWO/$MONTHTWO/
+  $OUTDIR/relay-descriptors/bandwidth/$YEARONE/$MONTHONE/
+  $OUTDIR/relay-descriptors/bandwidth/$YEARTWO/$MONTHTWO/
   $OUTDIR/relay-descriptors/server-descriptor/$YEARONE/$MONTHONE/
   $OUTDIR/relay-descriptors/server-descriptor/$YEARTWO/$MONTHTWO/
   $OUTDIR/relay-descriptors/extra-info/$YEARONE/$MONTHONE/
@@ -156,6 +160,9 @@ ln -f -s -t $ARCHIVEDIR/relay-descriptors/tor/ $TARBALLTARGETDIR/tor-20??-??.tar
 mkdir -p $ARCHIVEDIR/relay-descriptors/votes/
 ln -f -s -t $ARCHIVEDIR/relay-descriptors/votes/ $TARBALLTARGETDIR/votes-20??-??.tar.xz
 
+mkdir -p $ARCHIVEDIR/relay-descriptors/bandwidths/
+ln -f -s -t $ARCHIVEDIR/relay-descriptors/bandwidths/ $TARBALLTARGETDIR/bandwidths-20??-??.tar.xz
+
 mkdir -p $ARCHIVEDIR/torperf/
 ln -f -s -t $ARCHIVEDIR/torperf/ $TARBALLTARGETDIR/torperf-20??-??.tar.xz
 

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits