[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [collector/master] Add source name to locally imported bandwidth files.



commit abde3f7591117060ccf1afc9b26531d79619301f
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Wed Jul 10 10:34:09 2019 +0200

    Add source name to locally imported bandwidth files.
    
    Implements #30219.
---
 CHANGELOG.md                                                 |  7 +++++++
 .../metrics/collector/relaydescs/ArchiveReader.java          |  2 +-
 .../metrics/collector/relaydescs/ArchiveWriter.java          | 12 +++++++++---
 .../collector/relaydescs/CachedRelayDescriptorReader.java    |  6 +++---
 .../collector/relaydescs/RelayDescriptorDownloader.java      |  4 ++--
 .../metrics/collector/relaydescs/RelayDescriptorParser.java  |  7 +++++--
 src/main/resources/collector.properties                      |  5 ++++-
 7 files changed, 31 insertions(+), 12 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4682fef..613ef71 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,10 @@
+# Changes in version 1.10.0 - 2019-??-??
+
+ * Medium changes
+   - Changed local import of bandwidth files to include the parent
+     directory name as @source annotation and to the filename.
+
+
 # Changes in version 1.9.1 - 2019-05-29
 
  * Medium changes
diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java
index 27010a9..fee46c9 100644
--- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java
+++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveReader.java
@@ -141,7 +141,7 @@ public class ArchiveReader {
             }
             bis.close();
             byte[] allData = baos.toByteArray();
-            boolean stored = this.rdp.parse(allData);
+            boolean stored = this.rdp.parse(allData, pop);
             if (!stored) {
               filesToRetry.add(pop);
               continue;
diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
index e1279ee..911ac2a 100644
--- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
+++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
@@ -745,18 +745,24 @@ public class ArchiveWriter extends CollecTorMain {
 
   /** Stores a bandwidth file to disk. */
   void storeBandwidthFile(byte[] data, LocalDateTime fileCreatedOrTimestamp,
-      String bandwidthFileDigest) {
+      String sourceName, String bandwidthFileDigest) {
     DateTimeFormatter printFormat = DateTimeFormatter
         .ofPattern("uuuu/MM/dd/uuuu-MM-dd-HH-mm-ss").withZone(ZoneOffset.UTC);
     File tarballFile = Paths.get(this.outputDirectory, "bandwidth",
         fileCreatedOrTimestamp.format(printFormat) + "-bandwidth-"
+        + (null == sourceName ? "" : (sourceName + "-"))
         + bandwidthFileDigest).toFile();
+    StringBuilder sb = new StringBuilder();
+    sb.append(Annotation.BandwidthFile.toString());
+    if (null != sourceName) {
+      sb.append("@source ").append(sourceName).append('\n');
+    }
     boolean tarballFileExistedBefore = tarballFile.exists();
     File rsyncFile = Paths.get(recentPathName, RELAY_DESCRIPTORS, "bandwidths",
         tarballFile.getName()).toFile();
     File[] outputFiles = new File[] { tarballFile, rsyncFile };
-    if (this.store(Annotation.BandwidthFile.bytes(), data, outputFiles, null)) {
-      this.storedVotesCounter++;
+    if (this.store(sb.toString().getBytes(), data, outputFiles, null)) {
+      this.storedBandwidthsCounter++;
     }
     if (!tarballFileExistedBefore
         && this.nowLocalDateTime.isAfter(fileCreatedOrTimestamp.plusDays(3L))) {
diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java
index 3e1bba4..0308f05 100644
--- a/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java
+++ b/src/main/java/org/torproject/metrics/collector/relaydescs/CachedRelayDescriptorReader.java
@@ -158,7 +158,7 @@ public class CachedRelayDescriptorReader {
                 allData));
             if (!this.lastImportHistory.contains(digest)
                 && !this.currentImportHistory.contains(digest)) {
-              this.rdp.parse(allData);
+              this.rdp.parse(allData, null);
             } else {
               this.dumpStats.append(" (skipped)");
             }
@@ -183,7 +183,7 @@ public class CachedRelayDescriptorReader {
                     rawNetworkStatusBytes));
                 if (!this.lastImportHistory.contains(digest)
                     && !this.currentImportHistory.contains(digest)) {
-                  this.rdp.parse(rawNetworkStatusBytes);
+                  this.rdp.parse(rawNetworkStatusBytes, null);
                   parsedNum++;
                 } else {
                   skippedNum++;
@@ -229,7 +229,7 @@ public class CachedRelayDescriptorReader {
                   descBytes));
               if (!this.lastImportHistory.contains(digest)
                   && !this.currentImportHistory.contains(digest)) {
-                this.rdp.parse(descBytes);
+                this.rdp.parse(descBytes, null);
                 parsedNum++;
               } else {
                 skippedNum++;
diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java
index 5a241f4..f1179d8 100644
--- a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java
+++ b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorDownloader.java
@@ -899,7 +899,7 @@ public class RelayDescriptorDownloader {
     int receivedDescriptors = 0;
     if (allData != null) {
       if (resource.startsWith("/tor/status-vote/")) {
-        this.rdp.parse(allData);
+        this.rdp.parse(allData, null);
         receivedDescriptors = 1;
       } else if (resource.startsWith("/tor/server/")
           || resource.startsWith("/tor/extra/")) {
@@ -933,7 +933,7 @@ public class RelayDescriptorDownloader {
           end += endToken.length();
           byte[] descBytes = new byte[end - start];
           System.arraycopy(allData, start, descBytes, 0, end - start);
-          this.rdp.parse(descBytes);
+          this.rdp.parse(descBytes, null);
           receivedDescriptors++;
         }
       } else if (resource.startsWith("/tor/micro/")) {
diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java
index 113ac77..a610ed9 100644
--- a/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java
+++ b/src/main/java/org/torproject/metrics/collector/relaydescs/RelayDescriptorParser.java
@@ -10,6 +10,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.BufferedReader;
+import java.io.File;
 import java.io.IOException;
 import java.io.StringReader;
 import java.text.ParseException;
@@ -75,7 +76,7 @@ public class RelayDescriptorParser {
    * forwards them to the archive writer to store them to disk, and tells
    * the relay descriptor downloader and archive reader about the
    * contained descriptor and all referenced descriptors. */
-  public boolean parse(byte[] data) {
+  public boolean parse(byte[] data, File containingFile) {
     boolean stored = false;
     try {
       /* Convert descriptor to ASCII for parsing. This means we'll lose
@@ -357,7 +358,9 @@ public class RelayDescriptorParser {
             fileCreatedOrTimestamp = LocalDateTime.of(2000, 1, 1, 0, 0, 0);
           }
         }
-        this.aw.storeBandwidthFile(data, fileCreatedOrTimestamp,
+        String sourceName = containingFile == null ? null
+            : containingFile.getParentFile().getName();
+        this.aw.storeBandwidthFile(data, fileCreatedOrTimestamp, sourceName,
             DigestUtils.sha256Hex(data).toUpperCase());
         stored = true;
       }
diff --git a/src/main/resources/collector.properties b/src/main/resources/collector.properties
index 787f188..292e876 100644
--- a/src/main/resources/collector.properties
+++ b/src/main/resources/collector.properties
@@ -86,7 +86,10 @@ RelaySyncOrigins = https://collector.torproject.org
 ## Path to Tor data directory to read cached-* files from
 ## the listed path(s). If there is more that one separated by comma.
 RelayCacheOrigins = in/relay-descriptors/cacheddesc/
-## Relative path to directory to import directory archives from
+## Relative path to directory to import directory archives from.
+## Note that when importing bandwidth files, the parent directory name is
+## included in the @source annotation and in the file name. Recommended
+## source names are nicknames of directory authorities using these files.
 RelayLocalOrigins = in/relay-descriptors/archives/
 #
 ## Keep a history of imported directory archive files to know which files

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits