[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [collector/master] Include microdescriptors when syncing.
commit a1b8ebb9545b148524c6e3db2a1601f228784905
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Wed Dec 9 22:01:21 2020 +0100
Include microdescriptors when syncing.
---
CHANGELOG.md | 3 ++
build.xml | 2 +-
.../persist/MicrodescriptorPersistence.java | 36 ++++++++++++++++++++++
.../collector/relaydescs/ArchiveWriter.java | 11 +++++--
.../metrics/collector/sync/SyncPersistence.java | 10 ++++++
5 files changed, 58 insertions(+), 4 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2928937..22d3517 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,9 @@
- Clean up descriptors written to the `out/` directory by deleting
files that are older than seven weeks.
- Correctly index files that are moved away and back.
+ - Include microdescriptors when syncing from another CollecTor
+ instance.
+ - Update to metrics-lib 2.15.0.
# Changes in version 1.16.1 - 2020-08-16
diff --git a/build.xml b/build.xml
index 5dcb29d..f2bda4e 100644
--- a/build.xml
+++ b/build.xml
@@ -12,7 +12,7 @@
<property name="release.version" value="1.16.1-dev" />
<property name="project-main-class" value="org.torproject.metrics.collector.Main" />
<property name="name" value="collector"/>
- <property name="metricslibversion" value="2.14.0" />
+ <property name="metricslibversion" value="2.15.0" />
<property name="jarincludes" value="collector.properties logback.xml" />
<patternset id="runtime" >
diff --git a/src/main/java/org/torproject/metrics/collector/persist/MicrodescriptorPersistence.java b/src/main/java/org/torproject/metrics/collector/persist/MicrodescriptorPersistence.java
new file mode 100644
index 0000000..41929af
--- /dev/null
+++ b/src/main/java/org/torproject/metrics/collector/persist/MicrodescriptorPersistence.java
@@ -0,0 +1,36 @@
+/* Copyright 2020 The Tor Project
+ * See LICENSE for licensing information */
+
+package org.torproject.metrics.collector.persist;
+
+import org.torproject.descriptor.Microdescriptor;
+import org.torproject.metrics.collector.conf.Annotation;
+
+import java.nio.file.Paths;
+
+public class MicrodescriptorPersistence
+ extends DescriptorPersistence<Microdescriptor> {
+
+ private static final String RELAY_DESCRIPTORS = "relay-descriptors";
+
+ public MicrodescriptorPersistence(Microdescriptor descriptor, long received,
+ String year, String month) {
+ super(descriptor, Annotation.Microdescriptor.bytes());
+ calculatePaths(received, year, month);
+ }
+
+ private void calculatePaths(long received, String year, String month) {
+ String file = PersistenceUtils.dateTime(received);
+ this.recentPath = Paths.get(
+ RELAY_DESCRIPTORS, MICRODESCS, "micro",
+ file + "-micro-" + year + "-" + month).toString();
+ String digest = desc.getDigestSha256Hex();
+ this.storagePath = Paths.get(
+ RELAY_DESCRIPTORS,
+ MICRODESC, year, month, "micro",
+ digest.substring(0,1),
+ digest.substring(1,2),
+ digest).toString();
+ }
+}
+
diff --git a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
index 28472f8..5c58f23 100644
--- a/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
+++ b/src/main/java/org/torproject/metrics/collector/relaydescs/ArchiveWriter.java
@@ -7,6 +7,7 @@ import org.torproject.descriptor.BandwidthFile;
import org.torproject.descriptor.Descriptor;
import org.torproject.descriptor.DescriptorParser;
import org.torproject.descriptor.DescriptorSourceFactory;
+import org.torproject.descriptor.Microdescriptor;
import org.torproject.descriptor.RelayExtraInfoDescriptor;
import org.torproject.descriptor.RelayNetworkStatusConsensus;
import org.torproject.descriptor.RelayNetworkStatusVote;
@@ -109,6 +110,8 @@ public class ArchiveWriter extends CollecTorMain {
this.mapPathDescriptors.put(
"recent/relay-descriptors/microdescs/consensus-microdesc",
RelayNetworkStatusConsensus.class);
+ this.mapPathDescriptors.put("recent/relay-descriptors/microdescs/micro/",
+ Microdescriptor.class);
this.mapPathDescriptors.put("recent/relay-descriptors/server-descriptors",
RelayServerDescriptor.class);
this.mapPathDescriptors.put("recent/relay-descriptors/extra-infos",
@@ -801,15 +804,17 @@ public class ArchiveWriter extends CollecTorMain {
* file written in the first call. However, this method must be
* called twice to store the same microdescriptor in two different
* valid-after months. */
- SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy/MM/");
+ SimpleDateFormat descriptorFormat = new SimpleDateFormat("yyyy-MM");
+ String[] yearMonth = descriptorFormat.format(validAfter).split("-");
File tarballFile = Paths.get(this.outputDirectory, MICRODESC,
- descriptorFormat.format(validAfter), MICRO,
+ yearMonth[0], yearMonth[1], MICRO,
microdescriptorDigest.substring(0, 1),
microdescriptorDigest.substring(1, 2),
microdescriptorDigest).toFile();
boolean tarballFileExistedBefore = tarballFile.exists();
File rsyncCatFile = Paths.get(recentPathName, RELAY_DESCRIPTORS,
- MICRODESCS, MICRO, this.rsyncCatString + "-micro.tmp").toFile();
+ MICRODESCS, MICRO, this.rsyncCatString + "-micro-" + yearMonth[0] + "-"
+ + yearMonth[1] + ".tmp").toFile();
File[] outputFiles = new File[] { tarballFile, rsyncCatFile };
boolean[] append = new boolean[] { false, true };
if (this.store(Annotation.Microdescriptor.bytes(), data, outputFiles,
diff --git a/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java b/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java
index bbb61b7..e8f780e 100644
--- a/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java
+++ b/src/main/java/org/torproject/metrics/collector/sync/SyncPersistence.java
@@ -11,6 +11,7 @@ import org.torproject.descriptor.BridgeServerDescriptor;
import org.torproject.descriptor.BridgedbMetrics;
import org.torproject.descriptor.Descriptor;
import org.torproject.descriptor.ExitList;
+import org.torproject.descriptor.Microdescriptor;
import org.torproject.descriptor.RelayExtraInfoDescriptor;
import org.torproject.descriptor.RelayNetworkStatusConsensus;
import org.torproject.descriptor.RelayNetworkStatusVote;
@@ -32,6 +33,7 @@ import org.torproject.metrics.collector.persist.DescriptorPersistence;
import org.torproject.metrics.collector.persist.ExitlistPersistence;
import org.torproject.metrics.collector.persist.ExtraInfoPersistence;
import org.torproject.metrics.collector.persist.MicroConsensusPersistence;
+import org.torproject.metrics.collector.persist.MicrodescriptorPersistence;
import org.torproject.metrics.collector.persist.OnionPerfPersistence;
import org.torproject.metrics.collector.persist.PersistenceUtils;
import org.torproject.metrics.collector.persist.ServerDescriptorPersistence;
@@ -118,6 +120,14 @@ public class SyncPersistence {
descPersist = new ExtraInfoPersistence(
(RelayExtraInfoDescriptor) desc, received);
break;
+ case "Microdescriptor":
+ if (filename.contains("-micro-")) {
+ String[] yearMonth = filename.substring(
+ filename.indexOf("-micro-") + 7).split("-");
+ descPersist = new MicrodescriptorPersistence((Microdescriptor) desc,
+ received, yearMonth[0], yearMonth[1]);
+ }
+ break;
case "BridgeNetworkStatus": // need to infer authId from filename
String[] filenameParts = filename.split(DASH);
if (filenameParts.length < 3) {
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits