[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [metrics-db/master] Concatenate descriptors in rsync recent directory.
commit 73bb381575c5f63f25c546100ed412b5b3796d48
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Sat May 24 22:12:55 2014 +0200
Concatenate descriptors in rsync recent directory.
We're currently providing server descriptors, extra-info descriptors, and
microdescriptors in one file per descriptor. The result is an rsync
recent directory containing 300,000 files, which takes forever to sync.
The new approach is to concatenate descriptors of the same type into one
file per execution. We're still deleting files after 72 hours, so that
there will always be three days of data available. But the same data only
occupies about 900 files, which is much more efficient to sync.
The downside is that rsync clients will have to learn how to split up
files into single descriptors. Let's give them some time to adapt.
---
.../db/bridgedescs/SanitizedBridgesWriter.java | 40 +++++++++++---
.../ernie/db/relaydescs/ArchiveWriter.java | 58 +++++++++++++++-----
2 files changed, 77 insertions(+), 21 deletions(-)
diff --git a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
index 275e155..72cb65e 100644
--- a/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
+++ b/src/org/torproject/ernie/db/bridgedescs/SanitizedBridgesWriter.java
@@ -90,6 +90,8 @@ public class SanitizedBridgesWriter extends Thread {
*/
private Logger logger;
+ private String rsyncCatString;
+
private File bridgeDirectoriesDirectory;
/**
@@ -137,6 +139,12 @@ public class SanitizedBridgesWriter extends Thread {
this.logger = Logger.getLogger(
SanitizedBridgesWriter.class.getName());
+ SimpleDateFormat rsyncCatFormat = new SimpleDateFormat(
+ "yyyy-MM-dd-HH-mm-ss");
+ rsyncCatFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ this.rsyncCatString = rsyncCatFormat.format(
+ System.currentTimeMillis());
+
/* Initialize secure random number generator if we need it. */
if (this.replaceIPAddressesWithHashes) {
try {
@@ -827,12 +835,19 @@ public class SanitizedBridgesWriter extends Thread {
File rsyncFile = new File(
"rsync/bridge-descriptors/server-descriptors/"
+ tarballFile.getName());
- File[] outputFiles = new File[] { tarballFile, rsyncFile };
+ File rsyncCatFile = new File("rsync/bridge-descriptors/"
+ + "server-descriptors-cat/" + this.rsyncCatString
+ + "-server-descriptors.tmp");
+ File[] outputFiles = new File[] { tarballFile, rsyncFile,
+ rsyncCatFile };
+ boolean[] append = new boolean[] { false, false, true };
try {
- for (File outputFile : outputFiles) {
+ for (int i = 0; i < outputFiles.length; i++) {
+ File outputFile = outputFiles[i];
+ boolean appendToFile = append[i];
outputFile.getParentFile().mkdirs();
BufferedWriter bw = new BufferedWriter(new FileWriter(
- outputFile));
+ outputFile, appendToFile));
bw.write("@type bridge-server-descriptor 1.0\n");
bw.write(scrubbedDesc);
bw.write("router-digest " + descriptorDigest.toUpperCase() + "\n");
@@ -968,12 +983,18 @@ public class SanitizedBridgesWriter extends Thread {
+ descriptorDigest);
File rsyncFile = new File("rsync/bridge-descriptors/extra-infos/"
+ tarballFile.getName());
- File[] outputFiles = new File[] { tarballFile, rsyncFile };
+ File rsyncCatFile = new File("rsync/bridge-descriptors/"
+ + "extra-infos-cat/" + this.rsyncCatString + "-extra-infos.tmp");
+ File[] outputFiles = new File[] { tarballFile, rsyncFile,
+ rsyncCatFile };
+ boolean[] append = new boolean[] { false, false, true };
try {
- for (File outputFile : outputFiles) {
+ for (int i = 0; i < outputFiles.length; i++) {
+ File outputFile = outputFiles[i];
+ boolean appendToFile = append[i];
outputFile.getParentFile().mkdirs();
BufferedWriter bw = new BufferedWriter(new FileWriter(
- outputFile));
+ outputFile, appendToFile));
bw.write("@type bridge-extra-info 1.2\n");
bw.write(scrubbedDesc);
bw.write("router-digest " + descriptorDigest.toUpperCase() + "\n");
@@ -1062,7 +1083,8 @@ public class SanitizedBridgesWriter extends Thread {
}
/* Delete all files from the rsync directory that have not been modified
- * in the last three days. */
+ * in the last three days, and remove the .tmp extension from newly
+ * written files. */
public void cleanUpRsyncDirectory() {
long cutOffMillis = System.currentTimeMillis()
- 3L * 24L * 60L * 60L * 1000L;
@@ -1074,6 +1096,10 @@ public class SanitizedBridgesWriter extends Thread {
allFiles.addAll(Arrays.asList(file.listFiles()));
} else if (file.lastModified() < cutOffMillis) {
file.delete();
+ } else if (file.getName().endsWith(".tmp")) {
+ file.renameTo(new File(file.getParentFile(),
+ file.getName().substring(0,
+ file.getName().lastIndexOf(".tmp"))));
}
}
}
diff --git a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
index a9cb5a7..e0e52f2 100644
--- a/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
+++ b/src/org/torproject/ernie/db/relaydescs/ArchiveWriter.java
@@ -72,6 +72,7 @@ public class ArchiveWriter extends Thread {
private long now = System.currentTimeMillis();
private Logger logger;
private File outputDirectory;
+ private String rsyncCatString;
private DescriptorParser descriptorParser;
private int storedConsensusesCounter = 0,
storedMicrodescConsensusesCounter = 0, storedVotesCounter = 0,
@@ -260,6 +261,11 @@ public class ArchiveWriter extends Thread {
this.logger = Logger.getLogger(ArchiveWriter.class.getName());
this.outputDirectory = outputDirectory;
+ SimpleDateFormat rsyncCatFormat = new SimpleDateFormat(
+ "yyyy-MM-dd-HH-mm-ss");
+ rsyncCatFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ this.rsyncCatString = rsyncCatFormat.format(
+ System.currentTimeMillis());
this.descriptorParser =
DescriptorSourceFactory.createDescriptorParser();
@@ -317,7 +323,7 @@ public class ArchiveWriter extends Thread {
}
private boolean store(byte[] typeAnnotation, byte[] data,
- File[] outputFiles) {
+ File[] outputFiles, boolean[] append) {
try {
this.logger.finer("Storing " + outputFiles[0]);
if (this.descriptorParser.parseDescriptors(data,
@@ -326,10 +332,12 @@ public class ArchiveWriter extends Thread {
+ " doesn't contain exactly one descriptor. Not storing.");
return false;
}
- for (File outputFile : outputFiles) {
+ for (int i = 0; i < outputFiles.length; i++) {
+ File outputFile = outputFiles[i];
+ boolean appendToFile = append == null ? false : append[i];
outputFile.getParentFile().mkdirs();
BufferedOutputStream bos = new BufferedOutputStream(
- new FileOutputStream(outputFile));
+ new FileOutputStream(outputFile, appendToFile));
if (data.length > 0 && data[0] != '@') {
bos.write(typeAnnotation, 0, typeAnnotation.length);
}
@@ -360,7 +368,7 @@ public class ArchiveWriter extends Thread {
File rsyncFile = new File("rsync/relay-descriptors/consensuses/"
+ tarballFile.getName());
File[] outputFiles = new File[] { tarballFile, rsyncFile };
- if (this.store(CONSENSUS_ANNOTATION, data, outputFiles)) {
+ if (this.store(CONSENSUS_ANNOTATION, data, outputFiles, null)) {
this.storedConsensusesCounter++;
}
if (this.now - validAfter < 3L * 60L * 60L * 1000L) {
@@ -387,7 +395,8 @@ public class ArchiveWriter extends Thread {
File rsyncFile = new File("rsync/relay-descriptors/microdescs/"
+ "consensus-microdesc/" + tarballFile.getName());
File[] outputFiles = new File[] { tarballFile, rsyncFile };
- if (this.store(MICRODESCCONSENSUS_ANNOTATION, data, outputFiles)) {
+ if (this.store(MICRODESCCONSENSUS_ANNOTATION, data, outputFiles,
+ null)) {
this.storedMicrodescConsensusesCounter++;
}
if (this.now - validAfter < 3L * 60L * 60L * 1000L) {
@@ -410,7 +419,7 @@ public class ArchiveWriter extends Thread {
File rsyncFile = new File("rsync/relay-descriptors/votes/"
+ tarballFile.getName());
File[] outputFiles = new File[] { tarballFile, rsyncFile };
- if (this.store(VOTE_ANNOTATION, data, outputFiles)) {
+ if (this.store(VOTE_ANNOTATION, data, outputFiles, null)) {
this.storedVotesCounter++;
}
if (this.now - validAfter < 3L * 60L * 60L * 1000L) {
@@ -433,7 +442,7 @@ public class ArchiveWriter extends Thread {
File tarballFile = new File(this.outputDirectory + "/certs/"
+ fingerprint + "-" + printFormat.format(new Date(published)));
File[] outputFiles = new File[] { tarballFile };
- if (this.store(CERTIFICATE_ANNOTATION, data, outputFiles)) {
+ if (this.store(CERTIFICATE_ANNOTATION, data, outputFiles, null)) {
this.storedCertsCounter++;
}
}
@@ -450,8 +459,14 @@ public class ArchiveWriter extends Thread {
+ digest);
File rsyncFile = new File(
"rsync/relay-descriptors/server-descriptors/" + digest);
- File[] outputFiles = new File[] { tarballFile, rsyncFile };
- if (this.store(SERVER_DESCRIPTOR_ANNOTATION, data, outputFiles)) {
+ File rsyncCatFile = new File("rsync/relay-descriptors/"
+ + "server-descriptors-cat/" + this.rsyncCatString
+ + "-server-descriptors.tmp");
+ File[] outputFiles = new File[] { tarballFile, rsyncFile,
+ rsyncCatFile };
+ boolean[] append = new boolean[] { false, false, true };
+ if (this.store(SERVER_DESCRIPTOR_ANNOTATION, data, outputFiles,
+ append)) {
this.storedServerDescriptorsCounter++;
}
if (this.now - published < 48L * 60L * 60L * 1000L) {
@@ -477,8 +492,12 @@ public class ArchiveWriter extends Thread {
+ extraInfoDigest);
File rsyncFile = new File("rsync/relay-descriptors/extra-infos/"
+ extraInfoDigest);
- File[] outputFiles = new File[] { tarballFile, rsyncFile };
- if (this.store(EXTRA_INFO_ANNOTATION, data, outputFiles)) {
+ File rsyncCatFile = new File("rsync/relay-descriptors/"
+ + "extra-infos-cat/" + this.rsyncCatString + "-extra-infos.tmp");
+ File[] outputFiles = new File[] { tarballFile, rsyncFile,
+ rsyncCatFile };
+ boolean[] append = new boolean[] { false, false, true };
+ if (this.store(EXTRA_INFO_ANNOTATION, data, outputFiles, append)) {
this.storedExtraInfoDescriptorsCounter++;
}
if (this.now - published < 48L * 60L * 60L * 1000L) {
@@ -510,8 +529,14 @@ public class ArchiveWriter extends Thread {
+ microdescriptorDigest);
File rsyncFile = new File("rsync/relay-descriptors/microdescs/micro/"
+ microdescriptorDigest);
- File[] outputFiles = new File[] { tarballFile, rsyncFile };
- if (this.store(MICRODESCRIPTOR_ANNOTATION, data, outputFiles)) {
+ File rsyncCatFile = new File("rsync/relay-descriptors/"
+ + "microdescs/micro-cat/" + this.rsyncCatString
+ + "-micro.tmp");
+ File[] outputFiles = new File[] { tarballFile, rsyncFile,
+ rsyncCatFile };
+ boolean[] append = new boolean[] { false, false, true };
+ if (this.store(MICRODESCRIPTOR_ANNOTATION, data, outputFiles,
+ append)) {
this.storedMicrodescriptorsCounter++;
}
if (this.now - validAfter < 40L * 24L * 60L * 60L * 1000L) {
@@ -766,7 +791,8 @@ public class ArchiveWriter extends Thread {
}
/* Delete all files from the rsync directory that have not been modified
- * in the last three days. */
+ * in the last three days, and remove the .tmp extension from newly
+ * written files. */
public void cleanUpRsyncDirectory() {
long cutOffMillis = System.currentTimeMillis()
- 3L * 24L * 60L * 60L * 1000L;
@@ -778,6 +804,10 @@ public class ArchiveWriter extends Thread {
allFiles.addAll(Arrays.asList(file.listFiles()));
} else if (file.lastModified() < cutOffMillis) {
file.delete();
+ } else if (file.getName().endsWith(".tmp")) {
+ file.renameTo(new File(file.getParentFile(),
+ file.getName().substring(0,
+ file.getName().lastIndexOf(".tmp"))));
}
}
}
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits