[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [metrics-db/master] Copy recently processed files to an rsync/ directory.
commit e6a0e94535014926c6df53ed728e108a30d6386b
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Mon Jan 9 11:57:11 2012 +0100
Copy recently processed files to an rsync/ directory.
---
config.template | 10 ++
src/org/torproject/ernie/db/Configuration.java | 13 ++
src/org/torproject/ernie/db/Main.java | 13 ++
src/org/torproject/ernie/db/RsyncDataProvider.java | 146 ++++++++++++++++++++
4 files changed, 182 insertions(+), 0 deletions(-)
diff --git a/config.template b/config.template
index dc5d30e..6cfdce5 100644
--- a/config.template
+++ b/config.template
@@ -112,4 +112,14 @@
#
## Download exit list and store it to disk
#DownloadExitList 0
+#
+#
+######## Rsync directory ########
+#
+## Copy recently published decriptors to another directory to provide them
+## via rsync
+#ProvideFilesViaRsync 0
+#
+## Relative path to the directory that shall be provided via rsync
+#RsyncDirectory rsync/
diff --git a/src/org/torproject/ernie/db/Configuration.java b/src/org/torproject/ernie/db/Configuration.java
index b4e6f8b..7197c7d 100644
--- a/src/org/torproject/ernie/db/Configuration.java
+++ b/src/org/torproject/ernie/db/Configuration.java
@@ -49,6 +49,8 @@ public class Configuration {
private boolean processBridgePoolAssignments = false;
private String assignmentsDirectory = "assignments/";
private String sanitizedAssignmentsDirectory = "sanitized-assignments/";
+ private boolean provideFilesViaRsync = false;
+ private String rsyncDirectory = "rsync";
public Configuration() {
/* Initialize logger. */
@@ -168,6 +170,11 @@ public class Configuration {
this.assignmentsDirectory = line.split(" ")[1];
} else if (line.startsWith("SanitizedAssignmentsDirectory")) {
this.sanitizedAssignmentsDirectory = line.split(" ")[1];
+ } else if (line.startsWith("ProvideFilesViaRsync")) {
+ this.provideFilesViaRsync = Integer.parseInt(
+ line.split(" ")[1]) != 0;
+ } else if (line.startsWith("RsyncDirectory")) {
+ this.rsyncDirectory = line.split(" ")[1];
} else {
logger.severe("Configuration file contains unrecognized "
+ "configuration key in line '" + line + "'! Exiting!");
@@ -317,5 +324,11 @@ public class Configuration {
public String getSanitizedAssignmentsDirectory() {
return sanitizedAssignmentsDirectory;
}
+ public boolean getProvideFilesViaRsync() {
+ return this.provideFilesViaRsync;
+ }
+ public String getRsyncDirectory() {
+ return this.rsyncDirectory;
+ }
}
diff --git a/src/org/torproject/ernie/db/Main.java b/src/org/torproject/ernie/db/Main.java
index 5a06900..a20748a 100644
--- a/src/org/torproject/ernie/db/Main.java
+++ b/src/org/torproject/ernie/db/Main.java
@@ -136,6 +136,19 @@ public class Main {
new File(config.getSanitizedAssignmentsDirectory()));
}
+ // Copy recently published files to a local directory that can then
+ // be served via rsync.
+ if (config.getProvideFilesViaRsync()) {
+ new RsyncDataProvider(
+ config.getDirectoryArchivesOutputDirectory() == null ? null :
+ new File(config.getDirectoryArchivesOutputDirectory()),
+ config.getSanitizedBridgesWriteDirectory() == null ? null :
+ new File(config.getSanitizedBridgesWriteDirectory()),
+ config.getSanitizedAssignmentsDirectory() == null ? null :
+ new File(config.getSanitizedAssignmentsDirectory()),
+ new File(config.getRsyncDirectory()));
+ }
+
// Remove lock file
lf.releaseLock();
diff --git a/src/org/torproject/ernie/db/RsyncDataProvider.java b/src/org/torproject/ernie/db/RsyncDataProvider.java
new file mode 100644
index 0000000..1f0d68f
--- /dev/null
+++ b/src/org/torproject/ernie/db/RsyncDataProvider.java
@@ -0,0 +1,146 @@
+/* Copyright 2012 The Tor Project
+ * See LICENSE for licensing information */
+package org.torproject.ernie.db;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Copy files published in the last 3 days to a local directory that can
+ * then be served via rsync.
+ */
+public class RsyncDataProvider {
+ public RsyncDataProvider(File directoryArchivesOutputDirectory,
+ File sanitizedBridgesWriteDirectory,
+ File sanitizedAssignmentsDirectory, File rsyncDirectory) {
+
+ /* Determine the cut-off time for files in rsync/. */
+ long cutOffMillis = System.currentTimeMillis()
+ - 3L * 24L * 60L * 60L * 1000L;
+
+ /* Create rsync/ directory if it doesn't exist. */
+ if (!rsyncDirectory.exists()) {
+ rsyncDirectory.mkdirs();
+ }
+
+ /* Make a list of all files in the rsync/ directory to delete those
+ * that we didn't copy in this run. */
+ Set<String> fileNamesInRsync = new HashSet<String>();
+ Stack<File> files = new Stack<File>();
+ files.add(rsyncDirectory);
+ while (!files.isEmpty()) {
+ File pop = files.pop();
+ if (pop.isDirectory()) {
+ files.addAll(Arrays.asList(pop.listFiles()));
+ } else {
+ fileNamesInRsync.add(pop.getName());
+ }
+ }
+
+ /* Copy relay descriptors from the last 3 days. */
+ if (directoryArchivesOutputDirectory != null) {
+ files.add(directoryArchivesOutputDirectory);
+ while (!files.isEmpty()) {
+ File pop = files.pop();
+ if (pop.isDirectory()) {
+ files.addAll(Arrays.asList(pop.listFiles()));
+ } else if (pop.lastModified() >= cutOffMillis) {
+ String fileName = pop.getName();
+ if (pop.getAbsolutePath().contains("/consensus/")) {
+ this.copyFile(pop, new File(rsyncDirectory,
+ "relay-descriptors/consensuses/" + fileName));
+ } else if (pop.getAbsolutePath().contains("/vote/")) {
+ this.copyFile(pop, new File(rsyncDirectory,
+ "relay-descriptors/votes/" + fileName));
+ } else if (pop.getAbsolutePath().contains(
+ "/server-descriptor/")) {
+ this.copyFile(pop, new File(rsyncDirectory,
+ "relay-descriptors/server-descriptors/" + fileName));
+ } else if (pop.getAbsolutePath().contains("/extra-info/")) {
+ this.copyFile(pop, new File(rsyncDirectory,
+ "relay-descriptors/extra-infos/" + fileName));
+ } else {
+ continue;
+ }
+ fileNamesInRsync.remove(pop.getName());
+ }
+ }
+ }
+
+ /* Copy sanitized bridge descriptors from the last 3 days. */
+ if (sanitizedBridgesWriteDirectory != null) {
+ files.add(sanitizedBridgesWriteDirectory);
+ while (!files.isEmpty()) {
+ File pop = files.pop();
+ if (pop.isDirectory()) {
+ files.addAll(Arrays.asList(pop.listFiles()));
+ } else if (pop.lastModified() >= cutOffMillis) {
+ String fileName = pop.getName();
+ if (pop.getAbsolutePath().contains("/statuses/")) {
+ this.copyFile(pop, new File(rsyncDirectory,
+ "bridge-descriptors/statuses/" + fileName));
+ } else if (pop.getAbsolutePath().contains(
+ "/server-descriptors/")) {
+ this.copyFile(pop, new File(rsyncDirectory,
+ "bridge-descriptors/server-descriptors/" + fileName));
+ } else if (pop.getAbsolutePath().contains("/extra-infos/")) {
+ this.copyFile(pop, new File(rsyncDirectory,
+ "bridge-descriptors/extra-infos/" + fileName));
+ } else {
+ continue;
+ }
+ fileNamesInRsync.remove(pop.getName());
+ }
+ }
+ }
+
+ /* Copy sanitized bridge pool assignments from the last 3 days. */
+ if (sanitizedAssignmentsDirectory != null) {
+ files.add(sanitizedAssignmentsDirectory);
+ while (!files.isEmpty()) {
+ File pop = files.pop();
+ if (pop.isDirectory()) {
+ files.addAll(Arrays.asList(pop.listFiles()));
+ } else if (pop.lastModified() >= cutOffMillis) {
+ String fileName = pop.getName();
+ this.copyFile(pop, new File(rsyncDirectory,
+ "bridge-pool-assignments/" + fileName));
+ fileNamesInRsync.remove(pop.getName());
+ }
+ }
+ }
+
+ /* Delete all files that we didn't (over-)write in this run. */
+ files.add(rsyncDirectory);
+ while (!files.isEmpty()) {
+ File pop = files.pop();
+ if (pop.isDirectory()) {
+ files.addAll(Arrays.asList(pop.listFiles()));
+ } else if (fileNamesInRsync.contains(pop.getName())) {
+ pop.delete();
+ }
+ }
+ }
+
+ private void copyFile(File from, File to) {
+ if (to.exists()) {
+ return;
+ }
+ try {
+ to.getParentFile().mkdirs();
+ FileInputStream fis = new FileInputStream(from);
+ BufferedInputStream bis = new BufferedInputStream(fis);
+ FileOutputStream fos = new FileOutputStream(to);
+ int len;
+ byte[] data = new byte[1024];
+ while ((len = bis.read(data, 0, 1024)) >= 0) {
+ fos.write(data, 0, len);
+ }
+ bis.close();
+ fos.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+}
+
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits