[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [onionoo/master] Add four new execution modes.
commit 5cedec578e04116aa588eb791faf08969a69f712
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Sun Apr 26 17:06:39 2015 +0200
Add four new execution modes.
New modes are:
--single-run Run steps 1--3 only for a single time, then exit.
--download-only Only run step 1: download recent descriptors, then
exit.
--update-only Only run step 2: update internal status files, then
exit.
--write-only Only run step 3: write output document files, then
exit.
Default mode is:
[no argument] Run steps 1--3 repeatedly once per hour.
Use the lock file in default periodic-update mode, too, because we
don't want other execution modes to interfere with an ongoing periodic
update either.
Implements part of #13600.
---
.../java/org/torproject/onionoo/cron/Main.java | 276 ++++++++++++++------
.../onionoo/updater/DescriptorSource.java | 3 -
2 files changed, 200 insertions(+), 79 deletions(-)
diff --git a/src/main/java/org/torproject/onionoo/cron/Main.java b/src/main/java/org/torproject/onionoo/cron/Main.java
index c098dfe..2f45cea 100644
--- a/src/main/java/org/torproject/onionoo/cron/Main.java
+++ b/src/main/java/org/torproject/onionoo/cron/Main.java
@@ -1,4 +1,4 @@
-/* Copyright 2011, 2012 The Tor Project
+/* Copyright 2011--2015 The Tor Project
* See LICENSE for licensing information */
package org.torproject.onionoo.cron;
@@ -18,102 +18,226 @@ import org.torproject.onionoo.util.LockFile;
import org.torproject.onionoo.writer.DocumentWriterRunner;
/* Update search data and status data files. */
-public class Main {
-
- private static Logger log = LoggerFactory.getLogger(Main.class);
+public class Main implements Runnable {
private Main() {
}
- private static final ScheduledExecutorService scheduler =
- Executors.newScheduledThreadPool(1);
+ private Logger log = LoggerFactory.getLogger(Main.class);
public static void main(String[] args) {
- boolean runOnce = "true".equals(System.getProperty(
- "onionoo.cron.runonce", "true"));
- if (runOnce){
- log.info("Going to run one-time updater ... ");
- LockFile lf = new LockFile();
- log.info("Initializing.");
- if (lf.acquireLock()) {
- log.info("Acquired lock");
- } else {
- log.error("Could not acquire lock. Is Onionoo already running? "
- + "Terminating");
- return;
- }
- new Updater().run();
- log.info("Releasing lock.");
- if (lf.releaseLock()) {
- log.info("Released lock");
- } else {
- log.error("Could not release lock. The next execution may not "
- + "start as expected");
+ Main main = new Main();
+ main.parseArgsOrExit(args);
+ main.runOrScheduleExecutions();
+ }
+
+ boolean defaultMode = false, singleRun = false, downloadOnly = false,
+ updateOnly = false, writeOnly = false;
+
+ /* TODO Parsing command-line arguments is only a workaround until we're
+ * more certain what kind of options we want to support. We should then
+ * switch to some library that parses options for us. */
+ private void parseArgsOrExit(String[] args) {
+ boolean validArgs = true;
+ if (args.length == 0) {
+ this.defaultMode = true;
+ } else if (args.length == 1) {
+ switch (args[0]) {
+ case "--help":
+ this.printUsageAndExit(0);
+ break;
+ case "--single-run":
+ this.singleRun = true;
+ break;
+ case "--download-only":
+ this.downloadOnly = true;
+ break;
+ case "--update-only":
+ this.updateOnly = true;
+ break;
+ case "--write-only":
+ this.writeOnly = true;
+ break;
+ default:
+ validArgs = false;
}
- return;
+ } else if (args.length > 1) {
+ validArgs = false;
+ }
+ if (!validArgs) {
+ this.printUsageAndExit(1);
+ }
+ }
+
+ private void printUsageAndExit(int status) {
+ System.err.println("Please provide only a single execution:");
+ System.err.println(" [no argument] Run steps 1--3 repeatedly "
+ + "once per hour.");
+ System.err.println(" --single-run Run steps 1--3 only for a "
+ + "single time, then exit.");
+ System.err.println(" --download-only Only run step 1: download "
+ + "recent descriptors, then exit.");
+ System.err.println(" --update-only Only run step 2: update "
+ + "internal status files, then exit.");
+ System.err.println(" --write-only Only run step 3: write "
+ + "output document files, then exit.");
+ System.err.println(" --help Print out this help message "
+ + "and exit.");
+ System.exit(status);
+ }
+
+ private void runOrScheduleExecutions() {
+ if (!this.defaultMode) {
+ this.log.info("Going to run one-time updater ... ");
+ this.run();
} else {
- log.info("Periodic updater started.");
- final Runnable updater = new Updater();
- int currentMinute = Calendar.getInstance().get(Calendar.MINUTE);
- int initialDelay = (75 - currentMinute + currentMinute % 5) % 60;
+ this.scheduleExecutions();
+ }
+ }
+
+ private final ScheduledExecutorService scheduler =
+ Executors.newScheduledThreadPool(1);
+
+ private void scheduleExecutions() {
+ this.log.info("Periodic updater started.");
+ final Runnable mainRunnable = this;
+ int currentMinute = Calendar.getInstance().get(Calendar.MINUTE);
+ int initialDelay = (75 - currentMinute + currentMinute % 5) % 60;
+
+ /* Run after initialDelay delay and then every hour. */
+ this.log.info("Periodic updater will start every hour at minute "
+ + ((currentMinute + initialDelay) % 60) + ".");
+ this.scheduler.scheduleAtFixedRate(mainRunnable, initialDelay, 60,
+ TimeUnit.MINUTES);
+ }
+
+ public void run() {
+ this.acquireLockOrExit();
+ this.initialize();
+ this.downloadDescriptors();
+ this.updateStatuses();
+ this.writeDocuments();
+ this.shutDown();
+ this.gatherStatistics();
+ this.cleanUp();
+ this.releaseLock();
+ }
+
+ private LockFile lf;
- /* Run after initialDelay delay and then every hour. */
- log.info("Periodic updater will start every hour at minute "
- + ((currentMinute + initialDelay) % 60) + ".");
- scheduler.scheduleAtFixedRate(updater, initialDelay, 60,
- TimeUnit.MINUTES);
+ private void acquireLockOrExit() {
+ this.log.info("Initializing.");
+ this.lf = new LockFile();
+ if (this.lf.acquireLock()) {
+ this.log.info("Acquired lock");
+ } else {
+ this.log.error("Could not acquire lock. Is Onionoo already "
+ + "running? Terminating");
+ System.exit(1);
}
}
- private static class Updater implements Runnable{
+ private DescriptorSource dso;
- private Logger log = LoggerFactory.getLogger(Main.class);
+ private DocumentStore ds;
- public void run() {
- log.debug("Started update ...");
+ private StatusUpdateRunner sur;
- DescriptorSource dso =
- DescriptorSourceFactory.getDescriptorSource();
- log.info("Initialized descriptor source");
- DocumentStore ds = DocumentStoreFactory.getDocumentStore();
- log.info("Initialized document store");
- StatusUpdateRunner sur = new StatusUpdateRunner();
- log.info("Initialized status update runner");
- DocumentWriterRunner dwr = new DocumentWriterRunner();
- log.info("Initialized document writer runner");
+ private DocumentWriterRunner dwr;
- log.info("Downloading descriptors.");
- dso.downloadDescriptors();
+ private void initialize() {
+ this.log.debug("Started update ...");
+ if (!this.writeOnly) {
+ this.dso = DescriptorSourceFactory.getDescriptorSource();
+ this.log.info("Initialized descriptor source");
+ }
+ if (!this.downloadOnly) {
+ this.ds = DocumentStoreFactory.getDocumentStore();
+ this.log.info("Initialized document store");
+ }
+ if (!this.downloadOnly && !this.writeOnly) {
+ this.sur = new StatusUpdateRunner();
+ this.log.info("Initialized status update runner");
+ }
+ if (!this.downloadOnly && !this.updateOnly) {
+ this.dwr = new DocumentWriterRunner();
+ this.log.info("Initialized document writer runner");
+ }
+ }
- log.info("Reading descriptors.");
- dso.readDescriptors();
+ private void downloadDescriptors() {
+ if (this.updateOnly || this.writeOnly) {
+ return;
+ }
+ this.log.info("Downloading descriptors.");
+ this.dso.downloadDescriptors();
+ }
- log.info("Updating internal status files.");
- sur.updateStatuses();
+ private void updateStatuses() {
+ if (this.downloadOnly || this.writeOnly) {
+ return;
+ }
+ this.log.info("Reading descriptors.");
+ this.dso.readDescriptors();
+ this.log.info("Updating internal status files.");
+ this.sur.updateStatuses();
+ }
- log.info("Updating document files.");
- dwr.writeDocuments();
+ private void writeDocuments() {
+ if (this.downloadOnly || this.updateOnly) {
+ return;
+ }
+ log.info("Updating document files.");
+ this.dwr.writeDocuments();
+ }
- log.info("Shutting down.");
- dso.writeHistoryFiles();
+ private void shutDown() {
+ log.info("Shutting down.");
+ if (this.dso != null) {
+ this.dso.writeHistoryFiles();
log.info("Wrote parse histories");
- ds.flushDocumentCache();
- log.info("Flushed document cache");
-
- log.info("Gathering statistics.");
- sur.logStatistics();
- dwr.logStatistics();
- log.info("Descriptor source\n" + dso.getStatsString());
- log.info("Document store\n" + ds.getStatsString());
-
- /* Clean up to prevent out-of-memory exception, and to ensure that
- * the next execution starts with a fresh descriptor source. */
- log.info("Cleaning up.");
- ds.invalidateDocumentCache();
- DocumentStoreFactory.setDocumentStore(null);
- DescriptorSourceFactory.setDescriptorSource(null);
-
- log.info("Done.");
+ }
+ if (this.ds != null) {
+ this.ds.flushDocumentCache();
+ this.log.info("Flushed document cache");
+ }
+ }
+
+ private void gatherStatistics() {
+ this.log.info("Gathering statistics.");
+ if (this.sur != null) {
+ this.sur.logStatistics();
+ }
+ if (this.dwr != null) {
+ this.dwr.logStatistics();
+ }
+ if (this.dso != null) {
+ this.log.info("Descriptor source\n" + this.dso.getStatsString());
+ }
+ if (this.ds != null) {
+ this.log.info("Document store\n" + this.ds.getStatsString());
+ }
+ }
+
+ private void cleanUp() {
+ /* Clean up to prevent out-of-memory exception, and to ensure that the
+ * next execution starts with a fresh descriptor source. */
+ this.log.info("Cleaning up.");
+ if (this.ds != null) {
+ this.ds.invalidateDocumentCache();
+ }
+ DocumentStoreFactory.setDocumentStore(null);
+ DescriptorSourceFactory.setDescriptorSource(null);
+ this.log.info("Done.");
+ }
+
+ private void releaseLock() {
+ this.log.info("Releasing lock.");
+ if (this.lf.releaseLock()) {
+ this.log.info("Released lock");
+ } else {
+ this.log.error("Could not release lock. The next execution may "
+ + "not start as expected");
}
}
}
diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java
index 26b7b10..52e00c9 100644
--- a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java
+++ b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java
@@ -68,9 +68,6 @@ public class DescriptorSource {
downloadedFiles = 0, deletedLocalFiles = 0;
private void downloadDescriptors(DescriptorType descriptorType) {
- if (!this.descriptorListeners.containsKey(descriptorType)) {
- return;
- }
DescriptorDownloader descriptorDownloader =
new DescriptorDownloader(descriptorType);
this.localFilesBefore += descriptorDownloader.statLocalFiles();
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits