[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [webstats/master] Don't delete parsed files from in/.
commit b306f7fc0f0285ea1d008dbbe271495bc6d92b3e
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Wed Nov 7 17:18:43 2012 -0500
Don't delete parsed files from in/.
---
src/org/torproject/webstats/Main.java | 63 ++++++++++++--------------------
1 files changed, 24 insertions(+), 39 deletions(-)
diff --git a/src/org/torproject/webstats/Main.java b/src/org/torproject/webstats/Main.java
index 1c7780c..7241d68 100644
--- a/src/org/torproject/webstats/Main.java
+++ b/src/org/torproject/webstats/Main.java
@@ -25,7 +25,7 @@ import org.apache.commons.compress.compressors.gzip.*;
*
* The main operation is to parse Apache web log files from the in/
* directory and write sanitized web log files to the out/ directory.
- * Files in the in/ directory are assumed to never change and will be
+ * Files in the in/ directory are assumed to never change and may be
* deleted after processing by this program. Files in the out/ directory
* are guaranteed to never change and may be deleted by a subsequently
* running program.
@@ -34,8 +34,8 @@ import org.apache.commons.compress.compressors.gzip.*;
* in/ are not parsed more than once and that files in out/ do not need to
* be changed:
* - state/lock prevents concurrent executions of this program.
- * - state/in-history contains file names of previously read and deleted
- * files in the in/ directory.
+ * - state/in-history contains file names of previously read and possibly
+ * deleted files in the in/ directory.
* - state/in-history.new is the file written in the current execution
* that will replace state/in-history during the execution.
* - state/temp/ contains new or updated output files parsed in the
@@ -56,12 +56,12 @@ import org.apache.commons.compress.compressors.gzip.*;
* 2. Read the contents from state/in-history and state/out-history and
* the directory listings of in/ to memory.
* 3. For each file in in/:
- * a. Append the file name to state/in-history.new.
- * b. Check that the file name is not contained in state/in-history.
- * If it is, print out a warning and skip the file.
- * c. Parse and sanitize the file in chunks of 250,000 lines to reduce
+ * a. Append the file name to state/in-history.new if it was not
+ * contained in state/in-history. If it was contained, skip the
+ * file.
+ * b. Parse and sanitize the file in chunks of 250,000 lines to reduce
* writes.
- * d. When writing sanitized chunks to output files, for each output
+ * c. When writing sanitized chunks to output files, for each output
* file, check in the following order if there is already such a
* file in
* i. state/temp/,
@@ -73,8 +73,7 @@ import org.apache.commons.compress.compressors.gzip.*;
* 4. Rename state/in-history to state/in-history.old and rename
* state/in-history.new to state/in-history. Delete
* state/in-history.old.
- * 5. Delete files in in/ that have been parsed in this execution.
- * 6. For each file in state/temp/:
+ * 5. For each file in state/temp/:
* a. Check if there's a corresponding line in state/out-history. If
* so, check whether there is a file in state/full/ or out/. If
* so, move the file to state/full/. Otherwise move the file to
@@ -84,13 +83,13 @@ import org.apache.commons.compress.compressors.gzip.*;
* days old, move the file to state/full/.
* c. If b. does not apply, append a line to out-history.new and move
* the file to out/.
- * 7. For each file in state/full/, check whether the sanitized log is at
+ * 6. For each file in state/full/, check whether the sanitized log is at
* least four (4) days old and not contained in state/out-history. If
* so, append a line to out-history.new and move the file to out/.
- * 8. Rename state/out-history to state/out-history.old and rename
+ * 7. Rename state/out-history to state/out-history.old and rename
* state/out-history.new to state/out-history. Delete
* state/out-history.old.
- * 9. Delete state/lock and exit.
+ * 8. Delete state/lock and exit.
*
* If the program is interrupted and leaves a lock file in state/lock, it
* requires an operator to fix the state/ directory and make it work
@@ -133,20 +132,19 @@ public class Main {
readHistoryFiles(); /* Step 2 */
readInDirectoryListing();
for (File inFile : inFiles) { /* Step 3 */
- appendToInHistory(inFile);
+ appendToInHistoryIfNotContained(inFile);
if (!checkFileName(inFile) || checkParsedBefore(inFile)) {
continue;
}
sanitizeInFile(inFile);
}
overwriteInHistoryFile(); /* Step 4 */
- deleteParsedInFiles(); /* Step 5 */
- for (String outputFileName : updatedOutputFiles) { /* Step 6 */
+ for (String outputFileName : updatedOutputFiles) { /* Step 5 */
moveOutputFile(outputFileName);
}
- moveFullFilesToOut(); /* Step 7 */
- overwriteOutHistoryFile(); /* Step 8 */
- deleteLockFile(); /* Step 9 */
+ moveFullFilesToOut(); /* Step 6 */
+ overwriteOutHistoryFile(); /* Step 7 */
+ deleteLockFile(); /* Step 8 */
}
/* Define file and directory names. */
@@ -262,10 +260,12 @@ public class Main {
}
return result;
}
- private static void appendToInHistory(File inFile) {
- inHistoryNewFiles.add(inFile.getAbsolutePath());
- String line = inFile.getAbsolutePath();
- appendToHistoryFile(stateInHistoryNewFile, line);
+ private static void appendToInHistoryIfNotContained(File inFile) {
+ if (!inHistoryNewFiles.contains(inFile.getAbsolutePath())) {
+ inHistoryNewFiles.add(inFile.getAbsolutePath());
+ String line = inFile.getAbsolutePath();
+ appendToHistoryFile(stateInHistoryNewFile, line);
+ }
}
private static void appendToHistoryFile(File historyFile, String line) {
try {
@@ -286,14 +286,7 @@ public class Main {
inFile.getName().endsWith(".gz");
}
private static boolean checkParsedBefore(File inFile) {
- if (inHistoryFiles.contains(inFile.getAbsolutePath())) {
- System.err.println("Parsed and subsequently deleted input file '"
- + inFile.getAbsolutePath() + "' before. It shouldn't be "
- + "there again. Skipping it now and not deleting it later.");
- return true;
- } else {
- return false;
- }
+ return inHistoryFiles.contains(inFile.getAbsolutePath());
}
private static void sanitizeInFile(File inFile) {
try {
@@ -452,14 +445,6 @@ public class Main {
stateInHistoryNewFile.renameTo(stateInHistoryFile);
stateInHistoryOldFile.delete();
}
- private static void deleteParsedInFiles() {
- Set<String> filesToDelete = new HashSet<String>();
- filesToDelete.addAll(inHistoryNewFiles);
- filesToDelete.removeAll(inHistoryFiles);
- for (String file : filesToDelete) {
- new File(file).delete();
- }
- }
private static void moveOutputFile(String outputFileName) {
File outFile = new File(outDirectory, outputFileName);
File stateTempFile = new File(stateTempDirectory, outputFileName);
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits