[tor-commits] [webstats/master] .gz-compress newly written output files to save disk space.

[runa@xxxxxxxxxxxxxx]

commit 7bb3197fca8683c0a1f2d13cfacefda7f5f17d92
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Wed Nov 7 19:02:57 2012 -0500

    .gz-compress newly written output files to save disk space.
---
 src/org/torproject/webstats/Main.java |   55 ++++++++++++++++++++++++++++-----
 1 files changed, 47 insertions(+), 8 deletions(-)

diff --git a/src/org/torproject/webstats/Main.java b/src/org/torproject/webstats/Main.java
index 7241d68..4bb035f 100644
--- a/src/org/torproject/webstats/Main.java
+++ b/src/org/torproject/webstats/Main.java
@@ -328,36 +328,74 @@ public class Main {
   }
   private static void storeOutputFile(String outputFileName,
       List<String> cachedLinesList) {
-    updatedOutputFiles.add(outputFileName);
+    String outputGzFileName = outputFileName + ".gz";
     File stateTempFile = new File(stateTempDirectory, outputFileName);
+    File stateTempGzFile = new File(stateTempDirectory, outputGzFileName);
     File stateFullFile = new File(stateFullDirectory, outputFileName);
+    File stateFullGzFile = new File(stateFullDirectory, outputGzFileName);
     File outFile = new File(outDirectory, outputFileName);
+    File outGzFile = new File(outDirectory, outputGzFileName);
     File stateDiffFile = new File(stateDiffDirectory, outputFileName);
+    File stateDiffGzFile = new File(stateDiffDirectory, outputGzFileName);
     if (stateTempFile.exists()) {
+      updatedOutputFiles.add(outputFileName);
       File stateTempOldFile = new File(stateTempDirectory,
           outputFileName + ".old");
       stateTempFile.renameTo(stateTempOldFile);
       mergeOutputFile(stateTempOldFile, cachedLinesList, stateTempFile);
       stateTempOldFile.delete();
+    } else if (stateTempGzFile.exists()) {
+      updatedOutputFiles.add(outputGzFileName);
+      File stateTempGzOldFile = new File(stateTempDirectory,
+          outputGzFileName + ".old");
+      stateTempGzFile.renameTo(stateTempGzOldFile);
+      mergeOutputFile(stateTempGzOldFile, cachedLinesList,
+          stateTempGzFile);
+      stateTempGzOldFile.delete();
     } else if (stateFullFile.exists()) {
+      updatedOutputFiles.add(outputFileName);
       mergeOutputFile(stateFullFile, cachedLinesList, stateTempFile);
+    } else if (stateFullGzFile.exists()) {
+      updatedOutputFiles.add(outputGzFileName);
+      mergeOutputFile(stateFullGzFile, cachedLinesList, stateTempGzFile);
     } else if (outFile.exists()) {
+      updatedOutputFiles.add(outputFileName);
       mergeOutputFile(outFile, cachedLinesList, stateTempFile);
+    } else if (outGzFile.exists()) {
+      updatedOutputFiles.add(outputGzFileName);
+      mergeOutputFile(outGzFile, cachedLinesList, stateTempGzFile);
     } else if (stateDiffFile.exists()) {
+      updatedOutputFiles.add(outputFileName);
       mergeOutputFile(stateDiffFile, cachedLinesList, stateTempFile);
+    } else if (stateDiffGzFile.exists()) {
+      updatedOutputFiles.add(outputGzFileName);
+      mergeOutputFile(stateDiffGzFile, cachedLinesList, stateTempGzFile);
     } else {
-      writeNewOutputFile(cachedLinesList, stateTempFile);
+      updatedOutputFiles.add(outputGzFileName);
+      writeNewOutputFile(cachedLinesList, stateTempGzFile);
     }
   }
   private static void mergeOutputFile(File oldOutputFile,
       List<String> cachedLinesList, File newOutputFile) {
     try {
-      BufferedReader br = new BufferedReader(
-          new FileReader(oldOutputFile));
+      BufferedReader br;
+      if (oldOutputFile.getName().endsWith(".gz")) {
+        br = new BufferedReader(new InputStreamReader(
+            new GzipCompressorInputStream(new FileInputStream(
+            oldOutputFile))));
+      } else {
+        br = new BufferedReader(new FileReader(oldOutputFile));
+      }
       String line;
       newOutputFile.getParentFile().mkdirs();
-      BufferedWriter bw = new BufferedWriter(
-          new FileWriter(newOutputFile));
+      BufferedWriter bw;
+      if (newOutputFile.getName().endsWith(".gz")) {
+        bw = new BufferedWriter(new OutputStreamWriter(
+            new GzipCompressorOutputStream(new FileOutputStream(
+            newOutputFile))));
+      } else {
+        bw = new BufferedWriter(new FileWriter(newOutputFile));
+      }
       int cachedLinesListPosition = 0,
           totalCachedLines = cachedLinesList.size();
       while ((line = br.readLine()) != null) {
@@ -388,8 +426,9 @@ public class Main {
       File outputFile) {
     try {
       outputFile.getParentFile().mkdirs();
-      BufferedWriter bw = new BufferedWriter(
-          new FileWriter(outputFile));
+      BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(
+          new GzipCompressorOutputStream(new FileOutputStream(
+          outputFile))));
       for (String cachedLine : cachedLinesList) {
         bw.write(cachedLine + "\n");
       }



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits