[tor-commits] [onionoo/master] Only write changed status and document files.

[karsten@xxxxxxxxxxxxxx]

commit 4b7b13d37d2423f9b1dee865e203a524b8df488b
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Thu Dec 5 17:03:02 2019 +0100

    Only write changed status and document files.
    
    Fixes #32660.
---
 CHANGELOG.md                                       |  4 ++++
 .../metrics/onionoo/docs/DocumentStore.java        | 22 ++++++++++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6060551..cd27f30 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Changes in version 7.0-1.2?.? - 2019-1?-??
 
+ * Medium changes
+   - Only write status files and document files if their content has
+     changed.
+
 
 # Changes in version 7.0-1.22.0 - 2019-11-28
 
diff --git a/src/main/java/org/torproject/metrics/onionoo/docs/DocumentStore.java b/src/main/java/org/torproject/metrics/onionoo/docs/DocumentStore.java
index 81b428a..ea7786e 100644
--- a/src/main/java/org/torproject/metrics/onionoo/docs/DocumentStore.java
+++ b/src/main/java/org/torproject/metrics/onionoo/docs/DocumentStore.java
@@ -8,6 +8,7 @@ import org.torproject.metrics.onionoo.util.FormattingUtils;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
+import org.apache.commons.codec.digest.DigestUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -20,7 +21,9 @@ import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -59,6 +62,10 @@ public class DocumentStore {
 
   private long storedBytes = 0L;
 
+  private long unchangedFiles = 0L;
+
+  private long unchangedBytes = 0L;
+
   private long retrievedFiles = 0L;
 
   private long retrievedBytes = 0L;
@@ -351,6 +358,17 @@ public class DocumentStore {
             documentString.length());
       }
       documentFile.getParentFile().mkdirs();
+      if (documentFile.exists()) {
+        try (InputStream stream = Files.newInputStream(documentFile.toPath())) {
+          String existingFileDigest = DigestUtils.sha256Hex(stream);
+          String newFileDigest = DigestUtils.sha256Hex(documentString);
+          if (existingFileDigest.equals(newFileDigest)) {
+            this.unchangedFiles++;
+            this.unchangedBytes += documentString.length();
+            return true;
+          }
+        }
+      }
       File documentTempFile = new File(
           documentFile.getAbsolutePath() + ".tmp");
       writeToFile(documentTempFile, documentString);
@@ -820,6 +838,8 @@ public class DocumentStore {
         + "    %s files listed\n"
         + "    %s files stored\n"
         + "    %s stored\n"
+        + "    %s files not rewritten\n"
+        + "    %s not rewritten\n"
         + "    %s files retrieved\n"
         + "    %s retrieved\n"
         + "    %s files removed\n",
@@ -827,6 +847,8 @@ public class DocumentStore {
         FormattingUtils.formatDecimalNumber(listedFiles),
         FormattingUtils.formatDecimalNumber(storedFiles),
         FormattingUtils.formatBytes(storedBytes),
+        FormattingUtils.formatDecimalNumber(unchangedFiles),
+        FormattingUtils.formatBytes(unchangedBytes),
         FormattingUtils.formatDecimalNumber(retrievedFiles),
         FormattingUtils.formatBytes(retrievedBytes),
         FormattingUtils.formatDecimalNumber(removedFiles));



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits