[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [metrics-lib/master] Implement a history file to avoid reading files twice.



commit fa07b8d59739c2f1d4813c2d315ab3e11a116d0d
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Sun Jan 15 07:32:38 2012 +0100

    Implement a history file to avoid reading files twice.
---
 .../descriptor/RelayDescriptorReader.java          |   20 +---
 .../descriptor/impl/RelayDescriptorReaderImpl.java |   99 +++++++++++++++-----
 2 files changed, 83 insertions(+), 36 deletions(-)

diff --git a/src/org/torproject/descriptor/RelayDescriptorReader.java b/src/org/torproject/descriptor/RelayDescriptorReader.java
index 11735c9..90063c3 100644
--- a/src/org/torproject/descriptor/RelayDescriptorReader.java
+++ b/src/org/torproject/descriptor/RelayDescriptorReader.java
@@ -13,20 +13,12 @@ public interface RelayDescriptorReader {
   /* Add a local directory to read relay descriptors from. */
   public void addDirectory(File directory);
 
-  /* Exclude the given file from the results. */
-  public void setExcludeFile(File fileToExclude);
-
-  /* Exclude the given files from the results. */
-  public void setExcludeFiles(Set<File> filesToExclude);
-
-  /* Exclude the given file from the results if it wasn't modified since
-   * the given timestamp. */
-  public void setExcludeFile(File fileToExclude, long lastModifiedMillis);
-
-  /* Exclude the given files from the results if they were not modified
-   * since the given timestamps.  Map keys are files to exclude and map
-   * values are last modified timestamps. */
-  public void setExcludeFiles(Map<File, Long> filesToExclude);
+  /* Exclude files that are contained in the given history file and that
+   * haven't changed since they were last read.  Add reads from the
+   * current run to the history file.  Remove files that don't exist
+   * anymore from the history file.  Lines in the history file contain the
+   * last modified timestamp and the absolute path of a file. */
+  public void setExcludeFiles(File historyFile);
 
   /* Read the previously configured relay descriptors and make them
    * available via the returned blocking iterator.  Whenever the reader
diff --git a/src/org/torproject/descriptor/impl/RelayDescriptorReaderImpl.java b/src/org/torproject/descriptor/impl/RelayDescriptorReaderImpl.java
index b0e2b81..81435e0 100644
--- a/src/org/torproject/descriptor/impl/RelayDescriptorReaderImpl.java
+++ b/src/org/torproject/descriptor/impl/RelayDescriptorReaderImpl.java
@@ -3,9 +3,13 @@
 package org.torproject.descriptor.impl;
 
 import java.io.BufferedInputStream;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
+import java.io.FileReader;
 import java.io.FileInputStream;
+import java.io.FileWriter;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -13,7 +17,9 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.SortedMap;
 import java.util.Stack;
+import java.util.TreeMap;
 import org.torproject.descriptor.Descriptor;
 import org.torproject.descriptor.DescriptorFile;
 import org.torproject.descriptor.RelayDescriptorReader;
@@ -27,32 +33,16 @@ public class RelayDescriptorReaderImpl implements RelayDescriptorReader {
     this.directories.add(directory);
   }
 
-  public void setExcludeFile(File fileToExclude) {
-    throw new UnsupportedOperationException("Not implemented yet.");
-    /* TODO Implement me. */
-  }
-
-  public void setExcludeFiles(Set<File> filesToExclude) {
-    throw new UnsupportedOperationException("Not implemented yet.");
-    /* TODO Implement me. */
-  }
-
-  public void setExcludeFile(File fileToExclude,
-      long lastModifiedMillis) {
-    throw new UnsupportedOperationException("Not implemented yet.");
-    /* TODO Implement me. */
-  }
-
-  public void setExcludeFiles(Map<File, Long> filesToExclude) {
-    throw new UnsupportedOperationException("Not implemented yet.");
-    /* TODO Implement me. */
+  private File historyFile;
+  public void setExcludeFiles(File historyFile) {
+    this.historyFile = historyFile;
   }
 
   public Iterator<DescriptorFile> readDescriptors() {
     BlockingIteratorImpl<DescriptorFile> descriptorQueue =
         new BlockingIteratorImpl<DescriptorFile>();
     DescriptorReader reader = new DescriptorReader(this.directories,
-        descriptorQueue);
+        descriptorQueue, this.historyFile);
     new Thread(reader).start();
     return descriptorQueue;
   }
@@ -60,12 +50,69 @@ public class RelayDescriptorReaderImpl implements RelayDescriptorReader {
   private static class DescriptorReader implements Runnable {
     private List<File> directories;
     private BlockingIteratorImpl<DescriptorFile> descriptorQueue;
+    private File historyFile;
     private DescriptorReader(List<File> directories,
-        BlockingIteratorImpl<DescriptorFile> descriptorQueue) {
+        BlockingIteratorImpl<DescriptorFile> descriptorQueue,
+        File historyFile) {
       this.directories = directories;
       this.descriptorQueue = descriptorQueue;
+      this.historyFile = historyFile;
     }
     public void run() {
+      this.readOldHistory();
+      this.readDescriptors();
+      this.writeNewHistory();
+    }
+    private SortedMap<String, Long>
+        oldHistory = new TreeMap<String, Long>(),
+        newHistory = new TreeMap<String, Long>();
+    private void readOldHistory() {
+      if (this.historyFile == null) {
+        return;
+      }
+      try {
+        BufferedReader br = new BufferedReader(new FileReader(
+            this.historyFile));
+        String line;
+        while ((line = br.readLine()) != null) {
+          if (!line.contains(" ")) {
+            /* TODO Handle this problem? */
+            continue;
+          }
+          long lastModifiedMillis = Long.parseLong(line.substring(0,
+              line.indexOf(" ")));
+          String absolutePath = line.substring(line.indexOf(" ") + 1);
+          this.oldHistory.put(absolutePath, lastModifiedMillis);
+        }
+        br.close();
+      } catch (IOException e) {
+        /* TODO Handle this exception. */
+      } catch (NumberFormatException e) {
+        /* TODO Handle this exception. */
+      }
+    }
+    private void writeNewHistory() {
+      if (this.historyFile == null) {
+        return;
+      }
+      try {
+        if (this.historyFile.getParentFile() != null) {
+          this.historyFile.getParentFile().mkdirs();
+        }
+        BufferedWriter bw = new BufferedWriter(new FileWriter(
+            this.historyFile));
+        for (Map.Entry<String, Long> e : this.newHistory.entrySet()) {
+          String absolutePath = e.getKey();
+          long lastModifiedMillis = e.getValue();
+          bw.write(String.valueOf(lastModifiedMillis) + " " + absolutePath
+              + "\n");
+        }
+        bw.close();
+      } catch (IOException e) {
+        /* TODO Handle this exception. */
+      }
+    }
+    private void readDescriptors() {
       for (File directory : this.directories) {
         try {
           Stack<File> files = new Stack<File>();
@@ -75,13 +122,21 @@ public class RelayDescriptorReaderImpl implements RelayDescriptorReader {
             if (file.isDirectory()) {
               files.addAll(Arrays.asList(file.listFiles()));
             } else {
+              String absolutePath = file.getAbsolutePath();
+              long lastModifiedMillis = file.lastModified();
+              this.newHistory.put(absolutePath, lastModifiedMillis);
+              if (this.oldHistory.containsKey(absolutePath) &&
+                  this.oldHistory.get(absolutePath) ==
+                  lastModifiedMillis) {
+                continue;
+              }
               try {
                 List<Descriptor> parsedDescriptors = this.readFile(file);
                 DescriptorFileImpl descriptorFile =
                     new DescriptorFileImpl();
                 descriptorFile.setDirectory(directory);
                 descriptorFile.setFile(file);
-                descriptorFile.setLastModified(file.lastModified());
+                descriptorFile.setLastModified(lastModifiedMillis);
                 descriptorFile.setDescriptors(parsedDescriptors);
                 this.descriptorQueue.add(descriptorFile);
               } catch (DescriptorParseException e) {

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits