[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [onionoo/master] Support parsing archived descriptors from in/archive/.



commit 00494ff1f4ea85c8501cb56a0adba56fd3305616
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Sun Apr 26 19:10:34 2015 +0200

    Support parsing archived descriptors from in/archive/.
    
    Implements part of #13600.
---
 .../onionoo/updater/DescriptorQueue.java           |   29 +++--
 .../onionoo/updater/DescriptorSource.java          |  137 ++++++++++++++------
 2 files changed, 120 insertions(+), 46 deletions(-)

diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java b/src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java
index 93a291c..b86d585 100644
--- a/src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java
+++ b/src/main/java/org/torproject/onionoo/updater/DescriptorQueue.java
@@ -24,8 +24,6 @@ class DescriptorQueue {
   private final static Logger log = LoggerFactory.getLogger(
       DescriptorQueue.class);
 
-  private File inDir;
-
   private File statusDir;
 
   private DescriptorReader descriptorReader;
@@ -56,14 +54,21 @@ class DescriptorQueue {
     return this.returnedBytes;
   }
 
-  public DescriptorQueue(File inDir, File statusDir) {
-    this.inDir = inDir;
+  public DescriptorQueue(File inDir, DescriptorType descriptorType,
+      File statusDir) {
+    File directory = inDir;
+    if (descriptorType != null) {
+      directory = this.getDirectoryForDescriptorType(inDir,
+          descriptorType);
+    }
     this.statusDir = statusDir;
     this.descriptorReader =
         DescriptorSourceFactory.createDescriptorReader();
+    this.addDirectory(directory);
   }
 
-  public void addDirectory(DescriptorType descriptorType) {
+  public File getDirectoryForDescriptorType(File inDir,
+      DescriptorType descriptorType) {
     String directoryName = null;
     switch (descriptorType) {
     case RELAY_CONSENSUSES:
@@ -93,9 +98,16 @@ class DescriptorQueue {
     default:
       log.error("Unknown descriptor type.  Not adding directory "
           + "to descriptor reader.");
+      return null;
+    }
+    File directory = new File(inDir, directoryName);
+    return directory;
+  }
+
+  private void addDirectory(File directory) {
+    if (directory == null) {
       return;
     }
-    File directory = new File(this.inDir, directoryName);
     if (directory.exists() && directory.isDirectory()) {
       this.descriptorReader.addDirectory(directory);
       this.descriptorReader.setMaxDescriptorFilesInQueue(1);
@@ -107,6 +119,9 @@ class DescriptorQueue {
   }
 
   public void readHistoryFile(DescriptorHistory descriptorHistory) {
+    if (this.statusDir == null) {
+      return;
+    }
     String historyFileName = null;
     switch (descriptorHistory) {
     case RELAY_EXTRAINFO_HISTORY:
@@ -168,8 +183,6 @@ class DescriptorQueue {
 
   public void writeHistoryFile() {
     if (this.historyFile == null) {
-      log.error("Unable to write history file, possibly because of an "
-          + "unknown descriptor type.  Skipping.");
       return;
     }
     SortedMap<String, Long> excludedAndParsedFiles =
diff --git a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java
index 52e00c9..b26a692 100644
--- a/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java
+++ b/src/main/java/org/torproject/onionoo/updater/DescriptorSource.java
@@ -20,12 +20,15 @@ public class DescriptorSource {
   private static final Logger log = LoggerFactory.getLogger(
       DescriptorSource.class);
 
-  private final File inDir = new File("in/recent");
+  private final File inRecentDir = new File("in/recent"),
+      inArchiveDir = new File("in/archive");
 
   private final File statusDir = new File("status");
 
   private List<DescriptorQueue> descriptorQueues;
 
+  private DescriptorQueue archiveDescriptorQueue;
+
   public DescriptorSource() {
     this.descriptorQueues = new ArrayList<DescriptorQueue>();
     this.descriptorListeners =
@@ -35,9 +38,8 @@ public class DescriptorSource {
   private DescriptorQueue getDescriptorQueue(
       DescriptorType descriptorType,
       DescriptorHistory descriptorHistory) {
-    DescriptorQueue descriptorQueue = new DescriptorQueue(this.inDir,
-        this.statusDir);
-    descriptorQueue.addDirectory(descriptorType);
+    DescriptorQueue descriptorQueue = new DescriptorQueue(
+        this.inRecentDir, descriptorType, this.statusDir);
     if (descriptorHistory != null) {
       descriptorQueue.readHistoryFile(descriptorHistory);
     }
@@ -78,35 +80,32 @@ public class DescriptorSource {
   }
 
   public void readDescriptors() {
-    /* Careful when changing the order of parsing descriptor types!  The
-     * various status updaters may base assumptions on this order.  With
-     * #13674 being implemented, this may not be the case anymore. */
-
-    log.debug("Reading " + DescriptorType.RELAY_SERVER_DESCRIPTORS
+    this.readArchivedDescriptors();
+    log.debug("Reading recent " + DescriptorType.RELAY_SERVER_DESCRIPTORS
         + " ...");
     this.readDescriptors(DescriptorType.RELAY_SERVER_DESCRIPTORS,
         DescriptorHistory.RELAY_SERVER_HISTORY, true);
-    log.debug("Reading " + DescriptorType.RELAY_EXTRA_INFOS + " ...");
+    log.debug("Reading recent " + DescriptorType.RELAY_EXTRA_INFOS + " ...");
     this.readDescriptors(DescriptorType.RELAY_EXTRA_INFOS,
         DescriptorHistory.RELAY_EXTRAINFO_HISTORY, true);
-    log.debug("Reading " + DescriptorType.EXIT_LISTS + " ...");
+    log.debug("Reading recent " + DescriptorType.EXIT_LISTS + " ...");
     this.readDescriptors(DescriptorType.EXIT_LISTS,
         DescriptorHistory.EXIT_LIST_HISTORY, true);
-    log.debug("Reading " + DescriptorType.RELAY_CONSENSUSES + " ...");
+    log.debug("Reading recent " + DescriptorType.RELAY_CONSENSUSES + " ...");
     this.readDescriptors(DescriptorType.RELAY_CONSENSUSES,
         DescriptorHistory.RELAY_CONSENSUS_HISTORY, true);
-    log.debug("Reading " + DescriptorType.BRIDGE_SERVER_DESCRIPTORS
+    log.debug("Reading recent " + DescriptorType.BRIDGE_SERVER_DESCRIPTORS
         + " ...");
     this.readDescriptors(DescriptorType.BRIDGE_SERVER_DESCRIPTORS,
         DescriptorHistory.BRIDGE_SERVER_HISTORY, false);
-    log.debug("Reading " + DescriptorType.BRIDGE_EXTRA_INFOS + " ...");
+    log.debug("Reading recent " + DescriptorType.BRIDGE_EXTRA_INFOS + " ...");
     this.readDescriptors(DescriptorType.BRIDGE_EXTRA_INFOS,
         DescriptorHistory.BRIDGE_EXTRAINFO_HISTORY, false);
-    log.debug("Reading " + DescriptorType.BRIDGE_POOL_ASSIGNMENTS
+    log.debug("Reading recent " + DescriptorType.BRIDGE_POOL_ASSIGNMENTS
         + " ...");
     this.readDescriptors(DescriptorType.BRIDGE_POOL_ASSIGNMENTS,
         DescriptorHistory.BRIDGE_POOLASSIGN_HISTORY, false);
-    log.debug("Reading " + DescriptorType.BRIDGE_STATUSES + " ...");
+    log.debug("Reading recent " + DescriptorType.BRIDGE_STATUSES + " ...");
     this.readDescriptors(DescriptorType.BRIDGE_STATUSES,
         DescriptorHistory.BRIDGE_STATUS_HISTORY, false);
   }
@@ -128,34 +127,88 @@ public class DescriptorSource {
     }
     switch (descriptorType) {
     case RELAY_CONSENSUSES:
-      log.info("Read relay network consensuses");
+      log.info("Read recent relay network consensuses");
       break;
     case RELAY_SERVER_DESCRIPTORS:
-      log.info("Read relay server descriptors");
+      log.info("Read recent relay server descriptors");
       break;
     case RELAY_EXTRA_INFOS:
-      log.info("Read relay extra-info descriptors");
+      log.info("Read recent relay extra-info descriptors");
       break;
     case EXIT_LISTS:
-      log.info("Read exit lists");
+      log.info("Read recent exit lists");
       break;
     case BRIDGE_STATUSES:
-      log.info("Read bridge network statuses");
+      log.info("Read recent bridge network statuses");
       break;
     case BRIDGE_SERVER_DESCRIPTORS:
-      log.info("Read bridge server descriptors");
+      log.info("Read recent bridge server descriptors");
       break;
     case BRIDGE_EXTRA_INFOS:
-      log.info("Read bridge extra-info descriptors");
+      log.info("Read recent bridge extra-info descriptors");
       break;
     case BRIDGE_POOL_ASSIGNMENTS:
-      log.info("Read bridge-pool assignments");
+      log.info("Read recent bridge-pool assignments");
       break;
     }
   }
 
+  public void readArchivedDescriptors() {
+    if (!this.inArchiveDir.exists()) {
+      return;
+    }
+    log.info("Reading archived descriptors...");
+    this.archiveDescriptorQueue = new DescriptorQueue(this.inArchiveDir,
+        null, null);
+    Descriptor descriptor;
+    while ((descriptor = this.archiveDescriptorQueue.nextDescriptor())
+        != null) {
+      DescriptorType descriptorType = null;
+      boolean relay = false;
+      for (String annotation : descriptor.getAnnotations()) {
+        if (annotation.startsWith(
+            "@type network-status-consensus-3 1.")) {
+          descriptorType = DescriptorType.RELAY_CONSENSUSES;
+          relay = true;
+        } else if (annotation.startsWith("@type server-descriptor 1.")) {
+          descriptorType = DescriptorType.RELAY_SERVER_DESCRIPTORS;
+          relay = true;
+        } else if (annotation.startsWith("@type extra-info 1.")) {
+          descriptorType = DescriptorType.RELAY_EXTRA_INFOS;
+          relay = true;
+        } else if (annotation.startsWith("@type tordnsel 1.")) {
+          descriptorType = DescriptorType.EXIT_LISTS;
+          relay = true;
+        } else if (annotation.startsWith(
+            "@type bridge-network-status 1.")) {
+          descriptorType = DescriptorType.BRIDGE_STATUSES;
+          relay = false;
+        } else if (annotation.startsWith(
+            "@type bridge-server-descriptor 1.")) {
+          descriptorType = DescriptorType.BRIDGE_SERVER_DESCRIPTORS;
+          relay = false;
+        } else if (annotation.startsWith("@type bridge-extra-info 1.")) {
+          descriptorType = DescriptorType.BRIDGE_EXTRA_INFOS;
+          relay = false;
+        }
+      }
+      if (descriptorType == null) {
+        log.warn("Unrecognized descriptor in "
+            + this.inArchiveDir.getAbsolutePath() + " with annotations "
+            + descriptor.getAnnotations() + ".  Not reading any further"
+            + "archived descriptors.");
+        break;
+      }
+      for (DescriptorListener descriptorListener :
+          this.descriptorListeners.get(descriptorType)) {
+        descriptorListener.processDescriptor(descriptor, relay);
+      }
+    }
+    log.info("Read archived descriptors");
+  }
+
   public void writeHistoryFiles() {
-    log.debug("Writing history ");
+    log.debug("Writing parse histories for recent descriptors...");
     for (DescriptorQueue descriptorQueue : this.descriptorQueues) {
       descriptorQueue.writeHistoryFile();
     }
@@ -163,34 +216,42 @@ public class DescriptorSource {
 
   public String getStatsString() {
     StringBuilder sb = new StringBuilder();
-    sb.append("    " + this.localFilesBefore + " descriptor files found "
-        + "locally\n");
-    sb.append("    " + this.foundRemoteFiles + " descriptor files found "
-        + "remotely\n");
-    sb.append("    " + this.downloadedFiles + " descriptor files "
+    sb.append("    " + this.localFilesBefore + " recent descriptor files "
+        + "found locally\n");
+    sb.append("    " + this.foundRemoteFiles + " recent descriptor files "
+        + "found remotely\n");
+    sb.append("    " + this.downloadedFiles + " recent descriptor files "
         + "downloaded from remote\n");
-    sb.append("    " + this.deletedLocalFiles + " descriptor files "
-        + "deleted locally\n");
+    sb.append("    " + this.deletedLocalFiles + " recent descriptor "
+        + "files deleted locally\n");
     sb.append("    " + this.descriptorQueues.size() + " descriptor "
-        + "queues created\n");
+        + "queues created for recent descriptors\n");
     int historySizeBefore = 0, historySizeAfter = 0;
     long descriptors = 0L, bytes = 0L;
-    for (DescriptorQueue descriptorQueue : descriptorQueues) {
+    for (DescriptorQueue descriptorQueue : this.descriptorQueues) {
       historySizeBefore += descriptorQueue.getHistorySizeBefore();
       historySizeAfter += descriptorQueue.getHistorySizeAfter();
       descriptors += descriptorQueue.getReturnedDescriptors();
       bytes += descriptorQueue.getReturnedBytes();
     }
     sb.append("    " + FormattingUtils.formatDecimalNumber(
-        historySizeBefore) + " descriptors excluded from this "
+        historySizeBefore) + " recent descriptors excluded from this "
         + "execution\n");
     sb.append("    " + FormattingUtils.formatDecimalNumber(descriptors)
-        + " descriptors provided\n");
+        + " recent descriptors provided\n");
     sb.append("    " + FormattingUtils.formatBytes(bytes)
-        + " provided\n");
+        + " of recent descriptors provided\n");
     sb.append("    " + FormattingUtils.formatDecimalNumber(
-        historySizeAfter) + " descriptors excluded from next "
+        historySizeAfter) + " recent descriptors excluded from next "
         + "execution\n");
+    if (this.archiveDescriptorQueue != null) {
+      sb.append("    " + FormattingUtils.formatDecimalNumber(
+          this.archiveDescriptorQueue.getReturnedDescriptors()) +
+          " archived descriptors provided\n");
+      sb.append("    " + FormattingUtils.formatBytes(
+          this.archiveDescriptorQueue.getReturnedBytes()) + " of "
+          + "archived descriptors provided\n");
+    }
     return sb.toString();
   }
 }



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits