[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [metrics-web/master] Use metrics-lib for parsing relay descriptors.



commit 72c1aee10e13d199a2e604adb47c670c417ab0a0
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Wed Mar 21 14:44:30 2012 +0100

    Use metrics-lib for parsing relay descriptors.
---
 config.template                                    |    2 +-
 src/org/torproject/ernie/cron/ArchiveReader.java   |  573 +++++---------------
 src/org/torproject/ernie/cron/Configuration.java   |    2 +-
 .../cron/RelayDescriptorDatabaseImporter.java      |   32 +-
 4 files changed, 155 insertions(+), 454 deletions(-)

diff --git a/config.template b/config.template
index 8c56909..ce2dc94 100644
--- a/config.template
+++ b/config.template
@@ -2,7 +2,7 @@
 #ImportDirectoryArchives 0
 #
 ## Relative path to directory to import directory archives from
-#DirectoryArchivesDirectory archives/
+#DirectoryArchivesDirectory in/relay-descriptors/
 #
 ## Keep a history of imported directory archive files to know which files
 ## have been imported before. This history can be useful when importing
diff --git a/src/org/torproject/ernie/cron/ArchiveReader.java b/src/org/torproject/ernie/cron/ArchiveReader.java
index ed1e505..5703661 100644
--- a/src/org/torproject/ernie/cron/ArchiveReader.java
+++ b/src/org/torproject/ernie/cron/ArchiveReader.java
@@ -2,34 +2,29 @@
  * See LICENSE for licensing information */
 package org.torproject.ernie.cron;
 
-import java.io.BufferedInputStream;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.ByteArrayOutputStream;
 import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.StringReader;
-import java.text.ParseException;
+import java.io.UnsupportedEncodingException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.Stack;
 import java.util.TimeZone;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.codec.DecoderException;
 import org.apache.commons.codec.binary.Hex;
 import org.apache.commons.codec.digest.DigestUtils;
+import org.torproject.descriptor.Descriptor;
+import org.torproject.descriptor.DescriptorFile;
+import org.torproject.descriptor.DescriptorReader;
+import org.torproject.descriptor.DescriptorSourceFactory;
+import org.torproject.descriptor.ExtraInfoDescriptor;
+import org.torproject.descriptor.NetworkStatusEntry;
+import org.torproject.descriptor.RelayNetworkStatusConsensus;
+import org.torproject.descriptor.RelayNetworkStatusVote;
+import org.torproject.descriptor.ServerDescriptor;
 
 /**
  * Read in all files in a given directory and pass buffered readers of
@@ -72,439 +67,159 @@ public class ArchiveReader {
 
     int parsedFiles = 0, ignoredFiles = 0;
     this.logger = Logger.getLogger(ArchiveReader.class.getName());
-    SortedMap<String, Long>
-        lastArchivesImportHistory = new TreeMap<String, Long>(),
-        newArchivesImportHistory = new TreeMap<String, Long>();
-    File archivesImportHistoryFile = new File(statsDirectory,
-        "archives-import-history");
-    if (keepImportHistory && archivesImportHistoryFile.exists()) {
-      try {
-        BufferedReader br = new BufferedReader(new FileReader(
-            archivesImportHistoryFile));
-        String line = null;
-        while ((line = br.readLine()) != null) {
-          String[] parts = line.split(",");
-          if (parts.length < 2) {
-            logger.warning("Archives import history file does not "
-                + "contain timestamps. Skipping.");
-            break;
-          }
-          long lastModified = Long.parseLong(parts[0]);
-          String filename = parts[1];
-          lastArchivesImportHistory.put(filename, lastModified);
-        }
-        br.close();
-      } catch (IOException e) {
-        logger.log(Level.WARNING, "Could not read in archives import "
-            + "history file. Skipping.");
-      }
-    }
     if (archivesDirectory.exists()) {
       logger.fine("Importing files in directory " + archivesDirectory
           + "/...");
-      Stack<File> filesInInputDir = new Stack<File>();
-      filesInInputDir.add(archivesDirectory);
-      List<File> problems = new ArrayList<File>();
-      while (!filesInInputDir.isEmpty()) {
-        File pop = filesInInputDir.pop();
-        if (pop.isDirectory()) {
-          for (File f : pop.listFiles()) {
-            filesInInputDir.add(f);
-          }
-        } else {
-          try {
-            long lastModified = pop.lastModified();
-            String filename = pop.getName();
-            if (keepImportHistory) {
-              newArchivesImportHistory.put(filename, lastModified);
-            }
-            if (keepImportHistory &&
-                lastArchivesImportHistory.containsKey(filename) &&
-                lastArchivesImportHistory.get(filename) >= lastModified) {
-              ignoredFiles++;
-              continue;
-            } else if (filename.endsWith(".tar.bz2")) {
-              logger.warning("Cannot parse compressed tarball "
-                  + pop.getAbsolutePath() + ". Skipping.");
-              continue;
-            }
-            FileInputStream fis = new FileInputStream(pop);
-            BufferedInputStream bis = new BufferedInputStream(fis);
-            ByteArrayOutputStream baos = new ByteArrayOutputStream();
-            int len;
-            byte[] data = new byte[1024];
-            while ((len = bis.read(data, 0, 1024)) >= 0) {
-              baos.write(data, 0, len);
-            }
-            bis.close();
-            byte[] allData = baos.toByteArray();
-            this.parse(allData);
-            parsedFiles++;
-          } catch (IOException e) {
-            problems.add(pop);
-            if (problems.size() > 3) {
-              break;
-            }
-          }
-        }
+      DescriptorReader reader =
+          DescriptorSourceFactory.createDescriptorReader();
+      reader.addDirectory(archivesDirectory);
+      if (keepImportHistory) {
+        reader.setExcludeFiles(new File(statsDirectory,
+            "relay-descriptor-history"));
       }
-      if (problems.isEmpty()) {
-        logger.fine("Finished importing files in directory "
-            + archivesDirectory + "/.");
-      } else {
-        StringBuilder sb = new StringBuilder("Failed importing files in "
-            + "directory " + archivesDirectory + "/:");
-        int printed = 0;
-        for (File f : problems) {
-          sb.append("\n  " + f.getAbsolutePath());
-          if (++printed >= 3) {
-            sb.append("\n  ... more");
-            break;
+      Iterator<DescriptorFile> descriptorFiles = reader.readDescriptors();
+      while (descriptorFiles.hasNext()) {
+        DescriptorFile descriptorFile = descriptorFiles.next();
+        if (descriptorFile.getDescriptors() != null) {
+          for (Descriptor descriptor : descriptorFile.getDescriptors()) {
+            if (descriptor instanceof RelayNetworkStatusConsensus) {
+              this.addRelayNetworkStatusConsensus(
+                  (RelayNetworkStatusConsensus) descriptor);
+            } else if (descriptor instanceof RelayNetworkStatusVote) {
+              this.addRelayNetworkStatusVote(
+                  (RelayNetworkStatusVote) descriptor);
+            } else if (descriptor instanceof ServerDescriptor) {
+              this.addServerDescriptor((ServerDescriptor) descriptor);
+            } else if (descriptor instanceof ExtraInfoDescriptor) {
+              this.addExtraInfoDescriptor(
+                  (ExtraInfoDescriptor) descriptor);
+            }
           }
         }
-        logger.warning(sb.toString());
-      }
-    }
-    if (keepImportHistory) {
-      try {
-        archivesImportHistoryFile.getParentFile().mkdirs();
-        BufferedWriter bw = new BufferedWriter(new FileWriter(
-            archivesImportHistoryFile));
-        for (Map.Entry<String, Long> historyEntry :
-            newArchivesImportHistory.entrySet()) {
-          bw.write(String.valueOf(historyEntry.getValue()) + ","
-              + historyEntry.getKey() + "\n");
-        }
-        bw.close();
-      } catch (IOException e) {
-        logger.log(Level.WARNING, "Could not write archives import "
-            + "history file.");
       }
     }
+
     logger.info("Finished importing relay descriptors from local "
         + "directory:\nParsed " + parsedFiles + ", ignored "
         + ignoredFiles + " files.");
   }
 
-  public void parse(byte[] data) {
-    try {
-      /* Remove any @ lines at the beginning of the file and parse the
-       * first non-@ line to find out the descriptor type. */
-      BufferedReader br = new BufferedReader(new StringReader(new String(
-          data, "US-ASCII")));
-      String line = br.readLine();
-      while (line != null && line.startsWith("@")) {
-        line = br.readLine();
-      }
-      if (line == null) {
-        this.logger.fine("We were given a file that doesn't contain a "
-            + "single descriptor for parsing. Ignoring.");
-        return;
+  private void addRelayNetworkStatusConsensus(
+      RelayNetworkStatusConsensus consensus) {
+    for (NetworkStatusEntry statusEntry :
+      consensus.getStatusEntries().values()) {
+      this.rddi.addStatusEntry(consensus.getValidAfterMillis(),
+          statusEntry.getNickname(), statusEntry.getFingerprint(),
+          statusEntry.getDescriptor(), statusEntry.getPublishedMillis(),
+          statusEntry.getAddress(), statusEntry.getOrPort(),
+          statusEntry.getDirPort(), statusEntry.getFlags(),
+          statusEntry.getVersion(), statusEntry.getBandwidth(),
+          statusEntry.getPortList(), statusEntry.getStatusEntryBytes());
+      try {
+        this.bsfh.addHashedRelay(DigestUtils.shaHex(Hex.decodeHex(
+            statusEntry.getFingerprint().toCharArray())).toUpperCase());
+      } catch (DecoderException e) {
       }
-      br.close();
+    }
+    this.rddi.addConsensus(consensus.getValidAfterMillis(),
+        consensus.getRawDescriptorBytes());
+  }
 
-      /* Split the byte[] possibly containing multiple descriptors into
-       * byte[]'s with one descriptor each and parse them. */
-      String startToken = null;
-      if (line.equals("network-status-version 3")) {
-        startToken = "network-status-version 3";
-      } else if (line.startsWith("router ")) {
-        startToken = "router ";
-      } else if (line.startsWith("extra-info ")) {
-        startToken = "extra-info ";
-      } else if (line.equals("dir-key-certificate-version 3")) {
-        this.logger.fine("Not parsing dir key certificate.");
-        return;
-      } else {
-        this.logger.warning("Unknown descriptor type.  First line is '"
-            + line + "'.  Ignoring.");
-        return;
-      }
-      String splitToken = "\n" + startToken;
-      String ascii = new String(data, "US-ASCII");
-      int length = data.length, start = ascii.indexOf(startToken);
-      while (start < length) {
-        int end = ascii.indexOf(splitToken, start);
-        if (end < 0) {
-          end = length;
-        } else {
-          end += 1;
-        }
-        byte[] descBytes = new byte[end - start];
-        System.arraycopy(data, start, descBytes, 0, end - start);
-        parseSingleDescriptor(descBytes);
-        start = end;
+  private void addRelayNetworkStatusVote(RelayNetworkStatusVote vote) {
+    this.rddi.addVote(vote.getValidAfterMillis(), vote.getIdentity(),
+        vote.getRawDescriptorBytes());
+  }
+
+  private void addServerDescriptor(ServerDescriptor descriptor) {
+    String digest = null;
+    try {
+      String ascii = new String(descriptor.getRawDescriptorBytes(),
+          "US-ASCII");
+      String startToken = "router ";
+      String sigToken = "\nrouter-signature\n";
+      int start = ascii.indexOf(startToken);
+      int sig = ascii.indexOf(sigToken) + sigToken.length();
+      if (start >= 0 || sig >= 0 || sig > start) {
+        byte[] forDigest = new byte[sig - start];
+        System.arraycopy(descriptor.getRawDescriptorBytes(), start,
+            forDigest, 0, sig - start);
+        digest = DigestUtils.shaHex(forDigest);
       }
-    } catch (IOException e) {
-      this.logger.log(Level.WARNING, "Could not parse descriptor. "
-          + "Skipping.", e);
+    } catch (UnsupportedEncodingException e) {
+    }
+    if (digest != null) {
+      this.rddi.addServerDescriptor(digest, descriptor.getNickname(),
+          descriptor.getAddress(), descriptor.getOrPort(),
+          descriptor.getDirPort(), descriptor.getFingerprint(),
+          descriptor.getBandwidthRate(), descriptor.getBandwidthBurst(),
+          descriptor.getBandwidthObserved(), descriptor.getPlatform(),
+          descriptor.getPublishedMillis(), descriptor.getUptime(),
+          descriptor.getExtraInfoDigest(),
+          descriptor.getRawDescriptorBytes());
     }
   }
 
-  private void parseSingleDescriptor(byte[] data) {
+  private void addExtraInfoDescriptor(ExtraInfoDescriptor descriptor) {
+    if (descriptor.getDirreqV3Reqs() != null) {
+      int allUsers = 0;
+      Map<String, String> obs = new HashMap<String, String>();
+      for (Map.Entry<String, Integer> e :
+          descriptor.getDirreqV3Reqs().entrySet()) {
+        String country = e.getKey();
+        int users = e.getValue() - 4;
+        allUsers += users;
+        obs.put(country, "" + users);
+      }
+      obs.put("zy", "" + allUsers);
+      this.rddi.addDirReqStats(descriptor.getFingerprint(),
+          descriptor.getDirreqStatsEndMillis(),
+          descriptor.getDirreqStatsIntervalLength(), obs);
+    }
+    if (descriptor.getConnBiDirectStatsEndMillis() >= 0L) {
+      this.rddi.addConnBiDirect(descriptor.getFingerprint(),
+          descriptor.getConnBiDirectStatsEndMillis(),
+          descriptor.getConnBiDirectStatsIntervalLength(),
+          descriptor.getConnBiDirectBelow(),
+          descriptor.getConnBiDirectRead(),
+          descriptor.getConnBiDirectWrite(),
+          descriptor.getConnBiDirectBoth());
+    }
+    List<String> bandwidthHistoryLines = new ArrayList<String>();
+    if (descriptor.getWriteHistory() != null) {
+      bandwidthHistoryLines.add(descriptor.getWriteHistory().getLine());
+    }
+    if (descriptor.getReadHistory() != null) {
+      bandwidthHistoryLines.add(descriptor.getReadHistory().getLine());
+    }
+    if (descriptor.getDirreqWriteHistory() != null) {
+      bandwidthHistoryLines.add(
+          descriptor.getDirreqWriteHistory().getLine());
+    }
+    if (descriptor.getDirreqReadHistory() != null) {
+      bandwidthHistoryLines.add(
+          descriptor.getDirreqReadHistory().getLine());
+    }
+    String digest = null;
     try {
-      /* Convert descriptor to ASCII for parsing. This means we'll lose
-       * the non-ASCII chars, but we don't care about them for parsing
-       * anyway. */
-      BufferedReader br = new BufferedReader(new StringReader(new String(
-          data, "US-ASCII")));
-      String line = br.readLine();
-      SimpleDateFormat parseFormat =
-          new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
-      parseFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-      if (line.equals("network-status-version 3")) {
-        // TODO when parsing the current consensus, check the fresh-until
-        // time to see when we switch from hourly to half-hourly
-        // consensuses
-        boolean isConsensus = true;
-        String validAfterTime = null, nickname = null,
-            relayIdentity = null, serverDesc = null, version = null,
-            ports = null;
-        String dirSource = null, address = null;
-        long validAfter = -1L, published = -1L, bandwidth = -1L,
-            orPort = 0L, dirPort = 0L;
-        SortedSet<String> relayFlags = null;
-        StringBuilder rawStatusEntry = null;
-        SortedSet<String> hashedRelayIdentities = new TreeSet<String>();
-        while ((line = br.readLine()) != null) {
-          if (line.equals("vote-status vote")) {
-            isConsensus = false;
-          } else if (line.startsWith("valid-after ")) {
-            validAfterTime = line.substring("valid-after ".length());
-            validAfter = parseFormat.parse(validAfterTime).getTime();
-          } else if (line.startsWith("dir-source ")) {
-            dirSource = line.split(" ")[2];
-          } else if (line.startsWith("r ")) {
-            if (isConsensus && relayIdentity != null &&
-                this.rddi != null) {
-              byte[] rawDescriptor = rawStatusEntry.toString().getBytes();
-              this.rddi.addStatusEntry(validAfter, nickname,
-                  relayIdentity, serverDesc, published, address, orPort,
-                  dirPort, relayFlags, version, bandwidth, ports,
-                  rawDescriptor);
-              relayFlags = null;
-              version = null;
-              bandwidth = -1L;
-              ports = null;
-            }
-            rawStatusEntry = new StringBuilder(line + "\n");
-            String[] parts = line.split(" ");
-            if (parts.length < 9) {
-              this.logger.log(Level.WARNING, "Could not parse r line '"
-                  + line + "' in descriptor. Skipping.");
-              break;
-            }
-            nickname = parts[1];
-            relayIdentity = Hex.encodeHexString(
-                Base64.decodeBase64(parts[2] + "=")).
-                toLowerCase();
-            hashedRelayIdentities.add(DigestUtils.shaHex(
-                Base64.decodeBase64(parts[2] + "=")).
-                toUpperCase());
-            serverDesc = Hex.encodeHexString(Base64.decodeBase64(
-                parts[3] + "=")).toLowerCase();
-            published = parseFormat.parse(parts[4] + " " + parts[5]).
-                getTime();
-            address = parts[6];
-            orPort = Long.parseLong(parts[7]);
-            dirPort = Long.parseLong(parts[8]);
-          } else if (line.startsWith("s ") || line.equals("s")) {
-            rawStatusEntry.append(line + "\n");
-            relayFlags = new TreeSet<String>();
-            if (line.length() > 2) {
-              for (String flag : line.substring(2).split(" ")) {
-                relayFlags.add(flag);
-              }
-            }
-          } else if (line.startsWith("v ")) {
-            rawStatusEntry.append(line + "\n");
-            version = line.substring(2);
-          } else if (line.startsWith("w ")) {
-            rawStatusEntry.append(line + "\n");
-            String[] parts = line.split(" ");
-            for (String part : parts) {
-              if (part.startsWith("Bandwidth=")) {
-                bandwidth = Long.parseLong(part.substring(
-                    "Bandwidth=".length()));
-              }
-            }
-          } else if (line.startsWith("p ")) {
-            rawStatusEntry.append(line + "\n");
-            ports = line.substring(2);
-          }
-        }
-        if (isConsensus) {
-          if (this.bsfh != null) {
-            for (String hashedRelayIdentity : hashedRelayIdentities) {
-              this.bsfh.addHashedRelay(hashedRelayIdentity);
-            }
-          }
-          if (this.rddi != null) {
-            this.rddi.addConsensus(validAfter, data);
-            if (relayIdentity != null) {
-              byte[] rawDescriptor = rawStatusEntry.toString().getBytes();
-              this.rddi.addStatusEntry(validAfter, nickname,
-                  relayIdentity, serverDesc, published, address, orPort,
-                  dirPort, relayFlags, version, bandwidth, ports,
-                  rawDescriptor);
-            }
-          }
-        } else {
-          if (this.rddi != null) {
-            this.rddi.addVote(validAfter, dirSource, data);
-          }
-        }
-      } else if (line.startsWith("router ")) {
-        String platformLine = null, bandwidthLine = null,
-            extraInfoDigest = null, relayIdentifier = null;
-        String[] parts = line.split(" ");
-        String nickname = parts[1];
-        String address = parts[2];
-        int orPort = Integer.parseInt(parts[3]);
-        int dirPort = Integer.parseInt(parts[4]);
-        long published = -1L, uptime = -1L;
-        while ((line = br.readLine()) != null) {
-          if (line.startsWith("platform ")) {
-            platformLine = line;
-          } else if (line.startsWith("published ")) {
-            String publishedTime = line.substring("published ".length());
-            published = parseFormat.parse(publishedTime).getTime();
-          } else if (line.startsWith("opt fingerprint") ||
-              line.startsWith("fingerprint")) {
-            relayIdentifier = line.substring(line.startsWith("opt ") ?
-                "opt fingerprint".length() : "fingerprint".length()).
-                replaceAll(" ", "").toLowerCase();
-          } else if (line.startsWith("bandwidth ")) {
-            bandwidthLine = line;
-          } else if (line.startsWith("opt extra-info-digest ") ||
-              line.startsWith("extra-info-digest ")) {
-            extraInfoDigest = line.startsWith("opt ") ?
-                line.split(" ")[2].toLowerCase() :
-                line.split(" ")[1].toLowerCase();
-          } else if (line.startsWith("uptime ")) {
-            uptime = Long.parseLong(line.substring("uptime ".length()));
-          }
-        }
-        String ascii = new String(data, "US-ASCII");
-        String startToken = "router ";
-        String sigToken = "\nrouter-signature\n";
-        int start = ascii.indexOf(startToken);
-        int sig = ascii.indexOf(sigToken) + sigToken.length();
-        String digest = null;
-        if (start >= 0 || sig >= 0 || sig > start) {
-          byte[] forDigest = new byte[sig - start];
-          System.arraycopy(data, start, forDigest, 0, sig - start);
-          digest = DigestUtils.shaHex(forDigest);
-        }
-        if (this.rddi != null && digest != null) {
-          String[] bwParts = bandwidthLine.split(" ");
-          long bandwidthAvg = Long.parseLong(bwParts[1]);
-          long bandwidthBurst = Long.parseLong(bwParts[2]);
-          long bandwidthObserved = Long.parseLong(bwParts[3]);
-          String platform = platformLine.substring("platform ".length());
-          this.rddi.addServerDescriptor(digest, nickname, address, orPort,
-              dirPort, relayIdentifier, bandwidthAvg, bandwidthBurst,
-              bandwidthObserved, platform, published, uptime,
-              extraInfoDigest, data);
-        }
-      } else if (line.startsWith("extra-info ")) {
-        String nickname = line.split(" ")[1];
-        long published = -1L;
-        String dir = line.split(" ")[2];
-        String statsEnd = null;
-        long seconds = -1L;
-        List<String> bandwidthHistory = new ArrayList<String>();
-        while ((line = br.readLine()) != null) {
-          if (line.startsWith("published ")) {
-            String publishedTime = line.substring("published ".length());
-            published = parseFormat.parse(publishedTime).getTime();
-          } else if (line.startsWith("read-history ") ||
-              line.startsWith("write-history ") ||
-              line.startsWith("dirreq-read-history ") ||
-              line.startsWith("dirreq-write-history ")) {
-            bandwidthHistory.add(line);
-          } else if (line.startsWith("dirreq-stats-end ")) {
-            String[] parts = line.split(" ");
-            if (parts.length < 5) {
-              this.logger.warning("Could not parse dirreq-stats-end "
-                  + "line '" + line + "' in descriptor. Skipping.");
-              break;
-            }
-            statsEnd = parts[1] + " " + parts[2];
-            seconds = Long.parseLong(parts[3].substring(1));
-          } else if (line.startsWith("dirreq-v3-reqs ")
-              && line.length() > "dirreq-v3-reqs ".length()) {
-            if (this.rddi != null) {
-              try {
-                int allUsers = 0;
-                Map<String, String> obs = new HashMap<String, String>();
-                String[] parts = line.substring("dirreq-v3-reqs ".
-                    length()).split(",");
-                for (String p : parts) {
-                  String country = p.substring(0, 2);
-                  int users = Integer.parseInt(p.substring(3)) - 4;
-                  allUsers += users;
-                  obs.put(country, "" + users);
-                }
-                obs.put("zy", "" + allUsers);
-                this.rddi.addDirReqStats(dir, statsEnd, seconds, obs);
-              } catch (NumberFormatException e) {
-                this.logger.log(Level.WARNING, "Could not parse "
-                    + "dirreq-v3-reqs line '" + line + "' in descriptor. "
-                    + "Skipping.", e);
-                break;
-              }
-            }
-          } else if (line.startsWith("conn-bi-direct ")) {
-            if (this.rddi != null) {
-              String[] parts = line.split(" ");
-              if (parts.length == 6 &&
-                  parts[5].split(",").length == 4) {
-                try {
-                  String connBiDirectStatsEnd = parts[1] + " " + parts[2];
-                  long connBiDirectSeconds = Long.parseLong(parts[3].
-                      substring(1));
-                  String[] parts2 = parts[5].split(",");
-                  long below = Long.parseLong(parts2[0]);
-                  long read = Long.parseLong(parts2[1]);
-                  long write = Long.parseLong(parts2[2]);
-                  long both = Long.parseLong(parts2[3]);
-                  this.rddi.addConnBiDirect(dir, connBiDirectStatsEnd,
-                      connBiDirectSeconds, below, read, write, both);
-                } catch (NumberFormatException e) {
-                  this.logger.log(Level.WARNING, "Number format "
-                      + "exception while parsing conn-bi-direct stats "
-                      + "string '" + line + "'. Skipping.", e);
-                }
-              } else {
-                this.logger.warning("Skipping invalid conn-bi-direct "
-                    + "stats string '" + line + "'.");
-              }
-            }
-          }
-        }
-        String ascii = new String(data, "US-ASCII");
-        String startToken = "extra-info ";
-        String sigToken = "\nrouter-signature\n";
-        String digest = null;
-        int start = ascii.indexOf(startToken);
-        int sig = ascii.indexOf(sigToken) + sigToken.length();
-        if (start >= 0 || sig >= 0 || sig > start) {
-          byte[] forDigest = new byte[sig - start];
-          System.arraycopy(data, start, forDigest, 0, sig - start);
-          digest = DigestUtils.shaHex(forDigest);
-        }
-        if (this.rddi != null && digest != null) {
-          this.rddi.addExtraInfoDescriptor(digest, nickname,
-              dir.toLowerCase(), published, data, bandwidthHistory);
-        }
+      String ascii = new String(descriptor.getRawDescriptorBytes(),
+          "US-ASCII");
+      String startToken = "extra-info ";
+      String sigToken = "\nrouter-signature\n";
+      int start = ascii.indexOf(startToken);
+      int sig = ascii.indexOf(sigToken) + sigToken.length();
+      if (start >= 0 || sig >= 0 || sig > start) {
+        byte[] forDigest = new byte[sig - start];
+        System.arraycopy(descriptor.getRawDescriptorBytes(), start,
+            forDigest, 0, sig - start);
+        digest = DigestUtils.shaHex(forDigest);
       }
-    } catch (IOException e) {
-      this.logger.log(Level.WARNING, "Could not parse descriptor. "
-          + "Skipping.", e);
-    } catch (ParseException e) {
-      this.logger.log(Level.WARNING, "Could not parse descriptor. "
-          + "Skipping.", e);
+    } catch (UnsupportedEncodingException e) {
+    }
+    if (digest != null) {
+      this.rddi.addExtraInfoDescriptor(digest, descriptor.getNickname(),
+          descriptor.getFingerprint().toLowerCase(),
+          descriptor.getPublishedMillis(),
+          descriptor.getRawDescriptorBytes(), bandwidthHistoryLines);
     }
   }
 }
diff --git a/src/org/torproject/ernie/cron/Configuration.java b/src/org/torproject/ernie/cron/Configuration.java
index 0a493cb..1f30eb9 100644
--- a/src/org/torproject/ernie/cron/Configuration.java
+++ b/src/org/torproject/ernie/cron/Configuration.java
@@ -17,7 +17,7 @@ import java.util.logging.Logger;
  */
 public class Configuration {
   private boolean importDirectoryArchives = false;
-  private String directoryArchivesDirectory = "archives/";
+  private String directoryArchivesDirectory = "in/relay-descriptors/";
   private boolean keepDirectoryArchiveImportHistory = false;
   private boolean importSanitizedBridges = false;
   private String sanitizedBridgesDirectory = "bridges/";
diff --git a/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java b/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java
index 35ba90e..3ee7b6b 100644
--- a/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java
+++ b/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java
@@ -1017,21 +1017,14 @@ public final class RelayDescriptorDatabaseImporter {
   /**
    * Insert a conn-bi-direct stats string into the database.
    */
-  public void addConnBiDirect(String source, String statsEnd,
+  public void addConnBiDirect(String source, long statsEndMillis,
       long seconds, long below, long read, long write, long both) {
-    long statsEndTime = 0L;
-    try {
-      statsEndTime = this.dateTimeFormat.parse(statsEnd).getTime();
-    } catch (ParseException e) {
-      this.logger.log(Level.WARNING, "Could not add conn-bi-direct "
-          + "stats string with interval ending '" + statsEnd + "'.", e);
-      return;
-    }
+    String statsEnd = this.dateTimeFormat.format(statsEndMillis);
     if (this.importIntoDatabase) {
       try {
-        this.addDateToScheduledUpdates(statsEndTime);
+        this.addDateToScheduledUpdates(statsEndMillis);
         Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
-        Timestamp statsEndTimestamp = new Timestamp(statsEndTime);
+        Timestamp statsEndTimestamp = new Timestamp(statsEndMillis);
         this.psBs.setString(1, source);
         this.psBs.setTimestamp(2, statsEndTimestamp, cal);
         ResultSet rs = psBs.executeQuery();
@@ -1085,21 +1078,14 @@ public final class RelayDescriptorDatabaseImporter {
    * Adds observations on the number of directory requests by country as
    * seen on a directory at a given date to the database.
    */
-  public void addDirReqStats(String source, String statsEnd, long seconds,
-      Map<String, String> dirReqsPerCountry) {
-    long statsEndTime = 0L;
-    try {
-      statsEndTime = this.dateTimeFormat.parse(statsEnd).getTime();
-    } catch (ParseException e) {
-      this.logger.log(Level.WARNING, "Could not add dirreq stats with "
-          + "interval ending '" + statsEnd + "'.", e);
-      return;
-    }
+  public void addDirReqStats(String source, long statsEndMillis,
+      long seconds, Map<String, String> dirReqsPerCountry) {
+    String statsEnd = this.dateTimeFormat.format(statsEndMillis);
     if (this.importIntoDatabase) {
       try {
-        this.addDateToScheduledUpdates(statsEndTime);
+        this.addDateToScheduledUpdates(statsEndMillis);
         Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
-        Timestamp statsEndTimestamp = new Timestamp(statsEndTime);
+        Timestamp statsEndTimestamp = new Timestamp(statsEndMillis);
         this.psQs.setString(1, source);
         this.psQs.setTimestamp(2, statsEndTimestamp, cal);
         ResultSet rs = psQs.executeQuery();



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits