[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[or-cvs] [metrics-db/master] Import conn-bi-direct stats into the database.



Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Thu, 16 Dec 2010 16:31:27 +0100
Subject: Import conn-bi-direct stats into the database.
Commit: f7e74963fd95c457bfe3a9438db47afe1fd07ba6

---
 db/tordir.sql                                      |   13 +++
 .../ernie/db/RelayDescriptorDatabaseImporter.java  |   89 ++++++++++++++++++++
 .../torproject/ernie/db/RelayDescriptorParser.java |   26 ++++++
 3 files changed, 128 insertions(+), 0 deletions(-)

diff --git a/db/tordir.sql b/db/tordir.sql
index 7818f68..40afb32 100644
--- a/db/tordir.sql
+++ b/db/tordir.sql
@@ -101,6 +101,19 @@ CREATE TABLE vote (
     CONSTRAINT vote_pkey PRIMARY KEY (validafter, dirsource)
 );
 
+-- TABLE connbidirect
+-- Contain conn-bi-direct stats strings
+CREATE TABLE connbidirect (
+    source CHARACTER(40) NOT NULL,
+    statsend TIMESTAMP WITHOUT TIME ZONE NOT NULL,
+    seconds INTEGER NOT NULL,
+    belownum BIGINT NOT NULL,
+    readnum BIGINT NOT NULL,
+    writenum BIGINT NOT NULL,
+    bothnum BIGINT NOT NULL,
+    CONSTRAINT connbidirect_pkey PRIMARY KEY (source, statsend)
+);
+
 -- Create the various indexes we need for searching relays
 CREATE INDEX statusentry_validafter_address
   ON statusentry (validafter, address);
diff --git a/src/org/torproject/ernie/db/RelayDescriptorDatabaseImporter.java b/src/org/torproject/ernie/db/RelayDescriptorDatabaseImporter.java
index 3db51bc..2d25ca1 100644
--- a/src/org/torproject/ernie/db/RelayDescriptorDatabaseImporter.java
+++ b/src/org/torproject/ernie/db/RelayDescriptorDatabaseImporter.java
@@ -29,6 +29,7 @@ public final class RelayDescriptorDatabaseImporter {
   private int rrsCount = 0;
   private int rcsCount = 0;
   private int rvsCount = 0;
+  private int rbsCount = 0;
 
   /**
    * Relay descriptor database connection.
@@ -79,6 +80,12 @@ public final class RelayDescriptorDatabaseImporter {
   private PreparedStatement psVs;
 
   /**
+   * Prepared statement to check whether a given conn-bi-direct stats
+   * string has been imported into the database before.
+   */
+  private PreparedStatement psBs;
+
+  /**
    * Prepared statement to insert a network status consensus entry into
    * the database.
    */
@@ -114,6 +121,12 @@ public final class RelayDescriptorDatabaseImporter {
   private PreparedStatement psV;
 
   /**
+   * Prepared statement to insert a conn-bi-direct stats string into the
+   * database.
+   */
+  private PreparedStatement psB;
+
+  /**
    * Logger for this class.
    */
   private Logger logger;
@@ -154,6 +167,11 @@ public final class RelayDescriptorDatabaseImporter {
   private BufferedWriter voteOut;
 
   /**
+   * Raw import file containing conn-bi-direct stats strings.
+   */
+  private BufferedWriter connBiDirectOut;
+
+  /**
    * Date format to parse timestamps.
    */
   private SimpleDateFormat dateTimeFormat;
@@ -210,6 +228,8 @@ public final class RelayDescriptorDatabaseImporter {
             + "FROM consensus WHERE validafter = ?");
         this.psVs = conn.prepareStatement("SELECT COUNT(*) "
             + "FROM vote WHERE validafter = ? AND dirsource = ?");
+        this.psBs = conn.prepareStatement("SELECT COUNT(*) "
+            + "FROM connbidirect WHERE source = ? AND statsend = ?");
         this.psR = conn.prepareStatement("INSERT INTO statusentry "
             + "(validafter, nickname, fingerprint, descriptor, "
             + "published, address, orport, dirport, isauthority, "
@@ -234,6 +254,9 @@ public final class RelayDescriptorDatabaseImporter {
             + "(validafter, rawdesc) VALUES (?, ?)");
         this.psV = conn.prepareStatement("INSERT INTO vote "
             + "(validafter, dirsource, rawdesc) VALUES (?, ?, ?)");
+        this.psB = conn.prepareStatement("INSERT INTO connbidirect "
+            + "(source, statsend, seconds, belownum, readnum, writenum, "
+            + "bothnum) VALUES (?, ?, ?, ?, ?, ?, ?)");
       } catch (SQLException e) {
         this.logger.log(Level.WARNING, "Could not connect to database or "
             + "prepare statements.", e);
@@ -698,6 +721,68 @@ public final class RelayDescriptorDatabaseImporter {
   }
 
   /**
+   * Insert a conn-bi-direct stats string into the database.
+   */
+  public void addConnBiDirect(String source, String statsEnd,
+      long seconds, long below, long read, long write, long both) {
+    long statsEndTime = 0L;
+    try {
+      statsEndTime = this.dateTimeFormat.parse(statsEnd).getTime();
+    } catch (ParseException e) {
+      this.logger.log(Level.WARNING, "Could not add conn-bi-direct "
+          + "stats string with interval ending '" + statsEnd + "'.", e);
+      return;
+    }
+    if (this.psBs != null && this.psB != null) {
+      try {
+        Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
+        Timestamp statsEndTimestamp = new Timestamp(statsEndTime);
+        this.psBs.setString(1, source);
+        this.psBs.setTimestamp(2, statsEndTimestamp, cal);
+        ResultSet rs = psBs.executeQuery();
+        rs.next();
+        if (rs.getInt(1) == 0) {
+          this.psB.clearParameters();
+          this.psB.setString(1, source);
+          this.psB.setTimestamp(2, statsEndTimestamp, cal);
+          this.psB.setLong(3, seconds);
+          this.psB.setLong(4, below);
+          this.psB.setLong(5, read);
+          this.psB.setLong(6, write);
+          this.psB.setLong(7, both);
+          this.psB.executeUpdate();
+          rbsCount++;
+          if (rbsCount % autoCommitCount == 0)  {
+            this.conn.commit();
+            rbsCount = 0;
+          }
+        }
+      } catch (SQLException e) {
+        this.logger.log(Level.WARNING, "Could not add conn-bi-direct "
+            + "stats string.", e);
+      }
+    }
+    if (this.rawFilesDirectory != null) {
+      try {
+        if (this.connBiDirectOut == null) {
+          new File(rawFilesDirectory).mkdirs();
+          this.connBiDirectOut = new BufferedWriter(new FileWriter(
+              rawFilesDirectory + "/connbidirect.sql"));
+          this.connBiDirectOut.write(" COPY connbidirect (source, "
+              + "statsend, seconds, belownum, readnum, writenum, "
+              + "bothnum) FROM stdin;\n");
+        }
+        this.connBiDirectOut.write(source + "\t" + statsEnd + "\t"
+            + seconds + "\t" + below + "\t" + read + "\t" + write + "\t"
+            + both + "\n");
+      } catch (IOException e) {
+        this.logger.log(Level.WARNING, "Could not write conn-bi-direct "
+            + "stats string to raw database import file.", e);
+      }
+    }
+  }
+
+  /**
    * Close the relay descriptor database connection.
    */
   public void closeConnection() {
@@ -744,6 +829,10 @@ public final class RelayDescriptorDatabaseImporter {
         this.voteOut.write("\\.\n");
         this.voteOut.close();
       }
+      if (this.connBiDirectOut != null) {
+        this.connBiDirectOut.write("\\.\n");
+        this.connBiDirectOut.close();
+      }
     } catch (IOException e) {
       this.logger.log(Level.WARNING, "Could not close one or more raw "
           + "database import files.", e);
diff --git a/src/org/torproject/ernie/db/RelayDescriptorParser.java b/src/org/torproject/ernie/db/RelayDescriptorParser.java
index 27226f3..a1f6cd3 100644
--- a/src/org/torproject/ernie/db/RelayDescriptorParser.java
+++ b/src/org/torproject/ernie/db/RelayDescriptorParser.java
@@ -401,6 +401,32 @@ public class RelayDescriptorParser {
                 break;
               }
             }
+          } else if (line.startsWith("conn-bi-direct ")) {
+            if (this.rddi != null) {
+              String[] parts = line.split(" ");
+              if (parts.length == 6 &&
+                  parts[5].split(",").length == 4) {
+                try {
+                  String connBiDirectStatsEnd = parts[1] + " " + parts[2];
+                  long connBiDirectSeconds = Long.parseLong(parts[3].
+                      substring(1));
+                  String[] parts2 = parts[5].split(",");
+                  long below = Long.parseLong(parts2[0]);
+                  long read = Long.parseLong(parts2[1]);
+                  long write = Long.parseLong(parts2[2]);
+                  long both = Long.parseLong(parts2[3]);
+                  this.rddi.addConnBiDirect(dir, connBiDirectStatsEnd,
+                      connBiDirectSeconds, below, read, write, both);
+                } catch (NumberFormatException e) {
+                  this.logger.log(Level.WARNING, "Number format "
+                      + "exception while parsing conn-bi-direct stats "
+                      + "string '" + line + "'. Skipping.", e);
+                }
+              } else {
+                this.logger.warning("Skipping invalid conn-bi-direct "
+                    + "stats string '" + line + "'.");
+              }
+            }
           }
         }
         String ascii = new String(data, "US-ASCII");
-- 
1.7.1