[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[or-cvs] [metrics-db/master] Import conn-bi-direct stats into the database.
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date: Thu, 16 Dec 2010 16:31:27 +0100
Subject: Import conn-bi-direct stats into the database.
Commit: f7e74963fd95c457bfe3a9438db47afe1fd07ba6
---
db/tordir.sql | 13 +++
.../ernie/db/RelayDescriptorDatabaseImporter.java | 89 ++++++++++++++++++++
.../torproject/ernie/db/RelayDescriptorParser.java | 26 ++++++
3 files changed, 128 insertions(+), 0 deletions(-)
diff --git a/db/tordir.sql b/db/tordir.sql
index 7818f68..40afb32 100644
--- a/db/tordir.sql
+++ b/db/tordir.sql
@@ -101,6 +101,19 @@ CREATE TABLE vote (
CONSTRAINT vote_pkey PRIMARY KEY (validafter, dirsource)
);
+-- TABLE connbidirect
+-- Contain conn-bi-direct stats strings
+CREATE TABLE connbidirect (
+ source CHARACTER(40) NOT NULL,
+ statsend TIMESTAMP WITHOUT TIME ZONE NOT NULL,
+ seconds INTEGER NOT NULL,
+ belownum BIGINT NOT NULL,
+ readnum BIGINT NOT NULL,
+ writenum BIGINT NOT NULL,
+ bothnum BIGINT NOT NULL,
+ CONSTRAINT connbidirect_pkey PRIMARY KEY (source, statsend)
+);
+
-- Create the various indexes we need for searching relays
CREATE INDEX statusentry_validafter_address
ON statusentry (validafter, address);
diff --git a/src/org/torproject/ernie/db/RelayDescriptorDatabaseImporter.java b/src/org/torproject/ernie/db/RelayDescriptorDatabaseImporter.java
index 3db51bc..2d25ca1 100644
--- a/src/org/torproject/ernie/db/RelayDescriptorDatabaseImporter.java
+++ b/src/org/torproject/ernie/db/RelayDescriptorDatabaseImporter.java
@@ -29,6 +29,7 @@ public final class RelayDescriptorDatabaseImporter {
private int rrsCount = 0;
private int rcsCount = 0;
private int rvsCount = 0;
+ private int rbsCount = 0;
/**
* Relay descriptor database connection.
@@ -79,6 +80,12 @@ public final class RelayDescriptorDatabaseImporter {
private PreparedStatement psVs;
/**
+ * Prepared statement to check whether a given conn-bi-direct stats
+ * string has been imported into the database before.
+ */
+ private PreparedStatement psBs;
+
+ /**
* Prepared statement to insert a network status consensus entry into
* the database.
*/
@@ -114,6 +121,12 @@ public final class RelayDescriptorDatabaseImporter {
private PreparedStatement psV;
/**
+ * Prepared statement to insert a conn-bi-direct stats string into the
+ * database.
+ */
+ private PreparedStatement psB;
+
+ /**
* Logger for this class.
*/
private Logger logger;
@@ -154,6 +167,11 @@ public final class RelayDescriptorDatabaseImporter {
private BufferedWriter voteOut;
/**
+ * Raw import file containing conn-bi-direct stats strings.
+ */
+ private BufferedWriter connBiDirectOut;
+
+ /**
* Date format to parse timestamps.
*/
private SimpleDateFormat dateTimeFormat;
@@ -210,6 +228,8 @@ public final class RelayDescriptorDatabaseImporter {
+ "FROM consensus WHERE validafter = ?");
this.psVs = conn.prepareStatement("SELECT COUNT(*) "
+ "FROM vote WHERE validafter = ? AND dirsource = ?");
+ this.psBs = conn.prepareStatement("SELECT COUNT(*) "
+ + "FROM connbidirect WHERE source = ? AND statsend = ?");
this.psR = conn.prepareStatement("INSERT INTO statusentry "
+ "(validafter, nickname, fingerprint, descriptor, "
+ "published, address, orport, dirport, isauthority, "
@@ -234,6 +254,9 @@ public final class RelayDescriptorDatabaseImporter {
+ "(validafter, rawdesc) VALUES (?, ?)");
this.psV = conn.prepareStatement("INSERT INTO vote "
+ "(validafter, dirsource, rawdesc) VALUES (?, ?, ?)");
+ this.psB = conn.prepareStatement("INSERT INTO connbidirect "
+ + "(source, statsend, seconds, belownum, readnum, writenum, "
+ + "bothnum) VALUES (?, ?, ?, ?, ?, ?, ?)");
} catch (SQLException e) {
this.logger.log(Level.WARNING, "Could not connect to database or "
+ "prepare statements.", e);
@@ -698,6 +721,68 @@ public final class RelayDescriptorDatabaseImporter {
}
/**
+ * Insert a conn-bi-direct stats string into the database.
+ */
+ public void addConnBiDirect(String source, String statsEnd,
+ long seconds, long below, long read, long write, long both) {
+ long statsEndTime = 0L;
+ try {
+ statsEndTime = this.dateTimeFormat.parse(statsEnd).getTime();
+ } catch (ParseException e) {
+ this.logger.log(Level.WARNING, "Could not add conn-bi-direct "
+ + "stats string with interval ending '" + statsEnd + "'.", e);
+ return;
+ }
+ if (this.psBs != null && this.psB != null) {
+ try {
+ Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
+ Timestamp statsEndTimestamp = new Timestamp(statsEndTime);
+ this.psBs.setString(1, source);
+ this.psBs.setTimestamp(2, statsEndTimestamp, cal);
+ ResultSet rs = psBs.executeQuery();
+ rs.next();
+ if (rs.getInt(1) == 0) {
+ this.psB.clearParameters();
+ this.psB.setString(1, source);
+ this.psB.setTimestamp(2, statsEndTimestamp, cal);
+ this.psB.setLong(3, seconds);
+ this.psB.setLong(4, below);
+ this.psB.setLong(5, read);
+ this.psB.setLong(6, write);
+ this.psB.setLong(7, both);
+ this.psB.executeUpdate();
+ rbsCount++;
+ if (rbsCount % autoCommitCount == 0) {
+ this.conn.commit();
+ rbsCount = 0;
+ }
+ }
+ } catch (SQLException e) {
+ this.logger.log(Level.WARNING, "Could not add conn-bi-direct "
+ + "stats string.", e);
+ }
+ }
+ if (this.rawFilesDirectory != null) {
+ try {
+ if (this.connBiDirectOut == null) {
+ new File(rawFilesDirectory).mkdirs();
+ this.connBiDirectOut = new BufferedWriter(new FileWriter(
+ rawFilesDirectory + "/connbidirect.sql"));
+ this.connBiDirectOut.write(" COPY connbidirect (source, "
+ + "statsend, seconds, belownum, readnum, writenum, "
+ + "bothnum) FROM stdin;\n");
+ }
+ this.connBiDirectOut.write(source + "\t" + statsEnd + "\t"
+ + seconds + "\t" + below + "\t" + read + "\t" + write + "\t"
+ + both + "\n");
+ } catch (IOException e) {
+ this.logger.log(Level.WARNING, "Could not write conn-bi-direct "
+ + "stats string to raw database import file.", e);
+ }
+ }
+ }
+
+ /**
* Close the relay descriptor database connection.
*/
public void closeConnection() {
@@ -744,6 +829,10 @@ public final class RelayDescriptorDatabaseImporter {
this.voteOut.write("\\.\n");
this.voteOut.close();
}
+ if (this.connBiDirectOut != null) {
+ this.connBiDirectOut.write("\\.\n");
+ this.connBiDirectOut.close();
+ }
} catch (IOException e) {
this.logger.log(Level.WARNING, "Could not close one or more raw "
+ "database import files.", e);
diff --git a/src/org/torproject/ernie/db/RelayDescriptorParser.java b/src/org/torproject/ernie/db/RelayDescriptorParser.java
index 27226f3..a1f6cd3 100644
--- a/src/org/torproject/ernie/db/RelayDescriptorParser.java
+++ b/src/org/torproject/ernie/db/RelayDescriptorParser.java
@@ -401,6 +401,32 @@ public class RelayDescriptorParser {
break;
}
}
+ } else if (line.startsWith("conn-bi-direct ")) {
+ if (this.rddi != null) {
+ String[] parts = line.split(" ");
+ if (parts.length == 6 &&
+ parts[5].split(",").length == 4) {
+ try {
+ String connBiDirectStatsEnd = parts[1] + " " + parts[2];
+ long connBiDirectSeconds = Long.parseLong(parts[3].
+ substring(1));
+ String[] parts2 = parts[5].split(",");
+ long below = Long.parseLong(parts2[0]);
+ long read = Long.parseLong(parts2[1]);
+ long write = Long.parseLong(parts2[2]);
+ long both = Long.parseLong(parts2[3]);
+ this.rddi.addConnBiDirect(dir, connBiDirectStatsEnd,
+ connBiDirectSeconds, below, read, write, both);
+ } catch (NumberFormatException e) {
+ this.logger.log(Level.WARNING, "Number format "
+ + "exception while parsing conn-bi-direct stats "
+ + "string '" + line + "'. Skipping.", e);
+ }
+ } else {
+ this.logger.warning("Skipping invalid conn-bi-direct "
+ + "stats string '" + line + "'.");
+ }
+ }
}
}
String ascii = new String(data, "US-ASCII");
--
1.7.1