[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [metrics-web/master] Stop importing raw consensuses and server descriptors.



commit 1f2e10e5c1d0636335b521a33ef876778c13ded7
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Thu Dec 13 16:11:14 2012 +0100

    Stop importing raw consensuses and server descriptors.
---
 db/tordir.sql                                      |    2 -
 .../cron/RelayDescriptorDatabaseImporter.java      |   48 +++++--------------
 2 files changed, 13 insertions(+), 37 deletions(-)

diff --git a/db/tordir.sql b/db/tordir.sql
index 5f533c4..3749774 100644
--- a/db/tordir.sql
+++ b/db/tordir.sql
@@ -19,7 +19,6 @@ CREATE TABLE descriptor (
     published TIMESTAMP WITHOUT TIME ZONE NOT NULL,
     uptime BIGINT,
     extrainfo CHARACTER(40),
-    rawdesc BYTEA NOT NULL,
     CONSTRAINT descriptor_pkey PRIMARY KEY (descriptor)
 );
 
@@ -131,7 +130,6 @@ CREATE TRIGGER insert_statusentry_trigger
 -- Contains all of the consensuses published by the directories.
 CREATE TABLE consensus (
     validafter TIMESTAMP WITHOUT TIME ZONE NOT NULL,
-    rawdesc BYTEA NOT NULL,
     CONSTRAINT consensus_pkey PRIMARY KEY (validafter)
 );
 
diff --git a/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java b/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java
index 1bbd8d9..5d2724a 100644
--- a/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java
+++ b/src/org/torproject/ernie/cron/RelayDescriptorDatabaseImporter.java
@@ -256,12 +256,12 @@ public final class RelayDescriptorDatabaseImporter {
             + "(descriptor, nickname, address, orport, dirport, "
             + "fingerprint, bandwidthavg, bandwidthburst, "
             + "bandwidthobserved, platform, published, uptime, "
-            + "extrainfo, rawdesc) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, "
-            + "?, ?, ?, ?)");
+            + "extrainfo) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, "
+            + "?)");
         this.csH = conn.prepareCall("{call insert_bwhist(?, ?, ?, ?, ?, "
             + "?)}");
         this.psC = conn.prepareStatement("INSERT INTO consensus "
-            + "(validafter, rawdesc) VALUES (?, ?)");
+            + "(validafter) VALUES (?)");
         this.psQ = conn.prepareStatement("INSERT INTO dirreq_stats "
             + "(source, statsend, seconds, country, requests) VALUES "
             + "(?, ?, ?, ?, ?)");
@@ -454,7 +454,7 @@ public final class RelayDescriptorDatabaseImporter {
       String address, int orPort, int dirPort, String relayIdentifier,
       long bandwidthAvg, long bandwidthBurst, long bandwidthObserved,
       String platform, long published, long uptime,
-      String extraInfoDigest, byte[] rawDescriptor) {
+      String extraInfoDigest) {
     if (this.importIntoDatabase) {
       try {
         this.addDateToScheduledUpdates(published);
@@ -484,7 +484,6 @@ public final class RelayDescriptorDatabaseImporter {
           this.psD.setTimestamp(11, new Timestamp(published), cal);
           this.psD.setLong(12, uptime);
           this.psD.setString(13, extraInfoDigest);
-          this.psD.setBytes(14, rawDescriptor);
           this.psD.executeUpdate();
           rdsCount++;
           if (rdsCount % autoCommitCount == 0)  {
@@ -509,8 +508,7 @@ public final class RelayDescriptorDatabaseImporter {
           this.descriptorOut.write(" COPY descriptor (descriptor, "
               + "nickname, address, orport, dirport, fingerprint, "
               + "bandwidthavg, bandwidthburst, bandwidthobserved, "
-              + "platform, published, uptime, extrainfo, rawdesc) FROM "
-              + "stdin;\n");
+              + "platform, published, uptime, extrainfo) FROM stdin;\n");
         }
         this.descriptorOut.write(descriptor.toLowerCase() + "\t"
             + nickname + "\t" + address + "\t" + orPort + "\t" + dirPort
@@ -521,17 +519,9 @@ public final class RelayDescriptorDatabaseImporter {
             + "\t" + this.dateTimeFormat.format(published) + "\t"
             + (uptime >= 0 ? uptime : "\\N") + "\t"
             + (extraInfoDigest != null ? extraInfoDigest : "\\N")
-            + "\t");
-        this.descriptorOut.write(PGbytea.toPGString(rawDescriptor).
-            replaceAll("\\\\", "\\\\\\\\") + "\n");
+            + "\n");
       } catch (UnsupportedEncodingException e) {
         // US-ASCII is supported for sure
-      } catch (SQLException e) {
-        this.logger.log(Level.WARNING, "Could not write server "
-            + "descriptor to raw database import file.  We won't make "
-            + "any further attempts to write raw import files in this "
-            + "execution.", e);
-        this.writeRawImportFiles = false;
       } catch (IOException e) {
         this.logger.log(Level.WARNING, "Could not write server "
             + "descriptor to raw database import file.  We won't make "
@@ -547,7 +537,7 @@ public final class RelayDescriptorDatabaseImporter {
    */
   public void addExtraInfoDescriptor(String extraInfoDigest,
       String nickname, String fingerprint, long published,
-      byte[] rawDescriptor, List<String> bandwidthHistoryLines) {
+      List<String> bandwidthHistoryLines) {
     if (!bandwidthHistoryLines.isEmpty()) {
       this.addBandwidthHistory(fingerprint.toLowerCase(), published,
           bandwidthHistoryLines);
@@ -803,7 +793,7 @@ public final class RelayDescriptorDatabaseImporter {
   /**
    * Insert network status consensus into database.
    */
-  public void addConsensus(long validAfter, byte[] rawDescriptor) {
+  public void addConsensus(long validAfter) {
     if (this.importIntoDatabase) {
       try {
         this.addDateToScheduledUpdates(validAfter);
@@ -815,7 +805,6 @@ public final class RelayDescriptorDatabaseImporter {
         if (rs.getInt(1) == 0) {
           this.psC.clearParameters();
           this.psC.setTimestamp(1, validAfterTimestamp, cal);
-          this.psC.setBytes(2, rawDescriptor);
           this.psC.executeUpdate();
           rcsCount++;
           if (rcsCount % autoCommitCount == 0)  {
@@ -835,19 +824,11 @@ public final class RelayDescriptorDatabaseImporter {
           new File(rawFilesDirectory).mkdirs();
           this.consensusOut = new BufferedWriter(new FileWriter(
               rawFilesDirectory + "/consensus.sql"));
-          this.consensusOut.write(" COPY consensus (validafter, rawdesc) "
+          this.consensusOut.write(" COPY consensus (validafter) "
               + "FROM stdin;\n");
         }
         String validAfterString = this.dateTimeFormat.format(validAfter);
-        this.consensusOut.write(validAfterString + "\t");
-        this.consensusOut.write(PGbytea.toPGString(rawDescriptor).
-            replaceAll("\\\\", "\\\\\\\\") + "\n");
-      } catch (SQLException e) {
-        this.logger.log(Level.WARNING, "Could not write network status "
-            + "consensus to raw database import file.  We won't make "
-            + "any further attempts to write raw import files in this "
-            + "execution.", e);
-        this.writeRawImportFiles = false;
+        this.consensusOut.write(validAfterString + "\n");
       } catch (IOException e) {
         this.logger.log(Level.WARNING, "Could not write network status "
             + "consensus to raw database import file.  We won't make "
@@ -967,8 +948,7 @@ public final class RelayDescriptorDatabaseImporter {
           statusEntry.getBandwidth(), statusEntry.getPortList(),
           statusEntry.getStatusEntryBytes());
     }
-    this.addConsensus(consensus.getValidAfterMillis(),
-        consensus.getRawDescriptorBytes());
+    this.addConsensus(consensus.getValidAfterMillis());
   }
 
   private void addServerDescriptor(ServerDescriptor descriptor) {
@@ -978,8 +958,7 @@ public final class RelayDescriptorDatabaseImporter {
         descriptor.getFingerprint(), descriptor.getBandwidthRate(),
         descriptor.getBandwidthBurst(), descriptor.getBandwidthObserved(),
         descriptor.getPlatform(), descriptor.getPublishedMillis(),
-        descriptor.getUptime(), descriptor.getExtraInfoDigest(),
-        descriptor.getRawDescriptorBytes());
+        descriptor.getUptime(), descriptor.getExtraInfoDigest());
   }
 
   private void addExtraInfoDescriptor(ExtraInfoDescriptor descriptor) {
@@ -1016,8 +995,7 @@ public final class RelayDescriptorDatabaseImporter {
     this.addExtraInfoDescriptor(descriptor.getExtraInfoDigest(),
         descriptor.getNickname(),
         descriptor.getFingerprint().toLowerCase(),
-        descriptor.getPublishedMillis(),
-        descriptor.getRawDescriptorBytes(), bandwidthHistoryLines);
+        descriptor.getPublishedMillis(), bandwidthHistoryLines);
   }
 
   /**

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits