[tor-commits] [metrics-tasks/master] Skip previously imported files (#20008).

[karsten@xxxxxxxxxxxxxx]

commit 94ac93fe409d940531095e628453efec26d236ca
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Mon Sep 19 09:22:44 2016 +0200

    Skip previously imported files (#20008).
---
 task-20008/src/Importer.java | 40 +++++++++++++++++++++++++---------------
 1 file changed, 25 insertions(+), 15 deletions(-)

diff --git a/task-20008/src/Importer.java b/task-20008/src/Importer.java
index 6834bf0..3325e4c 100644
--- a/task-20008/src/Importer.java
+++ b/task-20008/src/Importer.java
@@ -103,6 +103,12 @@ public class Importer {
     long logDateMillis = this.dateFormat.parse(fileNameMatcher.group(2))
         .getTime();
     int fileId = writeFile(server, site, logDateMillis);
+    if (fileId < 0) {
+      System.out.println("Skipping previously imported file: "
+          + file.getAbsolutePath());
+      this.connection.rollback();
+      return;
+    }
     BufferedReader br = new BufferedReader(new InputStreamReader(
         new XZCompressorInputStream(new FileInputStream(file))));
     String line;
@@ -144,30 +150,34 @@ public class Importer {
     }
     br.close();
     this.connection.commit();
+    System.out.println("Successfully imported file: "
+        + file.getAbsolutePath());
   }
 
-  int writeFile(String server, String site, long logDateMillis)
-      throws Exception {
+  int writeFile(String server, String site, long logDateMillis) {
     int fileId = -1;
-    this.psFiles.clearParameters();
-    server = this.truncateString(server, 32);
-    this.psFiles.setString(1, server);
-    site = this.truncateString(site, 128);
-    this.psFiles.setString(2, site);
-    this.psFiles.setDate(3, new Date(logDateMillis));
-    this.psFiles.execute();
-    ResultSet resultSet = psFiles.getGeneratedKeys();
-    if (resultSet.next()) {
-      fileId = resultSet.getInt(1);
+    try {
+      this.psFiles.clearParameters();
+      server = this.truncateString(server, 32);
+      this.psFiles.setString(1, server);
+      site = this.truncateString(site, 128);
+      this.psFiles.setString(2, site);
+      this.psFiles.setDate(3, new Date(logDateMillis));
+      this.psFiles.execute();
+      ResultSet resultSet = psFiles.getGeneratedKeys();
+      if (resultSet.next()) {
+        fileId = resultSet.getInt(1);
+      }
+      resultSet.close();
+    } catch (SQLException e) {
+      System.err.println("Could not insert row into files table: "
+          + e.getMessage());
     }
-    resultSet.close();
     return fileId;
   }
 
   void writeRequest(int fileId, String method, String resource,
       int responseCode, int count, long totalBytesSent) throws Exception {
-    System.out.printf("Writing request to database: %d, %s, %s, %d, %d, %d%n",
-        fileId, method, resource, responseCode, count, totalBytesSent);
     int resourceId = -1;
     this.psResourcesSelect.clearParameters();
     this.psResourcesSelect.setString(1, resource);

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits