[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [metrics-tasks/master] Export test cases to a file (#6471).



commit 04a718c38e44d0d47223fcffeec8e31a7308d3f8
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Thu Oct 25 12:35:29 2012 -0400

    Export test cases to a file (#6471).
---
 .../torproject/task6471/DatabaseImporterImpl.java  |    5 +
 .../task6471/DatabasePerformanceExample.java       |  190 ++++++++++++--------
 2 files changed, 116 insertions(+), 79 deletions(-)

diff --git a/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java b/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java
index 384b2d0..b562c29 100644
--- a/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java
+++ b/task-6471/java/src/org/torproject/task6471/DatabaseImporterImpl.java
@@ -123,6 +123,11 @@ public class DatabaseImporterImpl extends DatabaseImpl
   void addRange(String databaseFileName, String countryCode,
       String startAddressString, long addresses) {
 
+    if (countryCode.length() != 2) {
+      /* Don't import illegal range. */
+      return;
+    }
+
     this.rangeImports++;
     String databaseDateString =
         databaseFileName.substring(databaseFileName.length() - 8);
diff --git a/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java b/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java
index 4338fad..77808c8 100644
--- a/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java
+++ b/task-6471/java/src/org/torproject/task6471/DatabasePerformanceExample.java
@@ -1,104 +1,132 @@
 package org.torproject.task6471;
 
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
 import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Random;
-import java.util.SortedMap;
 import java.util.SortedSet;
 import java.util.Stack;
-import java.util.TreeMap;
 import java.util.TreeSet;
 
 public class DatabasePerformanceExample {
-  public static void main(String[] args) {
+  public static void main(String[] args) throws IOException {
 
-    System.out.print("Generating test cases... ");
-    long startMillis = System.currentTimeMillis();
-    List<Long> tests = new ArrayList<Long>();
-    SortedMap<Long, String> results = new TreeMap<Long, String>();
-    Random rnd = new Random(1L);
-    int startDate = DatabaseImpl.convertDateStringToNumber("20071001");
-    int endDate = DatabaseImpl.convertDateStringToNumber("20120930");
-    /* Skipping Dec 1--3, 2009, because the first available database from
-     * December 2009 was published on the 4th, and generating test cases
-     * was just too confusing when taking that into account. */
-    List<Integer> skipDates = new ArrayList<Integer>();
-    skipDates.add(DatabaseImpl.convertDateStringToNumber("20091201"));
-    skipDates.add(DatabaseImpl.convertDateStringToNumber("20091202"));
-    skipDates.add(DatabaseImpl.convertDateStringToNumber("20091203"));
-    for (int i = 0; i < 100000; i++) {
-      long testAddress = rnd.nextLong() & ((1L << 32) - 1L);
-      int testDate = startDate + rnd.nextInt(endDate - startDate);
-      if (skipDates.contains(testDate)) {
-        i--;
-      } else {
-        tests.add((testAddress << 16) + testDate);
+    File testCasesCsvFile = new File("test-cases.csv");
+    if (!testCasesCsvFile.exists()) {
+      System.out.print("Generating test cases... ");
+      long startMillis = System.currentTimeMillis();
+      List<Long> tests = new ArrayList<Long>();
+      Random rnd = new Random(1L);
+      int startDate = DatabaseImpl.convertDateStringToNumber("20071001");
+      int endDate = DatabaseImpl.convertDateStringToNumber("20120930");
+      /* Skipping Dec 1--3, 2009, because the first available database
+       * from December 2009 was published on the 4th, and generating test
+       * cases was just too confusing when taking that into account. */
+      List<Integer> skipDates = new ArrayList<Integer>();
+      skipDates.add(DatabaseImpl.convertDateStringToNumber("20091201"));
+      skipDates.add(DatabaseImpl.convertDateStringToNumber("20091202"));
+      skipDates.add(DatabaseImpl.convertDateStringToNumber("20091203"));
+      for (int i = 0; i < 100000; i++) {
+        long testAddress = rnd.nextLong() & ((1L << 32) - 1L);
+        int testDate = startDate + rnd.nextInt(endDate - startDate);
+        if (skipDates.contains(testDate)) {
+          i--;
+        } else {
+          tests.add((testAddress << 16) + testDate);
+        }
       }
-    }
-    Stack<File> stackedFiles = new Stack<File>();
-    stackedFiles.add(new File("../data"));
-    SortedSet<File> files = new TreeSet<File>();
-    while (!stackedFiles.isEmpty()) {
-      File file = stackedFiles.pop();
-      if (file.isDirectory()) {
-        stackedFiles.addAll(Arrays.asList(file.listFiles()));
-      } else if (!file.getName().endsWith(".md5") &&
-          !file.getName().endsWith(".md5.gz") &&
-          !file.getName().endsWith(".asc") &&
-          !file.getName().endsWith(".asc.gz")) {
-        files.add(file);
+      Stack<File> stackedFiles = new Stack<File>();
+      stackedFiles.add(new File("../data"));
+      SortedSet<File> files = new TreeSet<File>();
+      while (!stackedFiles.isEmpty()) {
+        File file = stackedFiles.pop();
+        if (file.isDirectory()) {
+          stackedFiles.addAll(Arrays.asList(file.listFiles()));
+        } else if (!file.getName().endsWith(".md5") &&
+            !file.getName().endsWith(".md5.gz") &&
+            !file.getName().endsWith(".asc") &&
+            !file.getName().endsWith(".asc.gz")) {
+          files.add(file);
+        }
       }
-    }
-    for (File file : files) {
-      String dbMonth = file.getName().substring(
-          file.getName().length() - 8);
-      dbMonth = dbMonth.substring(0, 6);
-      DatabaseImporter temp = new DatabaseImporterImpl();
-      temp.importRegionalRegistryStatsFileOrDirectory(
-          file.getAbsolutePath());
+      Map<Long, String> results = new HashMap<Long, String>();
       for (long test : tests) {
-        int testDate = (int) (test & ((1 << 16) - 1));
-        String testMonth = DatabaseImpl.convertDateNumberToString(
-            testDate).substring(0, 6);
-        if (testMonth.equals(dbMonth)) {
-          String testAddressString = DatabaseImpl.
-              convertAddressNumberToString(test >> 16);
-          String testDateString = DatabaseImpl.convertDateNumberToString(
-              testDate);
-          String countryCode =
-              temp.lookupCountryCodeFromIpv4AddressAndDate(
-              testAddressString, testDateString);
-          if (countryCode != null) {
-            results.put(test, countryCode);
+        results.put(test, "??");
+      }
+      for (File file : files) {
+        String dbMonth = file.getName().substring(
+            file.getName().length() - 8);
+        dbMonth = dbMonth.substring(0, 6);
+        DatabaseImporter temp = new DatabaseImporterImpl();
+        temp.importRegionalRegistryStatsFileOrDirectory(
+            file.getAbsolutePath());
+        for (long test : tests) {
+          String testDateString = DatabaseImpl.
+              convertKeyToDateString(test);
+          String testMonth = testDateString.substring(0, 6);
+          if (testMonth.equals(dbMonth)) {
+            String testAddressString = DatabaseImpl.
+                convertKeyToAddressString(test);
+            String countryCode =
+                temp.lookupCountryCodeFromIpv4AddressAndDate(
+                testAddressString, testDateString);
+            if (countryCode != null) {
+              results.put(test, countryCode);
+            }
           }
         }
       }
+      BufferedWriter bw = new BufferedWriter(new FileWriter(
+          testCasesCsvFile));
+      for (Map.Entry<Long, String> e : results.entrySet()) {
+        String testAddressString = DatabaseImpl.
+            convertKeyToAddressString(e.getKey());
+        String testDateString = DatabaseImpl.
+            convertKeyToDateString(e.getKey());
+        String countryCode = e.getValue();
+        bw.write(testAddressString + "," + testDateString + ","
+            + countryCode + "\n");
+      }
+      bw.close();
+      long endMillis = System.currentTimeMillis();
+      System.out.println((endMillis - startMillis) + " millis.");
     }
-    long endMillis = System.currentTimeMillis();
-    System.out.println((endMillis - startMillis) + " millis.");
 
     System.out.print("Importing files... ");
-    startMillis = endMillis;
+    long startMillis = System.currentTimeMillis();
     DatabaseImporter combinedDatabase = new DatabaseImporterImpl();
     combinedDatabase.importRegionalRegistryStatsFileOrDirectory(
         "../data");
-    endMillis = System.currentTimeMillis();
+    long endMillis = System.currentTimeMillis();
     System.out.println((endMillis - startMillis) + " millis.");
 
     System.out.print("Making test requests... ");
     startMillis = endMillis;
-    int failures = 0;
-    for (long test : tests) {
-      String testAddress = DatabaseImpl.convertAddressNumberToString(
-          test >> 16);
-      String testDate = DatabaseImpl.convertDateNumberToString(
-          (int) (test & ((1 << 16) - 1)));
-      String expected = results.get(test);
+    BufferedReader br = new BufferedReader(new FileReader(
+        testCasesCsvFile));
+    String line;
+    int tests = 0, failures = 0;
+    while ((line = br.readLine()) != null) {
+      String[] parts = line.split(",");
+      String testAddress = parts[0];
+      String testDate = parts[1];
+      if (parts.length != 3) {
+        System.out.println(line);
+        System.exit(1);
+      }
+      String expected = "??".equals(parts[2]) ? null : parts[2];
       String result =
           combinedDatabase.lookupCountryCodeFromIpv4AddressAndDate(
           testAddress, testDate);
+      tests++;
       if ((expected == null && result != null) ||
           (expected != null && !expected.equals(result))) {
         //System.out.println("Expected " + expected + " for "
@@ -106,9 +134,10 @@ public class DatabasePerformanceExample {
         failures++;
       }
     }
+    br.close();
     endMillis = System.currentTimeMillis();
     System.out.println((endMillis - startMillis) + " millis, " + failures
-        + " out of " + tests.size() + " tests failed.");
+        + " out of " + tests + " tests failed.");
 
     System.out.println(combinedDatabase);
 
@@ -128,15 +157,17 @@ public class DatabasePerformanceExample {
 
     System.out.print("Making a second round of test requests... ");
     startMillis = endMillis;
-    failures = 0;
-    for (long test : tests) {
-      String testAddress = DatabaseImpl.convertAddressNumberToString(
-          test >> 16);
-      String testDate = DatabaseImpl.convertDateNumberToString(
-          (int) (test & ((1 << 16) - 1)));
-      String expected = results.get(test);
-      String result = database.lookupCountryCodeFromIpv4AddressAndDate(
+    br = new BufferedReader(new FileReader(testCasesCsvFile));
+    tests = failures = 0;
+    while ((line = br.readLine()) != null) {
+      String[] parts = line.split(",");
+      String testAddress = parts[0];
+      String testDate = parts[1];
+      String expected = parts[2].equals("??") ? null : parts[2];
+      String result =
+          combinedDatabase.lookupCountryCodeFromIpv4AddressAndDate(
           testAddress, testDate);
+      tests++;
       if ((expected == null && result != null) ||
           (expected != null && !expected.equals(result))) {
         //System.out.println("Expected " + expected + " for "
@@ -144,8 +175,9 @@ public class DatabasePerformanceExample {
         failures++;
       }
     }
+    br.close();
     endMillis = System.currentTimeMillis();
     System.out.println((endMillis - startMillis) + " millis, " + failures
-        + " out of " + tests.size() + " tests failed.");
+        + " out of " + tests + " tests failed.");
   }
 }



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits