[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [metrics-web/master] Break down totalcw numbers by Guard/(Bad)Exit flags.



commit 7b39042ecfae011d5891af963ae94bd35141fbc0
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Thu Nov 29 10:09:28 2018 +0100

    Break down totalcw numbers by Guard/(Bad)Exit flags.
    
    Requires updating the vote table and the totalcw view in the database.
    
    Implements #28328.
---
 src/main/R/rserver/graphs.R                        |  4 ++-
 .../torproject/metrics/stats/totalcw/Database.java | 36 ++++++++++------------
 .../metrics/stats/totalcw/OutputLine.java          | 12 ++++++--
 .../torproject/metrics/stats/totalcw/Parser.java   | 17 +++++-----
 .../totalcw/TotalcwRelayNetworkStatusVote.java     |  6 ++--
 src/main/sql/totalcw/init-totalcw.sql              | 16 +++++++---
 .../totalcw/TotalcwRelayNetworkStatusVoteTest.java | 18 ++++++-----
 7 files changed, 66 insertions(+), 43 deletions(-)

diff --git a/src/main/R/rserver/graphs.R b/src/main/R/rserver/graphs.R
index df108e2..e3ac598 100644
--- a/src/main/R/rserver/graphs.R
+++ b/src/main/R/rserver/graphs.R
@@ -1562,7 +1562,9 @@ prepare_totalcw <- function(start_p, end_p) {
     filter(if (!is.null(start_p))
         valid_after_date >= as.Date(start_p) else TRUE) %>%
     filter(if (!is.null(end_p))
-        valid_after_date <= as.Date(end_p) else TRUE)
+        valid_after_date <= as.Date(end_p) else TRUE) %>%
+    group_by(valid_after_date, nickname) %>%
+    summarize(measured_sum_avg = sum(measured_sum_avg))
 }
 
 plot_totalcw <- function(start_p, end_p, path_p) {
diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/Database.java b/src/main/java/org/torproject/metrics/stats/totalcw/Database.java
index 66b0366..b6dc87c 100644
--- a/src/main/java/org/torproject/metrics/stats/totalcw/Database.java
+++ b/src/main/java/org/torproject/metrics/stats/totalcw/Database.java
@@ -66,9 +66,8 @@ class Database implements AutoCloseable {
         "SELECT EXISTS (SELECT 1 FROM vote "
             + "WHERE valid_after = ? AND authority_id = ?)");
     this.psVoteInsert = this.connection.prepareStatement(
-        "INSERT INTO vote (valid_after, authority_id, measured_sum) "
-            + "VALUES (?, ?, ?)",
-        Statement.RETURN_GENERATED_KEYS);
+        "INSERT INTO vote (valid_after, authority_id, have_guard_flag, "
+        + "have_exit_flag, measured_sum) VALUES (?, ?, ?, ?, ?)");
   }
 
   /** Insert a parsed vote into the vote table. */
@@ -116,22 +115,17 @@ class Database implements AutoCloseable {
         }
       }
     }
-    int voteId = -1;
-    this.psVoteInsert.clearParameters();
-    this.psVoteInsert.setTimestamp(1,
-        Timestamp.from(ZonedDateTime.of(vote.validAfter,
-            ZoneId.of("UTC")).toInstant()), calendar);
-    this.psVoteInsert.setInt(2, authorityId);
-    this.psVoteInsert.setLong(3, vote.measuredSum);
-    this.psVoteInsert.execute();
-    try (ResultSet rs = this.psVoteInsert.getGeneratedKeys()) {
-      if (rs.next()) {
-        voteId = rs.getInt(1);
-      }
-    }
-    if (voteId < 0) {
-      throw new SQLException("Could not retrieve auto-generated key for new "
-          + "vote entry.");
+    for (int measuredSumsIndex = 0; measuredSumsIndex < 4;
+         measuredSumsIndex++) {
+      this.psVoteInsert.clearParameters();
+      this.psVoteInsert.setTimestamp(1,
+          Timestamp.from(ZonedDateTime.of(vote.validAfter,
+              ZoneId.of("UTC")).toInstant()), calendar);
+      this.psVoteInsert.setInt(2, authorityId);
+      this.psVoteInsert.setBoolean(3, 1 == (measuredSumsIndex & 1));
+      this.psVoteInsert.setBoolean(4, 2 == (measuredSumsIndex & 2));
+      this.psVoteInsert.setLong(5, vote.measuredSums[measuredSumsIndex]);
+      this.psVoteInsert.execute();
     }
   }
 
@@ -159,6 +153,10 @@ class Database implements AutoCloseable {
         outputLine.validAfterDate = rs.getDate(
             OutputLine.Column.VALID_AFTER_DATE.name(), calendar).toLocalDate();
         outputLine.nickname = rs.getString(OutputLine.Column.NICKNAME.name());
+        outputLine.haveGuardFlag = rs.getBoolean(
+            OutputLine.Column.HAVE_GUARD_FLAG.name());
+        outputLine.haveExitFlag = rs.getBoolean(
+            OutputLine.Column.HAVE_EXIT_FLAG.name());
         outputLine.measuredSumAvg = rs.getLong(
             OutputLine.Column.MEASURED_SUM_AVG.name());
         statistics.add(outputLine);
diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/OutputLine.java b/src/main/java/org/torproject/metrics/stats/totalcw/OutputLine.java
index 450dbac..5587e5d 100644
--- a/src/main/java/org/torproject/metrics/stats/totalcw/OutputLine.java
+++ b/src/main/java/org/torproject/metrics/stats/totalcw/OutputLine.java
@@ -13,7 +13,8 @@ class OutputLine {
   /** Column names used in the database and in the first line of the output
    * file. */
   enum Column {
-    VALID_AFTER_DATE, NICKNAME, MEASURED_SUM_AVG
+    VALID_AFTER_DATE, NICKNAME, HAVE_GUARD_FLAG, HAVE_EXIT_FLAG,
+    MEASURED_SUM_AVG
   }
 
   /** Column headers joined together with the given delimiter. */
@@ -28,6 +29,12 @@ class OutputLine {
   /** Server type, which can be "relay" or "bridge". */
   String nickname;
 
+  /** Whether contained relays all have the "Guard" flag. */
+  boolean haveGuardFlag;
+
+  /** Whether contained relays all have the "Exit" flag. */
+  boolean haveExitFlag;
+
   /** Mean value of total measured bandwidths of all relays over the day. */
   Long measuredSumAvg;
 
@@ -35,7 +42,8 @@ class OutputLine {
    * file. */
   @Override
   public String toString() {
-    return String.format("%s,%s,%d", validAfterDate, nickname, measuredSumAvg);
+    return String.format("%s,%s,%s,%s,%d", validAfterDate, nickname,
+        haveGuardFlag ? "t" : "f", haveExitFlag ? "t" : "f", measuredSumAvg);
   }
 }
 
diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java b/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java
index b6a35b4..6070822 100644
--- a/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java
+++ b/src/main/java/org/torproject/metrics/stats/totalcw/Parser.java
@@ -17,18 +17,21 @@ class Parser {
    * contain any bandwidth measurements. */
   TotalcwRelayNetworkStatusVote parseRelayNetworkStatusVote(
       RelayNetworkStatusVote vote) {
-    Long measuredSum = null;
+    boolean containsMeasuredBandwidths = false;
+    long[] measuredSums = new long[4];
     for (NetworkStatusEntry entry : vote.getStatusEntries().values()) {
       if (null == entry.getFlags() || !entry.getFlags().contains("Running")
           || entry.getMeasured() < 0L) {
         continue;
       }
-      if (null == measuredSum) {
-        measuredSum = 0L;
-      }
-      measuredSum += entry.getMeasured();
+      containsMeasuredBandwidths = true;
+      /* Encode flags as sum of Guard = 1 and (Exit and !BadExit) = 2. */
+      int measuredSumsIndex = (entry.getFlags().contains("Guard") ? 1 : 0)
+          + (entry.getFlags().contains("Exit")
+          && !entry.getFlags().contains("BadExit") ? 2 : 0);
+      measuredSums[measuredSumsIndex] += entry.getMeasured();
     }
-    if (null == measuredSum) {
+    if (!containsMeasuredBandwidths) {
       /* Return null, because we wouldn't want to add this vote to the database
        * anyway. */
       return null;
@@ -39,7 +42,7 @@ class Parser {
         .atZone(ZoneId.of("UTC")).toLocalDateTime();
     parsedVote.identityHex = vote.getIdentity();
     parsedVote.nickname = vote.getNickname();
-    parsedVote.measuredSum = measuredSum;
+    parsedVote.measuredSums = measuredSums;
     return parsedVote;
   }
 }
diff --git a/src/main/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVote.java b/src/main/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVote.java
index ff56d91..0c5a095 100644
--- a/src/main/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVote.java
+++ b/src/main/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVote.java
@@ -19,7 +19,9 @@ class TotalcwRelayNetworkStatusVote {
    * key. */
   String identityHex;
 
-  /** Sum of bandwidth measurements of all contained status entries. */
-  long measuredSum;
+  /** Sums of bandwidth measurements of all contained status entries with four
+   * entries: 0 = neither Exit nor Guard, 1 = only Guard, 2 = only Exit, and
+   * 3 = both Guard and Exit. */
+  long[] measuredSums;
 }
 
diff --git a/src/main/sql/totalcw/init-totalcw.sql b/src/main/sql/totalcw/init-totalcw.sql
index d723adb..cdba275 100644
--- a/src/main/sql/totalcw/init-totalcw.sql
+++ b/src/main/sql/totalcw/init-totalcw.sql
@@ -31,21 +31,27 @@ CREATE TABLE vote (
   -- Numeric identifier uniquely identifying the authority generating this vote.
   authority_id INTEGER REFERENCES authority (authority_id),
 
+  -- Whether contained relays had the Guard flag assigned.
+  have_guard_flag BOOLEAN NOT NULL,
+
+  -- Whether contained relays had the Exit flag assigned.
+  have_exit_flag BOOLEAN NOT NULL,
+
   -- Sum of bandwidth measurements of all contained status entries.
   measured_sum BIGINT NOT NULL,
 
-  UNIQUE (valid_after, authority_id)
+  UNIQUE (valid_after, authority_id, have_guard_flag, have_exit_flag)
 );
 
 -- View on aggregated total consensus weight statistics in a format that is
 -- compatible for writing to an output CSV file. Votes are only included in the
 -- output if at least 12 votes are known for a given authority and day.
 CREATE OR REPLACE VIEW totalcw AS
-SELECT DATE(valid_after) AS valid_after_date, nickname,
-  FLOOR(AVG(measured_sum)) AS measured_sum_avg
+SELECT DATE(valid_after) AS valid_after_date, nickname, have_guard_flag,
+  have_exit_flag, FLOOR(AVG(measured_sum)) AS measured_sum_avg
 FROM vote NATURAL JOIN authority
-GROUP BY DATE(valid_after), nickname
+GROUP BY DATE(valid_after), nickname, have_guard_flag, have_exit_flag
 HAVING COUNT(vote_id) >= 12
   AND DATE(valid_after) < (SELECT MAX(DATE(valid_after)) FROM vote)
-ORDER BY DATE(valid_after), nickname;
+ORDER BY DATE(valid_after), nickname, have_guard_flag, have_exit_flag;
 
diff --git a/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java b/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java
index 7c5ecc7..189b3b7 100644
--- a/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java
+++ b/src/test/java/org/torproject/metrics/stats/totalcw/TotalcwRelayNetworkStatusVoteTest.java
@@ -4,6 +4,7 @@
 package org.torproject.metrics.stats.totalcw;
 
 import static junit.framework.TestCase.assertEquals;
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertNull;
 
 import org.torproject.descriptor.Descriptor;
@@ -35,19 +36,22 @@ public class TotalcwRelayNetworkStatusVoteTest {
         { "2018-10-15-00-00-00-vote-0232AF901C31A04EE9848595AF9BB7620D4C5B2E-"
             + "55A38ED50848BE1F13C6A35C3CA637B0D962C2EF.part",
             ZonedDateTime.parse("2018-10-15T00:00:00Z").toLocalDateTime(),
-            "dannenberg", "0232AF901C31A04EE9848595AF9BB7620D4C5B2E", -1L },
+            "dannenberg", "0232AF901C31A04EE9848595AF9BB7620D4C5B2E", null },
         { "2018-10-15-00-00-00-vote-27102BC123E7AF1D4741AE047E160C91ADC76B21-"
             + "049AB3179B12DACC391F06A10C2A8904E4339D33.part",
             ZonedDateTime.parse("2018-10-15T00:00:00Z").toLocalDateTime(),
-            "bastet", "27102BC123E7AF1D4741AE047E160C91ADC76B21", 138700L },
+            "bastet", "27102BC123E7AF1D4741AE047E160C91ADC76B21",
+            new long[] { 13700L, 47220L, 17080L, 60700L } },
         { "2018-10-15-00-00-00-vote-ED03BB616EB2F60BEC80151114BB25CEF515B226-"
             + "2669AD153408F88E416CE6206D1A75EC3324A2F4.part",
             ZonedDateTime.parse("2018-10-15T00:00:00Z").toLocalDateTime(),
-            "gabelmoo", "ED03BB616EB2F60BEC80151114BB25CEF515B226", 133370L },
+            "gabelmoo", "ED03BB616EB2F60BEC80151114BB25CEF515B226",
+            new long[] { 18020L, 43200L, 26150L, 46000L } },
         { "2018-10-15-00-00-00-vote-EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97-"
             + "38C6A19F78948B689345EE41D7119D76246C4D3E.part",
             ZonedDateTime.parse("2018-10-15T00:00:00Z").toLocalDateTime(),
-            "Faravahar", "EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97", 158395L }
+            "Faravahar", "EFCBE720AB3A82B99F9E953CD5BF50F7EEFC7B97",
+            new long[] { 17365L, 52030L, 35400L, 53600L } }
     });
   }
 
@@ -64,7 +68,7 @@ public class TotalcwRelayNetworkStatusVoteTest {
   public String expectedIdentityHex;
 
   @Parameter(4)
-  public long expectedMeasuredSum;
+  public long[] expectedMeasuredSums;
 
   @Test
   public void testParseVote() throws Exception {
@@ -82,13 +86,13 @@ public class TotalcwRelayNetworkStatusVoteTest {
         sb.toString().getBytes(), new File(this.fileName), this.fileName)) {
       TotalcwRelayNetworkStatusVote parsedVote = new Parser()
           .parseRelayNetworkStatusVote((RelayNetworkStatusVote) descriptor);
-      if (this.expectedMeasuredSum < 0L) {
+      if (null == this.expectedMeasuredSums) {
         assertNull(parsedVote);
       } else {
         assertEquals(this.expectedValidAfter, parsedVote.validAfter);
         assertEquals(this.expectedNickname, parsedVote.nickname);
         assertEquals(this.expectedIdentityHex, parsedVote.identityHex);
-        assertEquals(this.expectedMeasuredSum, parsedVote.measuredSum);
+        assertArrayEquals(this.expectedMeasuredSums, parsedVote.measuredSums);
       }
     }
   }



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits