[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [metrics-web/master] Remove module on disagreement among dirauths.



commit 99f87e820289d1fdba7d6362cb8bb84693f09c94
Author: Karsten Loesing <karsten.loesing@xxxxxxx>
Date:   Thu Sep 29 17:21:43 2016 +0200

    Remove module on disagreement among dirauths.
    
    Implements #20227.
---
 .../org/torproject/metrics/collectdescs/Main.java  |   1 -
 modules/disagreement/.gitignore                    |   2 -
 modules/disagreement/build.xml                     |  38 --
 .../org/torproject/metrics/disagreement/Main.java  | 642 ---------------------
 shared/bin/20-run-disagreement-stats.sh            |   5 -
 shared/bin/99-copy-stats-files.sh                  |   1 -
 website/etc/metrics.json                           |   4 +-
 7 files changed, 2 insertions(+), 691 deletions(-)

diff --git a/modules/collectdescs/src/org/torproject/metrics/collectdescs/Main.java b/modules/collectdescs/src/org/torproject/metrics/collectdescs/Main.java
index 88955b4..8be2e0b 100644
--- a/modules/collectdescs/src/org/torproject/metrics/collectdescs/Main.java
+++ b/modules/collectdescs/src/org/torproject/metrics/collectdescs/Main.java
@@ -24,7 +24,6 @@ public class Main {
             "/recent/relay-descriptors/consensuses/",
             "/recent/relay-descriptors/extra-infos/",
             "/recent/relay-descriptors/server-descriptors/",
-            "/recent/relay-descriptors/votes/",
             "/recent/torperf/"
         }, 0L, new File("../../shared/in"), true);
   }
diff --git a/modules/disagreement/.gitignore b/modules/disagreement/.gitignore
deleted file mode 100644
index bb84c80..0000000
--- a/modules/disagreement/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-stats/
-
diff --git a/modules/disagreement/build.xml b/modules/disagreement/build.xml
deleted file mode 100644
index aed679b..0000000
--- a/modules/disagreement/build.xml
+++ /dev/null
@@ -1,38 +0,0 @@
-<project default="run" name="disagreement" basedir=".">
-
-  <property name="disagreement-sources" value="src/main/java"/>
-  <property name="disagreement-libs" value="../../shared/lib"/>
-  <property name="disagreement-classes" value="classes"/>
-  <path id="classpath">
-    <pathelement path="${disagreement-classes}"/>
-    <fileset dir="${disagreement-libs}">
-      <include name="descriptor-1.2.0.jar"/>
-      <include name="commons-compress-1.9.jar"/>
-      <include name="xz-1.0.jar"/>
-    </fileset>
-  </path>
-
-  <target name="compile">
-    <mkdir dir="${disagreement-classes}"/>
-    <javac destdir="${disagreement-classes}"
-           srcdir="${disagreement-sources}"
-           source="1.7"
-           target="1.7"
-           debug="true"
-           deprecation="true"
-           optimize="false"
-           failonerror="true"
-           includeantruntime="false">
-      <classpath refid="classpath"/>
-    </javac>
-  </target>
-
-  <target name="run" depends="compile">
-    <java fork="true"
-          maxmemory="2g"
-          classname="org.torproject.metrics.disagreement.Main">
-      <classpath refid="classpath"/>
-    </java>
-  </target>
-</project>
-
diff --git a/modules/disagreement/src/main/java/org/torproject/metrics/disagreement/Main.java b/modules/disagreement/src/main/java/org/torproject/metrics/disagreement/Main.java
deleted file mode 100644
index 05159a3..0000000
--- a/modules/disagreement/src/main/java/org/torproject/metrics/disagreement/Main.java
+++ /dev/null
@@ -1,642 +0,0 @@
-/* Copyright 2016 The Tor Project
- * See LICENSE for licensing information */
-
-package org.torproject.metrics.disagreement;
-
-import org.torproject.descriptor.Descriptor;
-import org.torproject.descriptor.DescriptorFile;
-import org.torproject.descriptor.DescriptorReader;
-import org.torproject.descriptor.DescriptorSourceFactory;
-import org.torproject.descriptor.NetworkStatusEntry;
-import org.torproject.descriptor.RelayNetworkStatusVote;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.LineNumberReader;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TimeZone;
-import java.util.TreeMap;
-
-/* Read all relay network status votes from the in/ subdirectory with a
- * valid-after time of 12:00:00, extract attributes like relay flags or
- * bandwidth measurements that the directory authorities assigned to
- * relays, and output aggregate statistics on disagreement among the
- * directory authorities.
- *
- * When initializing from descriptor archives, put tarballs in the in/
- * subdirectory, run this code, and then move the tarballs away.
- * Otherwise, tarballs will be re-processed in each subsequent execution.
- *
- * Recent descriptors can stay in the in/ subdirectory and be re-processed
- * in each execution. */
-public class Main {
-
-  /** Creates a new instance of this class that executes this
-   * data-processing module. */
-  public static void main(String[] args) throws Exception {
-    new Main().run();
-  }
-
-  /** Executes this data-processing module. */
-  public void run() throws Exception {
-    readResults();
-    readDescriptors();
-    aggregate();
-    writeResults();
-  }
-
-  /* We're processing a lot of strings, including authority identities,
-   * relay fingerprints, and attributes like relay flags.  It would be
-   * wasteful to store more than one instance of these strings in memory.
-   * We also want to store them as part of long integers below.
-   *
-   * That's why we're resolving all strings to integers and keeping maps
-   * from string to integers.  In case of authority identities and relay
-   * fingerprints we don't need to resolve integers back to strings, but
-   * in case of attributes we need to put attribute strings into the
-   * output file, so we're also keeping a list of attribute strings in
-   * insertion order. */
-  private Map<String, Integer> authorityIndexes =
-      new HashMap<String, Integer>();
-  private Map<String, Integer> fingerprintIndexes =
-      new HashMap<String, Integer>();
-  private Map<String, Integer> attributeIndexes =
-      new HashMap<String, Integer>();
-  private List<String> attributeStrings = new ArrayList<String>();
-
-  /** Initializes this class. */
-  public Main() {
-
-    /* Initialize maps from strings to integers and back by adding the
-     * empty string as 0-th element.  This is necessary, because we want
-     * to be able to treat 0 as special case below.  Once we add non-empty
-     * strings to maps, they'll be indexed starting at 1. */
-    this.getAuthorityIndexForString("");
-    this.getFingerprintIndexForString("");
-    this.getAttributeIndexForString("");
-  }
-
-  /* Resolve the given authority string to its authority index, possibly
-   * after adding it to the mapping if it wasn't contained before. */
-  protected int getAuthorityIndexForString(String authorityString) {
-    return putToMapsAndReturnIndex(this.authorityIndexes, null,
-        authorityString);
-  }
-
-  /* Resolve the given fingerprint string to its fingerprint index,
-   * possibly after adding it to the mapping if it wasn't contained
-   * before. */
-  protected int getFingerprintIndexForString(String fingerprintString) {
-    return putToMapsAndReturnIndex(this.fingerprintIndexes, null,
-        fingerprintString);
-  }
-
-  /* Resolve the given attribute string to its attribute index, possibly
-   * after adding it to the mapping if it wasn't contained before. */
-  protected int getAttributeIndexForString(String attributeString) {
-    return putToMapsAndReturnIndex(this.attributeIndexes,
-        this.attributeStrings, attributeString);
-  }
-
-  /* Resolve the given attribute index to its attribute string, or return
-   * null if this attribute index doesn't exist. */
-  protected String getAttributeStringForIndex(int attributeIndex) {
-    if (attributeIndex < 0 || attributeIndex >= attributeStrings.size()) {
-      return null;
-    } else {
-      return this.attributeStrings.get(attributeIndex);
-    }
-  }
-
-  /* Helper method: Return an index for a given string and possibly put
-   * the string into the map and corresponding list if it was not
-   * contained before. */
-  protected int putToMapsAndReturnIndex(
-      Map<String, Integer> stringToIntMap, List<String> strings,
-      String string) {
-    Integer index = stringToIntMap.get(string);
-    if (index == null) {
-      if (strings != null) {
-        strings.add(string);
-      }
-      stringToIntMap.put(string, stringToIntMap.size());
-    }
-    return stringToIntMap.get(string);
-  }
-
-  /* The following code is heavily optimized towards low memory usage,
-   * because we're potentially processing a lot of data, especially when
-   * initializing from descriptor archives.
-   *
-   * We need to be able to store weeks or even months of votes in memory,
-   * which is really a lot of data.  The reason is that we need all votes
-   * published at the same hour to be present when running aggregations,
-   * but we cannot rely on the order of incoming votes.  That's why we
-   * have to process votes in two steps: first, we extract everything we
-   * need from votes and store it in memory, and second, we aggregate what
-   * we have in memory and output aggregate results.
-   *
-   * We're going rather low-level here by converting each attribute that
-   * an authority assigned to a relay at a certain vote valid-after time
-   * into a single 64-bit signed long integer.  We're doing this by
-   * subdividing (most of) the available bits into ranges for the
-   * different parts we want to store.  As an added requirement we're
-   * arranging ranges in a way that we can later process long values in
-   * sorted order without keeping much state.  Ranges are:
-   *  - 22 bits for the valid-after time in half hours since 1970-01-01
-   *    00:00:00 (which won't overflow until year 2209),
-   *  - 8 bits for the attribute index (which allows up to 253 different
-   *    relay flags in addition to reserved 0, "Listed", and
-   *    "Measured"),
-   *  - 24 bits for the fingerprint index (which allows over 16 million
-   *    different relay fingerprints in the current execution), and
-   *  - 6 bits for the authority index (which allows up to 63 different
-   *    authorities in addition to the reserved 0).
-   *
-   * If any of these numbers overflows during the execution, we'll detect
-   * that, suggest to process less data at once, and exit with an error.
-   * Still, it should be possible to process months of data in a single
-   * execution.  For example, one set of votes published at the same
-   * valid-after hour in December 2015 required to keep 420,000 long
-   * values in memory, which is roughly 3.2 MiB plus list overhead. */
-  protected List<Long> assignments = new ArrayList<Long>();
-
-  protected static final int VALIDAFTER_LEN = 22;
-
-  protected static final int ATTRIBUTE_LEN = 8;
-
-  protected static final int FINGERPRINT_LEN = 24;
-
-  protected static final int AUTHORITY_LEN = 6;
-
-  protected static final int VALIDAFTER_SHIFT = ATTRIBUTE_LEN
-      + FINGERPRINT_LEN + AUTHORITY_LEN;
-
-  protected static final int ATTRIBUTE_SHIFT = FINGERPRINT_LEN
-      + AUTHORITY_LEN;
-
-  protected static final int FINGERPRINT_SHIFT = AUTHORITY_LEN;
-
-  protected static final int AUTHORITY_SHIFT = 0;
-
-  /* Define some constants for timestamp math. */
-  protected static final long HALF_HOUR = 30L * 60L * 1000L;
-
-  protected static final long ONE_HOUR = 2L * HALF_HOUR;
-
-  protected static final long HALF_DAY = 12L * ONE_HOUR;
-
-  protected static final long ONE_DAY = 2L * HALF_DAY;
-
-  /* Convert the given valid-after time in milliseconds, attribute index,
-   * fingerprint index, and authority index to a long integer following
-   * the conversion rules stated above.  Return -1 in case of
-   * overflows. */
-  protected static long convertToLongValue(long validAfterMillis,
-      int attributeIndex, int fingerprintIndex, int authorityIndex) {
-    long validAfterHalfHours = validAfterMillis / HALF_HOUR;
-    if (validAfterHalfHours < 0L
-        || validAfterHalfHours >= (1L << VALIDAFTER_LEN)) {
-      return -1;
-    }
-    if (attributeIndex < 0 || attributeIndex >= (1 << ATTRIBUTE_LEN)) {
-      return -1;
-    }
-    if (fingerprintIndex < 0
-        || fingerprintIndex >= (1 << FINGERPRINT_LEN)) {
-      return -1;
-    }
-    if (authorityIndex < 0 || authorityIndex >= (1 << AUTHORITY_LEN)) {
-      return -1;
-    }
-    long longValue = (validAfterHalfHours << VALIDAFTER_SHIFT)
-        + ((long) attributeIndex << ATTRIBUTE_SHIFT)
-        + ((long) fingerprintIndex << FINGERPRINT_SHIFT)
-        + ((long) authorityIndex << AUTHORITY_SHIFT);
-    return longValue;
-  }
-
-  /* Extract the valid-after time in milliseconds from the given long
-   * integer value. */
-  protected static long extractValidAfterMillisFromLongValue(
-      long longValue) {
-    return (longValue >> VALIDAFTER_SHIFT) * HALF_HOUR;
-  }
-
-  /* Extract the attribute index from the given long integer value. */
-  protected static int extractAttributeIndexFromLongValue(
-      long longValue) {
-    return (int) ((longValue >> ATTRIBUTE_SHIFT) % (1 << ATTRIBUTE_LEN));
-  }
-
-  /* Extract the fingerprint index from the given long integer value. */
-  protected static int extractFingerprintIndexFromLongValue(
-      long longValue) {
-    return (int) ((longValue >> FINGERPRINT_SHIFT)
-        % (1 << FINGERPRINT_LEN));
-  }
-
-  /* Extract the authority index from the given long integer value. */
-  protected static int extractAuthorityIndexFromLongValue(
-      long longValue) {
-    return (int) ((longValue >> AUTHORITY_SHIFT) % (1 << AUTHORITY_LEN));
-  }
-
-  /* Keep all aggregated results in memory, so that we easily merge new
-   * results obtained in the current execution.
-   *
-   * Another reason for keeping all results in memory is that we may have
-   * processed some votes from a given valid-after time before, and now we
-   * need to decide whether we keep old results or replace them with new
-   * results.  We decide this by using results that are based on more
-   * votes, which could be old or new results.
-   *
-   * Map keys are valid-after times formatted as strings, map values are
-   * sorted lists of all subsequent columns starting with that valid-after
-   * time. */
-  protected SortedMap<String, List<String>> results =
-      new TreeMap<String, List<String>>();
-
-  /* Store all results in this .csv file. */
-  protected File resultsFile = new File("stats/disagreement.csv");
-
-  /* Use the following .csv header line for results. */
-  protected String resultsHeaderLine =
-      "validafter,attribute,votes,required,max,relays";
-
-  /* Read results from the previous execution to memory and do some minor
-   * validation of the file format while doing so.  Return immediately
-   * without error if the file does not exist (yet). */
-  private void readResults() throws Exception {
-    if (!this.resultsFile.exists()) {
-      return;
-    }
-    LineNumberReader lnr = new LineNumberReader(new BufferedReader(
-        new FileReader(this.resultsFile)));
-    String line;
-    if ((line = lnr.readLine()) == null
-        || !line.equals(this.resultsHeaderLine)) {
-      lnr.close();
-      throw new IOException("Unexpected line " + lnr.getLineNumber()
-          + " in " + this.resultsFile + ".");
-    }
-    while ((line = lnr.readLine()) != null) {
-      if (!line.contains(",")) {
-        lnr.close();
-        throw new IOException("Cannot parse invalid line "
-            + lnr.getLineNumber() + " in " + this.resultsFile + ".");
-      }
-      int indexOfFirstComma = line.indexOf(",");
-      String validafter = line.substring(0, indexOfFirstComma);
-      String otherColumns = line.substring(indexOfFirstComma);
-      if (!this.results.containsKey(validafter)) {
-        this.results.put(validafter, new ArrayList<String>());
-      }
-      this.results.get(validafter).add(otherColumns);
-    }
-    lnr.close();
-  }
-
-  /* Write results to the results file.  More precisely, write them to a
-   * temporary file and rename it to the target file to avoid failing half
-   * way through and losing the previous results file. */
-  private void writeResults() throws Exception {
-    this.resultsFile.getParentFile().mkdirs();
-    File resultsTmpFile = new File(this.resultsFile + ".tmp");
-    BufferedWriter bw = new BufferedWriter(new FileWriter(
-        resultsTmpFile));
-    bw.write(this.resultsHeaderLine + "\n");
-    for (Map.Entry<String, List<String>> e : this.results.entrySet()) {
-      String validafter = e.getKey();
-      for (String otherColumns : e.getValue()) {
-        bw.write(validafter + otherColumns + "\n");
-      }
-    }
-    bw.close();
-    resultsTmpFile.renameTo(this.resultsFile);
-  }
-
-  /* Read relay network status votes from this directory. */
-  protected File[] inDirectories = new File[] {
-      new File("../../shared/in/archive/relay-descriptors/votes"),
-      new File("../../shared/in/recent/relay-descriptors/votes")
-  };
-
-  /* Read relay network status votes from disk and extract all relevant
-   * pieces from them. */
-  protected void readDescriptors() throws Exception {
-    DescriptorReader descriptorReader =
-        DescriptorSourceFactory.createDescriptorReader();
-    descriptorReader.setMaxDescriptorFilesInQueue(5);
-    for (File inDirectory : this.inDirectories) {
-      descriptorReader.addDirectory(inDirectory);
-    }
-    Iterator<DescriptorFile> descriptorFiles =
-        descriptorReader.readDescriptors();
-    while (descriptorFiles.hasNext()) {
-      DescriptorFile descriptorFile = descriptorFiles.next();
-      for (Descriptor descriptor : descriptorFile.getDescriptors()) {
-        if ((descriptor instanceof RelayNetworkStatusVote)) {
-          RelayNetworkStatusVote vote =
-              (RelayNetworkStatusVote) descriptor;
-          processVote(vote);
-        }
-      }
-    }
-  }
-
-  private static final String LISTED_ATTRIBUTE = "Listed";
-
-  private static final String MEASURED_ATTRIBUTE = "Measured";
-
-  /* Process a single relay network status vote. */
-  private void processVote(RelayNetworkStatusVote vote) throws Exception {
-    long validAfterMillis = vote.getValidAfterMillis();
-    if (validAfterMillis % ONE_DAY != HALF_DAY) {
-      /* Only process votes with a valid-after time of 12:00:00 as a means
-       * to reduce the overall amount of data. */
-      return;
-    }
-
-    /* Use the authority's identity to distinguish votes. */
-    String authorityIdentity = vote.getIdentity();
-
-    /* Collect a set of all attributes that the authority assigns in this
-     * vote, which includes all known flags, the general "Listed"
-     * attribute for listing relays, and possibly the "Measured" attribute
-     * for bandwidth-measured relays. */
-    Set<String> knownAttributes = new HashSet<String>(
-        vote.getKnownFlags());
-    knownAttributes.add(LISTED_ATTRIBUTE);
-
-    /* Go through all status entries in this vote and remember which
-     * attributes this authority assigns to which relays. */
-    for (NetworkStatusEntry entry
-        : vote.getStatusEntries().values()) {
-
-      /* Use the relay's fingerprint to distinguish relays. */
-      String fingerprint = entry.getFingerprint();
-
-      /* Compile a set of all attributes assigned to this relay, including
-       * all relay flags, "Listed", and possibly "Measured". */
-      Set<String> attributes = new HashSet<String>(entry.getFlags());
-      attributes.add(LISTED_ATTRIBUTE);
-      if (entry.getMeasured() >= 0L) {
-        attributes.add(MEASURED_ATTRIBUTE);
-        knownAttributes.add(MEASURED_ATTRIBUTE);
-      }
-
-      /* Remember all attributes assigned to this relay. */
-      this.addAssignedAttributes(validAfterMillis, attributes,
-          fingerprint, authorityIdentity);
-    }
-
-    /* Remember all attributes assigned by the authority in this vote. */
-    this.addKnownAttributes(validAfterMillis, knownAttributes,
-        authorityIdentity);
-
-    /* Remember all fingerprints voted on by the authority at the given
-     * valid-after time. */
-    this.addKnownFingerprints(validAfterMillis,
-        vote.getStatusEntries().keySet());
-  }
-
-  /* Remember all attributes assigned to a given relay by an authority at
-   * a given valid-after time.  These are converted to long integers
-   * with all components being non-zero. */
-  protected void addAssignedAttributes(long validAfterMillis,
-      Set<String> attributes, String fingerprint,
-      String authorityIdentity) throws Exception {
-    int fingerprintIndex = this.getFingerprintIndexForString(fingerprint);
-    int authorityIndex = this.getAuthorityIndexForString(
-        authorityIdentity);
-    for (String attribute : attributes) {
-      int attributeIndex = this.getAttributeIndexForString(attribute);
-      long longValue = convertToLongValue(validAfterMillis,
-          attributeIndex, fingerprintIndex, authorityIndex);
-      if (longValue < 0L) {
-        throw new Exception("Could not convert vote data to the internal "
-            + "format.  Try processing fewer votes at once.");
-      }
-      this.assignments.add(longValue);
-    }
-  }
-
-  /* Remember all attributes voted on by an authority at a given
-   * valid-after time.  These are converted to long integers with
-   * fingerprint component being zero. */
-  protected void addKnownAttributes(long validAfterMillis,
-      Set<String> knownAttributes, String authorityIdentity)
-      throws Exception {
-    int authorityIndex = this.getAuthorityIndexForString(
-        authorityIdentity);
-    for (String attribute : knownAttributes) {
-      int attributeIndex = this.getAttributeIndexForString(attribute);
-      long longValue = convertToLongValue(validAfterMillis,
-          attributeIndex, 0, authorityIndex);
-      if (longValue < 0L) {
-        throw new Exception("Could not convert vote data to the internal "
-            + "format.  Try processing fewer votes at once.");
-      }
-      this.assignments.add(longValue);
-    }
-  }
-
-  /* Remember all fingerprints known by an authority at a given
-   * valid-after time.  These are converted to long integers with
-   * attribute and authority component being zero. */
-  protected void addKnownFingerprints(long validAfterMillis,
-      Set<String> knownFingerprints) throws Exception {
-    for (String fingerprint : knownFingerprints) {
-      int fingerprintIndex = this.getFingerprintIndexForString(
-          fingerprint);
-      long longValue = convertToLongValue(validAfterMillis, 0,
-          fingerprintIndex, 0);
-      if (longValue < 0L) {
-        throw new Exception("Could not convert vote data to the internal "
-            + "format.  Try processing fewer votes at once.");
-      }
-      this.assignments.add(longValue);
-    }
-  }
-
-  /* Aggregate everything we extracted from votes earlier into statistics
-   * on disagreement among directory authorities. */
-  protected void aggregate() {
-
-    /* Initialize a date format for formatting valid-after times. */
-    DateFormat dateTimeFormat = new SimpleDateFormat(
-        "yyyy-MM-dd HH:mm:ss", Locale.US);
-    dateTimeFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
-
-    /* Sort long integer values and append the largest possible long
-     * value, so that we can process values in the following order
-     * (columns are valid-after time, attribute, fingerprint, authority;
-     * + stands for a value > 0, 0 stands for 0):
-     *  (+, 0, +, 0): all fingerprints known at a given valid-after time
-     *  (+, +, 0, +): all attributes known by a given authority
-     *  (+, +, +, +): attributes assigned to relays by authority
-     *  ...
-     *  (MAX_VALUE): end-of-list marker */
-    Collections.sort(this.assignments);
-    this.assignments.add(Long.MAX_VALUE);
-
-    /* Remember long value and some of its components from the last
-     * iteration. */
-    long lastLongValue = -1L;
-    long lastValidAfterMillis = -1L;
-    int lastAttributeIndex = -1;
-    int lastFingerprintIndex = -1;
-
-    /* Keep a list of all output lines for a single valid-after time. */
-    List<String> outputLines = new ArrayList<String>();
-
-    /* Keep counters for the number of fingerprints seen at a valid-after
-     * time, the number of authorities voting on an attribute, and the
-     * number of votes that a relay received for a given attribute. */
-    int knownFingerprintsByAllAuthorities = 0;
-    int authoritiesVotingOnAttribute = 0;
-    int votesForAttribute = 0;
-
-    /* Keep counters of relays receiving a given number of votes on an
-     * attribute.  The number at element i is the number of relays
-     * receiving i votes. */
-    int[] relays = new int[(1 << AUTHORITY_LEN)];
-
-    /* Go through all long values in ascending order. */
-    for (long longValue : this.assignments) {
-
-      /* Skip duplicate values. */
-      if (lastLongValue == longValue) {
-        continue;
-      }
-
-      /* If we're looking at attributes assigned to relays and just moved
-       * from one non-zero fingerprint to the next, we need to wrap up
-       * results for the last fingerprint before moving on. */
-      int fingerprintIndex = extractFingerprintIndexFromLongValue(
-          longValue);
-      if (lastAttributeIndex > 0 && lastFingerprintIndex > 0
-          && lastFingerprintIndex != fingerprintIndex) {
-
-        /* This relay received at least one vote for the given attribute,
-         * or otherwise it wouldn't be contained in the list of long
-         * values.  Increment the counter for the number of votes we
-         * counted for this attribute, and then reset that counter. */
-        relays[0]--;
-        relays[votesForAttribute]++;
-        votesForAttribute = 0;
-      }
-
-      /* If we're looking at attributes and just moved from one non-zero
-       * attribute to the next, we need to wrap up results for the last
-       * attribute before moving on.  And if we just moved to the first
-       * attribute, initialize counters. */
-      int attributeIndex = extractAttributeIndexFromLongValue(longValue);
-      if (lastAttributeIndex >= 0
-          && lastAttributeIndex != attributeIndex) {
-
-        /* If we just finished a non-zero attribute, wrap it up.
-         * Determine the number of votes required for getting into the
-         * consensus, which is typically the majority of votes, except for
-         * the "Measured" attribute, where it's set to 3.  Format one
-         * output line for each possible number of votes, from 0 to the
-         * total number of authorities voting on the attribute. */
-        if (lastAttributeIndex > 0) {
-          String lastAttribute = this.getAttributeStringForIndex(
-              lastAttributeIndex);
-          int required = lastAttribute.equals(MEASURED_ATTRIBUTE) ? 3
-              : (authoritiesVotingOnAttribute / 2) + 1;
-          for (int i = 0; i <= authoritiesVotingOnAttribute; i++) {
-            outputLines.add(String.format(",%s,%d,%d,%d,%d",
-                lastAttribute, i, required, authoritiesVotingOnAttribute,
-                relays[i]));
-          }
-        }
-
-        /* Reset counters and initialize the bucket at index 0 with the
-         * total number of fingerprints known at this valid-after time. */
-        authoritiesVotingOnAttribute = 0;
-        relays = new int[(1 << AUTHORITY_LEN)];
-        relays[0] = knownFingerprintsByAllAuthorities;
-      }
-
-      /* If we just moved from one valid-after time to the next, we need
-       * to wrap up results for the last valid-after time before moving
-       * on. */
-      long validAfterMillis = extractValidAfterMillisFromLongValue(
-          longValue);
-      if (lastValidAfterMillis >= 0L
-          && lastValidAfterMillis < validAfterMillis) {
-
-        /* Check if results already contain lines for this valid-after
-         * time.  If so, only replace them with new results lines if there
-         * are more new lines than old lines.  The rationale is that more
-         * lines are very likely based on more votes, and we want to
-         * include as many votes as possible in the aggregation. */
-        String validAfterString = dateTimeFormat.format(
-            lastValidAfterMillis);
-        if (!this.results.containsKey(validAfterString)
-            || this.results.get(validAfterString).size()
-            < outputLines.size()) {
-
-          /* Sort results lines, and then put them in. */
-          Collections.sort(outputLines);
-          this.results.put(validAfterString, outputLines);
-        }
-
-        /* Prepare for aggregating votes from the next valid-after
-         * time. */
-        outputLines = new ArrayList<String>();
-        knownFingerprintsByAllAuthorities = 0;
-      }
-
-      /* If we reached our end-of-list marker, stop here. */
-      if (longValue == Long.MAX_VALUE) {
-        break;
-      }
-
-      /* Look at the current indexes and increment one of the three
-       * counters.  If this value doesn't contain an attribute index, it
-       * was put in for counting all known fingerprints at this
-       * valid-after time. */
-      if (attributeIndex == 0) {
-        knownFingerprintsByAllAuthorities++;
-
-      /* Otherwise, if this value doesn't contain a fingerprint index, it
-       * was put in for counting authorities voting on a given attribute
-       * at the current valid-after time. */
-      } else if (fingerprintIndex == 0) {
-        authoritiesVotingOnAttribute++;
-
-      /* Otherwise, if both indexes are non-zero, this value was put in to
-       * count how many authorities assign the attribute to this relay at
-       * this valid-after time. */
-      } else {
-        votesForAttribute++;
-      }
-
-      /* Prepare moving on to the next value. */
-      lastLongValue = longValue;
-      lastValidAfterMillis = validAfterMillis;
-      lastAttributeIndex = attributeIndex;
-      lastFingerprintIndex = fingerprintIndex;
-    }
-  }
-}
-
diff --git a/shared/bin/20-run-disagreement-stats.sh b/shared/bin/20-run-disagreement-stats.sh
deleted file mode 100755
index 9abe1e2..0000000
--- a/shared/bin/20-run-disagreement-stats.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/sh
-cd modules/disagreement/
-ant | grep "\[java\]"
-cd ../../
-
diff --git a/shared/bin/99-copy-stats-files.sh b/shared/bin/99-copy-stats-files.sh
index 5ced875..504216a 100755
--- a/shared/bin/99-copy-stats-files.sh
+++ b/shared/bin/99-copy-stats-files.sh
@@ -6,5 +6,4 @@ cp -a modules/advbwdist/stats/advbwdist.csv shared/stats/
 cp -a modules/hidserv/stats/hidserv.csv shared/stats/
 cp -a modules/clients/stats/clients.csv shared/stats/
 cp -a modules/clients/stats/userstats-combined.csv shared/stats/
-cp -a modules/disagreement/stats/disagreement.csv shared/stats/
 
diff --git a/website/etc/metrics.json b/website/etc/metrics.json
index 64633ca..ffec652 100644
--- a/website/etc/metrics.json
+++ b/website/etc/metrics.json
@@ -870,13 +870,13 @@
   },
   {
     "id": "disagreement-data",
-    "title": "Disagreement among the directory authorities",
+    "title": "Disagreement among the directory authorities (deprecated)",
     "tags": [
       "Relays"
     ],
     "type": "Data",
     "level": "Advanced",
-    "description": "<p>The following data file contains statistics about agreement or disagreement among the <a href=\"about.html#directory-authority\">directory authorities</a>.  Once per hour the directory authorities exchange votes with their view of the <a href=\"about.html#relay\">relays</a> in the network including attributes like <a href=\"about.html#relay-flag\">relay flags</a> or bandwidth measurements.  This data file includes counts of relays by number of directory authorities assigning them a given attribute.</p>",
+    "description": "<p><font color=\"red\">As of September 29, 2016, the linked data file is not updated anymore.  This page and the linked data file will be removed in the future.</font></p><p>The following data file contains statistics about agreement or disagreement among the <a href=\"about.html#directory-authority\">directory authorities</a>.  Once per hour the directory authorities exchange votes with their view of the <a href=\"about.html#relay\">relays</a> in the network including attributes like <a href=\"about.html#relay-flag\">relay flags</a> or bandwidth measurements.  This data file includes counts of relays by number of directory authorities assigning them a given attribute.</p>",
     "data_file": "stats/disagreement.csv",
     "data_column_spec": [
       "<b>validafter:</b> UTC timestamp (YYYY-MM-DD HH:MM:SS) after which votes became valid, which may be used as the vote publication time.",



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits