[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [sbws/maint-1.1] chg: relaylist, v3bwfile: Count consensus with timestamps



commit e098ebb9540144fbd04854f05ea3cf958eb006bf
Author: juga0 <juga@xxxxxxxxxx>
Date:   Sat Mar 21 13:42:18 2020 +0000

    chg: relaylist, v3bwfile: Count consensus with timestamps
    
    in RelayList:
    - Rename consensus_timestamps to recent_consensus
    - Rename recent_consensus_count to recent_consensus when there is
      no counting
    - Use the timestamps class to manage/count consensuses
    - Remove method not needed anymore
---
 sbws/globals.py                  |  4 ++++
 sbws/lib/relaylist.py            | 35 +++++++++++----------------------
 sbws/lib/v3bwfile.py             | 10 +++++-----
 tests/data/.sbws/state.dat       | 42 ++++++++++++++++++++++++++++++++--------
 tests/unit/lib/test_relaylist.py | 10 ++++++----
 tests/unit/lib/test_v3bwfile.py  | 17 +++++++++++++---
 6 files changed, 74 insertions(+), 44 deletions(-)

diff --git a/sbws/globals.py b/sbws/globals.py
index a71fbc6..a9dca10 100644
--- a/sbws/globals.py
+++ b/sbws/globals.py
@@ -148,6 +148,10 @@ MAX_NUM_DESTINATION_FAILURES = 3
 # destination fail again.
 FACTOR_INCREMENT_DESTINATION_RETRY = 2
 
+# Constants to check health KeyValues in the bandwidth file
+PERIOD_DAYS = int(MEASUREMENTS_PERIOD / (24 * 60 * 60))
+MAX_RECENT_CONSENSUS_COUNT = PERIOD_DAYS * 24  # 120
+
 
 def fail_hard(*a, **kw):
     ''' Log something ... and then exit as fast as possible '''
diff --git a/sbws/lib/relaylist.py b/sbws/lib/relaylist.py
index 9ffff49..5ae1c2d 100644
--- a/sbws/lib/relaylist.py
+++ b/sbws/lib/relaylist.py
@@ -8,8 +8,11 @@ import random
 import logging
 from threading import Lock
 
-from ..globals import MEASUREMENTS_PERIOD
-from ..util import timestamp
+from ..globals import (
+    MAX_RECENT_CONSENSUS_COUNT,
+    MEASUREMENTS_PERIOD
+)
+from ..util import timestamp, timestamps
 
 log = logging.getLogger(__name__)
 
@@ -322,7 +325,9 @@ class RelayList:
         self.rng = random.SystemRandom()
         self._refresh_lock = Lock()
         # To track all the consensus seen.
-        self._consensus_timestamps = []
+        self._recent_consensus = timestamps.DateTimeSeq(
+            [], MAX_RECENT_CONSENSUS_COUNT, state, "recent_consensus"
+        )
         # Initialize so that there's no error trying to access to it.
         # In future refactor, change to a dictionary, where the keys are
         # the relays' fingerprint.
@@ -345,15 +350,7 @@ class RelayList:
     @property
     def last_consensus_timestamp(self):
         """Returns the datetime when the last consensus was obtained."""
-        if (getattr(self, "_consensus_timestamps")
-                and self._consensus_timestamps):
-            return self._consensus_timestamps[-1]
-        # If the object was not created from __init__, it won't have
-        # consensus_timestamps attribute or it might be empty.
-        # In this case force new update.
-        # Anytime more than 1h in the past will be old.
-        self._consensus_timestamps = []
-        return datetime.utcnow() - timedelta(seconds=60*61)
+        return self._recent_consensus.last()
 
     @property
     def relays(self):
@@ -419,12 +416,6 @@ class RelayList:
     def _relays_without_flag(self, flag):
         return [r for r in self.relays if flag not in r.flags]
 
-    def _remove_old_consensus_timestamps(self):
-        self._consensus_timestamps = remove_old_consensus_timestamps(
-            copy.deepcopy(self._consensus_timestamps),
-            self._measurements_period
-            )
-
     def _init_relays(self):
         """Returns a new list of relays that are in the current consensus.
         And update the consensus timestamp list with the current one.
@@ -441,8 +432,7 @@ class RelayList:
 
         # Find the timestamp of the last consensus.
         timestamp = valid_after_from_network_statuses(network_statuses)
-        self._consensus_timestamps.append(timestamp)
-        self._remove_old_consensus_timestamps()
+        self._recent_consensus.update(timestamp)
 
         new_relays = []
 
@@ -502,14 +492,11 @@ class RelayList:
         log.info("Number of consensuses obtained in the last %s days: %s.",
                  int(self._measurements_period / 24 / 60 / 60),
                  self.recent_consensus_count)
-        # NOTE: blocking, writes to file!
-        if self._state is not None:
-            self._state['recent_consensus_count'] = self.recent_consensus_count
 
     @property
     def recent_consensus_count(self):
         """Number of times a new consensus was obtained."""
-        return len(self._consensus_timestamps)
+        return len(self._recent_consensus)
 
     def exits_not_bad_allowing_port(self, port):
         return [r for r in self.exits
diff --git a/sbws/lib/v3bwfile.py b/sbws/lib/v3bwfile.py
index 80f1e34..70304a9 100644
--- a/sbws/lib/v3bwfile.py
+++ b/sbws/lib/v3bwfile.py
@@ -367,7 +367,7 @@ class V3BWHeader(object):
         if destinations_countries is not None:
             kwargs['destinations_countries'] = destinations_countries
         if recent_consensus_count is not None:
-            kwargs['recent_consensus_count'] = str(recent_consensus_count)
+            kwargs['recent_consensus_count'] = recent_consensus_count
 
         recent_measurement_attempt_count = \
             cls.recent_measurement_attempt_count_from_file(state_fpath)
@@ -457,10 +457,10 @@ class V3BWHeader(object):
     @staticmethod
     def consensus_count_from_file(state_fpath):
         state = State(state_fpath)
-        if 'recent_consensus_count' in state:
-            return state['recent_consensus_count']
-        else:
-            return None
+        count = state.count("recent_consensus")
+        if count:
+            return str(count)
+        return None
 
     # NOTE: in future refactor store state in the class
     @staticmethod
diff --git a/tests/data/.sbws/state.dat b/tests/data/.sbws/state.dat
index 38acf6c..8793d73 100644
--- a/tests/data/.sbws/state.dat
+++ b/tests/data/.sbws/state.dat
@@ -1,9 +1,35 @@
 {
-    "uuid": "806218a0-3ce5-4778-b839-d6faf6798405",
-    "scanner_started": "2019-03-25T13:03:06",
-    "recent_consensus_count": 1,
-    "recent_priority_list_count": 1,
-    "recent_measurement_attempt_count": 15,
-    "min_perc_reached": null,
-    "recent_priority_relay_count": 15
-}
\ No newline at end of file
+    "scanner_started": "2020-02-29T10:00:00",
+    "uuid": "e4ecf294-f253-478c-8660-28cbdfc690de",
+    "tor_version": "0.4.2.6",
+    "recent_consensus": [
+        "2020-03-18T13:26:46"
+    ],
+    "recent_priority_list": [
+        "2020-03-18T13:26:46"
+    ],
+    "recent_priority_relay": [
+        [
+            "2020-03-18T13:26:46",
+            15
+        ]
+    ],
+    "recent_measurement_attempt": [
+        "2020-03-18T13:26:46",
+        "2020-03-18T13:26:46",
+        "2020-03-18T13:26:46",
+        "2020-03-18T13:26:46",
+        "2020-03-18T13:26:46",
+        "2020-03-18T13:26:46",
+        "2020-03-18T13:26:46",
+        "2020-03-18T13:26:46",
+        "2020-03-18T13:26:46",
+        "2020-03-18T13:26:46",
+        "2020-03-18T13:26:46",
+        "2020-03-18T13:26:46",
+        "2020-03-18T13:26:46",
+        "2020-03-18T13:26:46",
+        "2020-03-18T13:26:46"
+    ],
+    "min_perc_reached": null
+}
diff --git a/tests/unit/lib/test_relaylist.py b/tests/unit/lib/test_relaylist.py
index 846677f..1b22425 100644
--- a/tests/unit/lib/test_relaylist.py
+++ b/tests/unit/lib/test_relaylist.py
@@ -8,6 +8,7 @@ from datetime import datetime, timedelta
 from freezegun import freeze_time
 
 from sbws.lib.relaylist import RelayList, remove_old_consensus_timestamps
+from sbws.util.state import State
 
 
 def test_remove_old_consensus_timestamps():
@@ -31,13 +32,14 @@ def test_init_relays(
     new consensus is received.
     Test that the number of consesus timesamps and relays is correct.
     """
+    state = State(conf['paths']['state_fpath'])
     # There is no need to mock datetime to update the consensus, since the
     # actual date will be always later.
     # But it's needed to have the correct list of timestamps both for RelayList
     # and Relay.
     with freeze_time("2020-02-29 10:00:00"):
-        relay_list = RelayList(args, conf, controller=controller)
-    assert len(relay_list._consensus_timestamps) == 1
+        relay_list = RelayList(args, conf, controller, state=state)
+    assert relay_list.recent_consensus_count == 1
     assert len(relay_list._relays[0]._consensus_timestamps) == 1
     # The actual number of relays in the consensus
     assert len(relay_list._relays) == 6433
@@ -48,7 +50,7 @@ def test_init_relays(
     with freeze_time("2020-02-29 11:00:00"):
         # Call relays update the list of relays.
         relay_list.relays
-    assert len(relay_list._consensus_timestamps) == 2
+    assert relay_list.recent_consensus_count == 2
     assert len(relay_list._relays[0]._consensus_timestamps) == 2
     # Check that the number of relays is now the previous one plus the relays
     # that are in the new consensus that there were not in the previous one.
@@ -61,7 +63,7 @@ def test_init_relays(
     relay_list._controller = controller_5days_later
     with freeze_time("2020-03-05 10:00:01"):
         relay_list.relays
-    assert len(relay_list._consensus_timestamps) == 2
+    assert relay_list.recent_consensus_count == 2
     assert len(relay_list._relays[0]._consensus_timestamps) == 2
     fps_5days_later = {r.fingerprint for r in relay_list._relays}
     # The number of added relays will be the number of relays in this
diff --git a/tests/unit/lib/test_v3bwfile.py b/tests/unit/lib/test_v3bwfile.py
index 8fe2dab..c9c0e93 100644
--- a/tests/unit/lib/test_v3bwfile.py
+++ b/tests/unit/lib/test_v3bwfile.py
@@ -16,6 +16,7 @@ from sbws.lib.v3bwfile import (
     V3BWFile, round_sig_dig,
     HEADER_RECENT_MEASUREMENTS_EXCLUDED_KEYS
     )
+from sbws.util.state import CustomDecoder
 from sbws.util.timestamp import now_fname, now_isodt_str, now_unixts
 
 timestamp = 1523974147
@@ -248,7 +249,7 @@ def test_v3bwline_from_results_file(datadir):
     lines = datadir.readlines('results.txt')
     d = dict()
     for line in lines:
-        r = Result.from_dict(json.loads(line.strip()))
+        r = Result.from_dict(json.loads(line.strip(), cls=CustomDecoder))
         fp = r.fingerprint
         if fp not in d:
             d[fp] = []
@@ -521,11 +522,21 @@ def test_set_under_min_report(mock_consensus, conf, datadir):
 def test_generator_started(root_data_path, datadir):
     state_fpath = os.path.join(root_data_path, '.sbws/state.dat')
     # The method is correct
-    assert "2019-03-25T13:03:06" == V3BWHeader.generator_started_from_file(
+    assert "2020-02-29T10:00:00" == V3BWHeader.generator_started_from_file(
         state_fpath
     )
     # `results` does not matter here, using them to not have an empty list.
     results = load_result_file(str(datadir.join("results.txt")))
     header = V3BWHeader.from_results(results, '', '', state_fpath)
     # And the header is correct
-    assert "2019-03-25T13:03:06" == header.generator_started
+    assert "2020-02-29T10:00:00" == header.generator_started
+
+
+def test_recent_consensus_count(root_data_path, datadir):
+    # This state has recent_consensus_count
+    state_fpath = os.path.join(root_data_path, '.sbws/state.dat')
+    assert "1" == V3BWHeader.consensus_count_from_file(state_fpath)
+    # `results` does not matter here, using them to not have an empty list.
+    results = load_result_file(str(datadir.join("results.txt")))
+    header = V3BWHeader.from_results(results, '', '', state_fpath)
+    assert "1" == header.recent_consensus_count



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits