[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [bridgedb/develop] Refactor Main.load() to use new Stem-based parsers.



commit 2fb7f89751bd853fc0ff1c30f752064c12dc852b
Author: Isis Lovecruft <isis@xxxxxxxxxxxxxx>
Date:   Wed Dec 24 09:14:49 2014 +0000

    Refactor Main.load() to use new Stem-based parsers.
---
 lib/bridgedb/Main.py |  156 ++++++++++++++++++++++++--------------------------
 1 file changed, 76 insertions(+), 80 deletions(-)

diff --git a/lib/bridgedb/Main.py b/lib/bridgedb/Main.py
index 47a869c..7bc5cff 100644
--- a/lib/bridgedb/Main.py
+++ b/lib/bridgedb/Main.py
@@ -28,10 +28,16 @@ from bridgedb import safelog
 from bridgedb import schedule
 from bridgedb import util
 from bridgedb.bridges import InvalidPluggableTransportIP
+from bridgedb.bridges import MalformedBridgeInfo
 from bridgedb.bridges import MalformedPluggableTransport
+from bridgedb.bridges import MissingServerDescriptorDigest
+from bridgedb.bridges import ServerDescriptorDigestMismatch
+from bridgedb.bridges import ServerDescriptorWithoutNetworkstatus
 from bridgedb.bridges import PluggableTransport
+from bridgedb.bridges import Bridge
 from bridgedb.configure import loadConfig
 from bridgedb.configure import Conf
+from bridgedb.parse import descriptors
 from bridgedb.parse import options
 from bridgedb.parse.addr import isIPAddress
 from bridgedb.schedule import toUnixSeconds
@@ -103,91 +109,72 @@ def load(state, splitter, clear=False):
     logging.info("Loading bridges...")
 
     bridges = {}
-    status = {}
-    addresses = {}
     timestamps = {}
-    bridges = {}
-    desc_digests = {}
-    ei_digests = {}
-
-    logging.info("Opening network status file: %s" % state.STATUS_FILE)
-    f = open(state.STATUS_FILE, 'r')
-    for (ID, nickname, desc_digest, running, stable,
-         ORaddr, ORport, or_addresses,
-         timestamp) in Bridges.parseStatusFile(f):
-        bridge = Bridges.Bridge(nickname, ORaddr, ORport, id_digest=ID,
-                                or_addresses=or_addresses)
-        bridge.assertOK()
-        bridge.setStatus(running, stable)
-        bridge.setDescriptorDigest(desc_digest)
-        bridges[ID] = bridge
-
-        if ID in timestamps.keys():
-            timestamps[ID].append(timestamp)
+
+    logging.info("Opening networkstatus file: %s" % state.STATUS_FILE)
+    networkstatuses = descriptors.parseNetworkStatusFile(state.STATUS_FILE)
+    logging.debug("Closing networkstatus file: %s" % state.STATUS_FILE)
+
+    logging.info("Processing networkstatus descriptors...")
+    for router in networkstatuses:
+        bridge = Bridge()
+        bridge.updateFromNetworkStatus(router)
+        bridge.flags.update(router.flags)
+
+        try:
+            bridge.assertOK()
+        except MalformedBridgeInfo as error:
+            logging.warn(str(error))
         else:
-            timestamps[ID] = [timestamp]
-    logging.debug("Closing network status file")
-    f.close()
-
-    for fname in state.BRIDGE_FILES:
-        logging.info("Opening bridge-server-descriptor file: '%s'" % fname)
-        f = open(fname, 'r')
-        desc_digests.update(Bridges.getDescriptorDigests(f))
-        if state.COLLECT_TIMESTAMPS:
-            for bridge in bridges.values():
-                if bridge.getID() in timestamps.keys():
-                    ts = timestamps[bridge.getID()][:]
-                    ts.sort()
-                    for timestamp in ts:
-                        logging.debug(
-                           "Adding/updating timestamps in BridgeHistory for "\
-                           "'%s' in database: %s"
-                           % (bridge.fingerprint, timestamp))
-                        bridgedb.Stability.addOrUpdateBridgeHistory(
-                           bridge, timestamp)
-        logging.debug("Closing bridge-server-descriptor file: '%s'" % fname)
-        f.close()
+            bridges[bridge.fingerprint] = bridge
 
-    for ID in bridges.keys():
-        bridge = bridges[ID]
-        if bridge.desc_digest in desc_digests:
-            bridge.setVerified()
-            bridge.setExtraInfoDigest(desc_digests[bridge.desc_digest])
-        # We attempt to insert all bridges. If the bridge is not
-        # running, then it is skipped during the insertion process.
-        splitter.insert(bridge)
+    for filename in state.BRIDGE_FILES:
+        logging.info("Opening bridge-server-descriptor file: '%s'" % filename)
+        serverdescriptors = descriptors.parseServerDescriptorsFile(filename)
+        logging.debug("Closing bridge-server-descriptor file: '%s'" % filename)
 
-    # read pluggable transports from extra-info document
-    # XXX: should read from networkstatus after bridge-authority
-    # does a reachability test
-    for filename in state.EXTRA_INFO_FILES:
-        logging.info("Opening extra-info file: '%s'" % filename)
-        f = open(filename, 'r')
-        for transport in Bridges.parseExtraInfoFile(f):
-            ID, method_name, address, port, argdict = transport
+        for router in serverdescriptors:
             try:
-                if bridges[ID].running:
-                    logging.info("Adding %s transport to running bridge"
-                                 % method_name)
-                    try:
-                        bridgePT = PluggableTransport(Bridges.toHex(ID),
-                                                      method_name, address,
-                                                      port, argdict)
-                    except (InvalidPluggableTransportIP,
-                            MalformedPluggableTransport) as error:
-                        logging.warn(error)
-                    else:
-                        bridges[ID].transports.append(bridgePT)
-                        if not bridgePT in bridges[ID].transports:
-                            logging.critical(
-                                "Added a transport, but it disappeared!",
-                                "\tTransport: %r" % bridgePT)
-            except KeyError as error:
-                logging.error("Could not find bridge with fingerprint '%s'."
-                              % Bridges.toHex(ID))
-        logging.debug("Closing extra-info file: '%s'" % filename)
-        f.close()
-
+                bridges[router.fingerprint].updateFromServerDescriptor(router)
+            except KeyError:
+                logging.warn(
+                    ("Received server descriptor for bridge '%s' which wasn't "
+                     "in the networkstatus!") % router.fingerprint)
+                continue
+            except (ServerDescriptorWithoutNetworkstatus,
+                    MissingServerDescriptorDigest,
+                    ServerDescriptorDigestMismatch) as error:
+                logging.warn(str(error))
+                # Reject any routers whose server descriptors didn't pass
+                # :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those
+                # bridges who don't have corresponding networkstatus
+                # documents, or whose server descriptor digests don't check
+                # out:
+                bridges.pop(router.fingerprint)
+                continue
+
+            if state.COLLECT_TIMESTAMPS:
+                # Update timestamps from server descriptors, not from network
+                # status descriptors (because networkstatus documents and
+                # descriptors aren't authenticated in any way):
+                if bridge.fingerprint in timestamps.keys():
+                    timestamps[bridge.fingerprint].append(router.published)
+                else:
+                    timestamps[bridge.fingerprint] = [router.published]
+
+    extrainfos = descriptors.parseExtraInfoFiles(*state.EXTRA_INFO_FILES)
+    for fingerprint, router in extrainfos.items():
+        try:
+            bridges[fingerprint].updateFromExtraInfoDescriptor(router)
+        except MalformedBridgeInfo as error:
+            logging.warn(str(error))
+        except KeyError as error:
+            logging.warn(("Received extrainfo descriptor for bridge '%s', "
+                          "but could not find bridge with that fingerprint.")
+                         % router.fingerprint)
+
+    # XXX TODO refactor the next block according with new parsers for OONI
+    # bridge-reachability reports:
     if state.COUNTRY_BLOCK_FILE:
         logging.info("Opening Blocking Countries file %s"
                      % state.COUNTRY_BLOCK_FILE)
@@ -206,6 +193,15 @@ def load(state, splitter, clear=False):
         logging.debug("Closing blocking-countries document")
         f.close()
 
+    inserted = 0
+    logging.info("Inserting %d bridges into splitter..." % len(bridges))
+    for fingerprint, bridge in bridges.items():
+        # We attempt to insert all bridges. If the bridge is not running, then
+        # it is skipped during the insertion process.
+        splitter.insert(bridge)
+        inserted += 1
+    logging.info("Done inserting %d bridges into splitter." % inserted)
+
     if state.COLLECT_TIMESTAMPS:
         reactor.callInThread(updateBridgeHistory, bridges, timestamps)
 



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits