[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Misc improvements



commit bfd84aa39d4323e4ee2262b8fef4caeab2ec14a0
Author: Arturo Filastò <hellais@xxxxxxxxx>
Date:   Tue Feb 21 10:15:29 2012 -0800

    Misc improvements
    
    * Rename ooni-probe to ooniprobe
    * Improve error collection
---
 refactor/ooni-probe.conf  |    1 +
 refactor/ooni-probe.py    |  142 ---------------------------------------------
 refactor/ooniprobe.py     |  142 +++++++++++++++++++++++++++++++++++++++++++++
 refactor/plugoo.py        |    9 +++-
 refactor/tests/bridget.py |   32 +++++++----
 refactor/utils.py         |   21 ++++++-
 6 files changed, 191 insertions(+), 156 deletions(-)

diff --git a/refactor/ooni-probe.conf b/refactor/ooni-probe.conf
index e74b241..8ea67e3 100644
--- a/refactor/ooni-probe.conf
+++ b/refactor/ooni-probe.conf
@@ -56,6 +56,7 @@ tor_bridges_timeout = 40
 
 [report]
 file = report.log
+timestamp = true
 #ssh = 127.0.0.1:22
 #ssh_user = theusername
 #ssh_password = thepassword
diff --git a/refactor/ooni-probe.py b/refactor/ooni-probe.py
deleted file mode 100644
index 64d35f5..0000000
--- a/refactor/ooni-probe.py
+++ /dev/null
@@ -1,142 +0,0 @@
-# -*- coding: UTF-8
-#!/usr/bin/env python
-"""
-    ooni-probe
-    **********
-
-    Open Observatory of Network Interference
-
-    "The Net interprets censorship as damage and routes around it."
-                    - John Gilmore; TIME magazine (6 December 1993)
-
-    The goal of ooni-probe is to collect data about censorship around
-    the world.
-
-    :copyright: (c) 2012 by Arturo Filastò
-    :license: see LICENSE for more details.
-"""
-
-import imp
-import os
-import sys
-
-from pprint import pprint
-
-import plugoo
-from utils import Storage, parse_asset, import_test, get_logger
-from config import Config
-
-class ooni(object):
-    """ooni-probe is a suite designed to run tests on your
-    network to detect censorship.
-    This is the main class that is used to start ooni probe
-    select the assets and run tests.
-    """
-    def __init__(self):
-        self.config = Storage()
-        self.config.main = Config("main")
-        self.config.tests = Config("tests")
-        self.config.report = Config("report")
-
-        self.logger = get_logger(self.config.main)
-
-        self.logger.info("Started ooni-probe")
-
-        self.assets = []
-        self.get_assets()
-
-        self.tests = Storage()
-        self.load_tests()
-
-        self.runtests = self.config.tests.run.split(",")
-
-
-    def get_assets(self):
-        """Parse all the assets in the asset directory.
-        Assets can optionaly contain the ooni-probe asset file
-        format: #:<something> <something_else>, that will then
-        be used to render the asset details to the user.
-        It is also possible to have an asset file link to multiple
-        other files.
-        """
-        for root, dir, files in os.walk(self.config.main.assetdir):
-            for name in files:
-                asset = os.path.join(root, name)
-                self.assets.append(parse_asset(asset))
-
-    def list_assets(self):
-        """Enumerate all the assets in the directory specified
-        in the config file
-        """
-        print "[-] There are a total of %s assets loaded" % len(self.assets)
-        for asset in self.assets:
-            print "    name: %s" % asset.name
-            if asset.desc:
-                print "    description: %s" % asset.desc
-            if asset.files:
-                print "    files: %s" % asset.files
-            if asset.tests:
-                print "    tests: %s" % asset.tests
-            print ""
-
-    def load_tests(self):
-        """Iterate through the plugoos insite the folder specified by the
-        config file and instantiate them.
-        """
-        pluginfiles = [fname[:-3] for fname in os.listdir(self.config.main.testdir)\
-                         if fname.endswith(".py")]
-        for fname in pluginfiles:
-            test = Storage()
-            test_name = fname
-            if not self.config.main.testdir in sys.path:
-                sys.path.insert(0, self.config.main.testdir)
-
-            module = __import__(fname)
-            try:
-                test.name = module.__plugoo__
-                test.desc = module.__desc__
-                test.module = module
-            except Exception, e:
-                self.logger.warning("Soft fail %s", e)
-                test.name = test_name
-                test.desc = ""
-                test.module = module
-
-            try:
-                self.tests[test_name] = test
-            except Exception, e:
-                print "Failed to load the test %s %s" % (name, e)
-
-    def list_tests(self):
-        """Print the loaded plugoonis to screen
-        """
-        print "[-] There are a total of %s tests available" % len(self.tests)
-        for name, test in self.tests.items():
-            print "    name: %s" % name
-            if test.name:
-                print "    long name: %s" % test.name
-            if test.desc:
-                print "    description: %s" % test.desc
-            print ""
-
-
-    def run_tests(self):
-        """Run all the tests that have been loaded
-        """
-        for name in self.runtests:
-            print "running %s" % name
-            try:
-                self.tests[name].module.run(self)
-            except Exception, e:
-                print "ERR: %s" % e
-
-    def run_test(self, test):
-        """Run a single test
-        """
-        self.tests[test].module.run(self)
-
-if __name__ == "__main__":
-    o = ooni()
-    o.list_tests()
-    o.run_test("bridget")
-
diff --git a/refactor/ooniprobe.py b/refactor/ooniprobe.py
new file mode 100644
index 0000000..64d35f5
--- /dev/null
+++ b/refactor/ooniprobe.py
@@ -0,0 +1,142 @@
+# -*- coding: UTF-8
+#!/usr/bin/env python
+"""
+    ooni-probe
+    **********
+
+    Open Observatory of Network Interference
+
+    "The Net interprets censorship as damage and routes around it."
+                    - John Gilmore; TIME magazine (6 December 1993)
+
+    The goal of ooni-probe is to collect data about censorship around
+    the world.
+
+    :copyright: (c) 2012 by Arturo Filastò
+    :license: see LICENSE for more details.
+"""
+
+import imp
+import os
+import sys
+
+from pprint import pprint
+
+import plugoo
+from utils import Storage, parse_asset, import_test, get_logger
+from config import Config
+
+class ooni(object):
+    """ooni-probe is a suite designed to run tests on your
+    network to detect censorship.
+    This is the main class that is used to start ooni probe
+    select the assets and run tests.
+    """
+    def __init__(self):
+        self.config = Storage()
+        self.config.main = Config("main")
+        self.config.tests = Config("tests")
+        self.config.report = Config("report")
+
+        self.logger = get_logger(self.config.main)
+
+        self.logger.info("Started ooni-probe")
+
+        self.assets = []
+        self.get_assets()
+
+        self.tests = Storage()
+        self.load_tests()
+
+        self.runtests = self.config.tests.run.split(",")
+
+
+    def get_assets(self):
+        """Parse all the assets in the asset directory.
+        Assets can optionaly contain the ooni-probe asset file
+        format: #:<something> <something_else>, that will then
+        be used to render the asset details to the user.
+        It is also possible to have an asset file link to multiple
+        other files.
+        """
+        for root, dir, files in os.walk(self.config.main.assetdir):
+            for name in files:
+                asset = os.path.join(root, name)
+                self.assets.append(parse_asset(asset))
+
+    def list_assets(self):
+        """Enumerate all the assets in the directory specified
+        in the config file
+        """
+        print "[-] There are a total of %s assets loaded" % len(self.assets)
+        for asset in self.assets:
+            print "    name: %s" % asset.name
+            if asset.desc:
+                print "    description: %s" % asset.desc
+            if asset.files:
+                print "    files: %s" % asset.files
+            if asset.tests:
+                print "    tests: %s" % asset.tests
+            print ""
+
+    def load_tests(self):
+        """Iterate through the plugoos insite the folder specified by the
+        config file and instantiate them.
+        """
+        pluginfiles = [fname[:-3] for fname in os.listdir(self.config.main.testdir)\
+                         if fname.endswith(".py")]
+        for fname in pluginfiles:
+            test = Storage()
+            test_name = fname
+            if not self.config.main.testdir in sys.path:
+                sys.path.insert(0, self.config.main.testdir)
+
+            module = __import__(fname)
+            try:
+                test.name = module.__plugoo__
+                test.desc = module.__desc__
+                test.module = module
+            except Exception, e:
+                self.logger.warning("Soft fail %s", e)
+                test.name = test_name
+                test.desc = ""
+                test.module = module
+
+            try:
+                self.tests[test_name] = test
+            except Exception, e:
+                print "Failed to load the test %s %s" % (name, e)
+
+    def list_tests(self):
+        """Print the loaded plugoonis to screen
+        """
+        print "[-] There are a total of %s tests available" % len(self.tests)
+        for name, test in self.tests.items():
+            print "    name: %s" % name
+            if test.name:
+                print "    long name: %s" % test.name
+            if test.desc:
+                print "    description: %s" % test.desc
+            print ""
+
+
+    def run_tests(self):
+        """Run all the tests that have been loaded
+        """
+        for name in self.runtests:
+            print "running %s" % name
+            try:
+                self.tests[name].module.run(self)
+            except Exception, e:
+                print "ERR: %s" % e
+
+    def run_test(self, test):
+        """Run a single test
+        """
+        self.tests[test].module.run(self)
+
+if __name__ == "__main__":
+    o = ooni()
+    o.list_tests()
+    o.run_test("bridget")
+
diff --git a/refactor/plugoo.py b/refactor/plugoo.py
index 49b3e16..0ce1d28 100644
--- a/refactor/plugoo.py
+++ b/refactor/plugoo.py
@@ -12,7 +12,7 @@
 """
 
 import os
-
+from datetime import datetime
 import yaml
 
 try:
@@ -95,6 +95,13 @@ class Report:
         self.config = ooni.config.report
         self.logger = ooni.logger
 
+        if self.config.timestamp:
+            tmp = self.file.split('.')
+            self.file = '.'.join(tmp[:-1]) + "-" + \
+                        datetime.now().isoformat('-') + '.' + \
+                        tmp[-1]
+            print self.file
+
         try:
             import paramiko
         except:
diff --git a/refactor/tests/bridget.py b/refactor/tests/bridget.py
index 7c1ca8b..1e18175 100644
--- a/refactor/tests/bridget.py
+++ b/refactor/tests/bridget.py
@@ -82,6 +82,8 @@ class BridgeT(Plugoo):
         datadir = "/tmp/" + randomname
         if bridge.startswith("obfs://"):
             obfsbridge = bridge.split("//")[1]
+
+            self.logger.debug("Genearting torrc file for obfs bridge")
             torrc = """SocksPort %s
 UseBridges 1
 Bridge obfs2 %s
@@ -90,6 +92,7 @@ ClientTransportPlugin obfs2 exec /usr/local/bin/obfsproxy --managed
 ControlPort %s
 """ % (socksport, obfsbridge, datadir, controlport)
         else:
+            self.logger.debug("Generating torrc file for bridge")
             torrc = """SocksPort %s
 UseBridges 1
 bridge %s
@@ -97,7 +100,8 @@ DataDirectory %s
 usemicrodescriptors 0
 ControlPort %s
 """ % (socksport, bridge, datadir, controlport)
-        print torrc
+        #print torrc
+
 
         with open(randomname, "wb") as f:
             f.write(torrc)
@@ -117,17 +121,15 @@ ControlPort %s
 
     #Can't use @torify as it doesn't support concurrency right now
     def download_file(self, socksport):
-        time_start=time.time()
-
         opener = urllib2.build_opener(SocksiPyHandler(socks.PROXY_TYPE_SOCKS5,
                                                       '127.0.0.1', int(socksport)))
-        f = opener.open('http://check.torproject.org')
-        data= f.readlines()
-        print data
-        print len(data)
+
+        time_start=time.time()
+        f = opener.open('http://38.229.72.16/bwauth.torproject.org/256k')
+        f.read()
         time_end = time.time()
         print (time_end-time_start)
-        return len(data)/(time_end-time_start)
+        return str(256/(time_end-time_start)) + " KB/s"
 
     def connect(self, bridge, timeout=None):
         if not timeout:
@@ -150,6 +152,7 @@ ControlPort %s
         except:
             self.logger.error("Unable to set file descriptor to non blocking")
 
+        self.logger.info("Testing bridge: %s" % bridge)
         while True:
             o = ""
             try:
@@ -158,15 +161,17 @@ ControlPort %s
                     self.logger.debug(str(o))
                 if re.search("100%", o):
                     self.logger.info("Success in connecting to %s" % bridge)
+
                     print "%s bridge works" % bridge
-                    print "%s controlport" % controlport
+                    # print "%s controlport" % controlport
                     try:
                         c = TorCtl.connect('127.0.0.1', controlport)
                         bridgeinfo = self.parsebridgeinfo(c.get_info('dir/server/all')['dir/server/all'])
-                        bandwidth=self.download_file(socksport)
+                        bandwidth = self.download_file(socksport)
                     except:
                         self.logger.error("Error in connecting to Tor Control port")
-                    print bandwidth
+
+                    self.logger.info("Bandwidth: %s" % bandwidth)
                     c.close()
                     p.stdout.close()
                     os.unlink(os.path.join(os.getcwd(), torrc))
@@ -182,6 +187,7 @@ ControlPort %s
 
                 if re.search("%", o):
                     # Keep updating the timeout if there is progress
+                    self.logger.debug("Updating time...")
                     tupdate = time.time()
                     #print o
                     continue
@@ -189,13 +195,17 @@ ControlPort %s
             except IOError:
                 ex = sys.exc_info()[1]
                 if ex[0] != errno.EAGAIN:
+                    self.logger.error("Error IOError: EAGAIN")
                     raise
                 sys.exc_clear()
+
             try:
                 # Set the timeout for the socket wait
                 ct = timeout-(time.time() - tupdate)
                 socket.wait_read(p.stdout.fileno(), timeout=ct)
+
             except:
+                self.logger.info("%s bridge does not work (%s s timeout)" % (bridge, timeout))
                 print "%s bridge does not work (%s s timeout)" % (bridge, timeout)
                 self.failures.append(bridge)
                 p.stdout.close()
diff --git a/refactor/utils.py b/refactor/utils.py
index 5c743ba..93bd2fd 100644
--- a/refactor/utils.py
+++ b/refactor/utils.py
@@ -63,7 +63,7 @@ def get_logger(config):
     console.setFormatter(formatter)
     logging.getLogger('').addHandler(console)
 
-    return logging.getLogger('ooni-probe')
+    return logging.getLogger('ooniprobe')
 
 def parse_asset(asset):
     parsed = Storage()
@@ -80,7 +80,7 @@ def parse_asset(asset):
                         parsed[n] = v.split(",")
                     else:
                         parsed[n] = v
-                        
+
                 elif line.startswith("#"):
                     continue
                 else:
@@ -113,3 +113,20 @@ def import_test(name, config):
 
     return None, None
 
+class Log():
+    def __init__(self, file=None):
+        if file:
+            self.fh = open(file)
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        try:
+            line = self.fh.readline()
+            while not line.startswith("---"):
+                line = self.fh.readline()
+        except:
+            raise StopIteration
+
+

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits