[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Fixed a couple things:



commit 4f4cf2e0e2adfb8ae3d71fb29ac413d9e4c81ae7
Author: kudrom <kudrom@xxxxxxxxxx>
Date:   Sat Jul 19 18:09:12 2014 +0200

    Fixed a couple things:
      * sniffer subsystem to allow parallel recording of pcaps
      * keyword_filtering nettest
      * improved some tests in test_oonibclient
---
 .gitignore                                      |    3 +-
 data/ooniprobe.1                                |    2 +-
 data/ooniprobe.conf.sample                      |    2 +-
 ooni/director.py                                |   56 +++++++++++++----------
 ooni/nettests/experimental/keyword_filtering.py |   31 +++++++------
 ooni/oonicli.py                                 |    4 ++
 ooni/settings.py                                |    5 --
 ooni/templates/scapyt.py                        |    8 +---
 ooni/tests/test_oonibclient.py                  |   18 ++------
 ooni/utils/txscapy.py                           |   21 +++------
 10 files changed, 68 insertions(+), 82 deletions(-)

diff --git a/.gitignore b/.gitignore
index 9632014..1e2dba3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,6 +29,7 @@ private/*
 ooni/report*.yaml
 report*.yaml
 *.yamloo
+*.pcap
 pcaps/
 
 ooniprobe.conf
@@ -41,4 +42,4 @@ docs/build/*
 .vagrant/*
 
 cover/*
-ooni_home/*
\ No newline at end of file
+ooni_home/*
diff --git a/data/ooniprobe.1 b/data/ooniprobe.1
index cb5309b..7da9ae6 100644
--- a/data/ooniprobe.1
+++ b/data/ooniprobe.1
@@ -95,7 +95,7 @@ the addresses of test helpers. default: httpo://nkvphnp3p6agi5qq.onion
 Path to the log file to write
 .TP
 .BR \-\^O " or " \-\-pcapfile
-Path to the pcap file name.
+Prefix to the pcap file name.
 .TP
 .BR \-\^f " or " \-\-configfile
 Specify a path to the ooniprobe configuration file.
diff --git a/data/ooniprobe.conf.sample b/data/ooniprobe.conf.sample
index d32248f..b08edf6 100644
--- a/data/ooniprobe.conf.sample
+++ b/data/ooniprobe.conf.sample
@@ -17,7 +17,7 @@ privacy:
     # Should we collect a full packet capture on the client?
     includepcap: false
 reports:
-    # This is a packet capture file (.pcap) to load as a test:
+    # This is a prefix for each packet capture file (.pcap) per test:
     pcap: null
     collector: null
 advanced:
diff --git a/ooni/director.py b/ooni/director.py
index 901088f..3634115 100644
--- a/ooni/director.py
+++ b/ooni/director.py
@@ -1,4 +1,6 @@
 import os
+import otime
+from psutil import Process
 
 from ooni.managers import ReportEntryManager, MeasurementManager
 from ooni.reporter import Report
@@ -15,7 +17,6 @@ from twisted.internet.endpoints import TCP4ClientEndpoint
 
 
 class Director(object):
-
     """
     Singleton object responsible for coordinating the Measurements Manager
     and the Reporting Manager.
@@ -90,14 +91,13 @@ class Director(object):
         # This deferred is fired once all the measurements and their reporting
         # tasks are completed.
         self.allTestsDone = defer.Deferred()
-        self.sniffer = None
+        self.sniffers = {}
 
     def getNetTests(self):
         nettests = {}
 
         def is_nettest(filename):
-            return not filename == '__init__.py' \
-                and filename.endswith('.py')
+            return not filename == '__init__.py' and filename.endswith('.py')
 
         for category in self.categories:
             dirname = os.path.join(config.nettest_directory, category)
@@ -191,6 +191,11 @@ class Director(object):
         self.totalMeasurementRuntime += measurement.runtime
         self.successfulMeasurements += 1
         measurement.result = result
+        test_name = measurement.testInstance.__class__.__name__
+        sniffer = self.sniffers[test_name]
+        config.scapyFactory.unRegisterProtocol(sniffer)
+        sniffer.close()
+        del self.sniffers[test_name]
         return measurement
 
     def measurementFailed(self, failure, measurement):
@@ -229,16 +234,11 @@ class Director(object):
             net_test_loader:
                 an instance of :class:ooni.nettest.NetTestLoader
         """
-
         if self.allTestsDone.called:
             self.allTestsDone = defer.Deferred()
 
         if config.privacy.includepcap:
-            if not config.reports.pcap:
-                config.reports.pcap = config.generate_pcap_filename(
-                    net_test_loader.testDetails
-                )
-            self.startSniffing()
+            self.startSniffing(net_test_loader.testDetails)
 
         report = Report(net_test_loader.testDetails, report_filename,
                         self.reportEntryManager, collector_address)
@@ -258,24 +258,30 @@ class Director(object):
         finally:
             self.netTestDone(net_test)
 
-    def startSniffing(self):
+    def startSniffing(self, testDetails):
         """ Start sniffing with Scapy. Exits if required privileges (root) are not
         available.
         """
-        from ooni.utils.txscapy import ScapyFactory, ScapySniffer
-        config.scapyFactory = ScapyFactory(config.advanced.interface)
-
-        if os.path.exists(config.reports.pcap):
-            log.msg("Report PCAP already exists with filename %s" %
-                    config.reports.pcap)
-            log.msg("Renaming files with such name...")
-            pushFilenameStack(config.reports.pcap)
-
-        if self.sniffer:
-            config.scapyFactory.unRegisterProtocol(self.sniffer)
-        self.sniffer = ScapySniffer(config.reports.pcap)
-        config.scapyFactory.registerProtocol(self.sniffer)
-        log.msg("Starting packet capture to: %s" % config.reports.pcap)
+        from ooni.utils.txscapy import ScapySniffer
+
+        test_name, start_time = testDetails['test_name'], testDetails['start_time']
+        start_time = otime.epochToTimestamp(start_time)
+        suffix = "%s-%s.%s" % (test_name, start_time, "pcap")
+        if not config.reports.pcap:
+            filename_pcap= "%s-%s" % ("report", suffix)
+        else:
+            filename_pcap = "%s-%s" % (config.reports.pcap, suffix)
+
+        if len(self.sniffers) > 1:
+            pcap_filenames = set(sniffer.pcapwriter.filename for sniffer in self.sniffers)
+            pcap_filenames.add(filename_pcap)
+            log.msg("pcap files %s can be messed up because several netTests are being executed in parallel." %
+                    ','.join(pcap_filenames))
+
+        sniffer = ScapySniffer(filename_pcap)
+        self.sniffers[testDetails['test_name']] = sniffer
+        config.scapyFactory.registerProtocol(sniffer)
+        log.msg("Starting packet capture to: %s" % filename_pcap)
 
     @defer.inlineCallbacks
     def getTorState(self):
diff --git a/ooni/nettests/experimental/keyword_filtering.py b/ooni/nettests/experimental/keyword_filtering.py
index 0937efd..b54ba0d 100644
--- a/ooni/nettests/experimental/keyword_filtering.py
+++ b/ooni/nettests/experimental/keyword_filtering.py
@@ -9,23 +9,26 @@ from twisted.internet import defer
 from ooni.utils import log
 from ooni.templates import scapyt
 
-from scapy.all import *
+from scapy.layers.inet import TCP, IP
+from scapy.volatile import RandShort
+
 
 class UsageOptions(usage.Options):
     optParameters = [
-                    ['backend', 'b', '127.0.0.1:57002', 'Test backend running TCP echo'],
-                    ['timeout', 't', 5, 'Timeout after which to give up waiting for RST packets']
-                    ]
+        ['backend', 'b', '127.0.0.1:57002', 'Test backend running TCP echo'],
+        ['timeout', 't', 5, 'Timeout after which to give up waiting for RST packets']
+    ]
+
 
 class KeywordFiltering(scapyt.BaseScapyTest):
     name = "Keyword Filtering detection based on RST packets"
     author = "Arturo Filastò"
-    version = "0.1"
+    version = "0.2"
 
     usageOptions = UsageOptions
 
-    inputFile = ['file', 'f', None, 
-            'List of keywords to use for censorship testing']
+    inputFile = ['file', 'f', None,
+                 'List of keywords to use for censorship testing']
     requiresRoot = True
     requiresTor = False
 
@@ -36,19 +39,19 @@ class KeywordFiltering(scapyt.BaseScapyTest):
             though this should not be an issue since we are testing all 
             the keywords in parallel.
         """
+        backend_ip, backend_port = self.localOptions['backend'].split(':')
+        timeout = int(self.localOptions['timeout'])
+        keyword_to_test = str(self.input)
+        packets = IP(dst=backend_ip, id=RandShort()) / TCP(sport=4000, dport=int(backend_port)) / keyword_to_test
+        d = self.sr(packets, timeout=timeout)
+
+        @d.addCallback
         def finished(packets):
-            log.debug("Finished running TCP traceroute test on port %s" % port)
             answered, unanswered = packets
             self.report['rst_packets'] = []
             for snd, rcv in answered:
                 # The received packet has the RST flag
                 if rcv[TCP].flags == 4:
                     self.report['rst_packets'].append(rcv)
-
-        backend_ip, backend_port = self.localOptions['backend']
-        keyword_to_test = str(self.input)
-        packets = IP(dst=backend_ip,id=RandShort())/TCP(dport=backend_port)/keyword_to_test
-        d = self.sr(packets, timeout=timeout)
-        d.addCallback(finished)
         return d
 
diff --git a/ooni/oonicli.py b/ooni/oonicli.py
index 8313822..7c3e9a2 100644
--- a/ooni/oonicli.py
+++ b/ooni/oonicli.py
@@ -12,6 +12,7 @@ from ooni.settings import config
 from ooni.director import Director
 from ooni.deck import Deck, nettest_to_path
 from ooni.nettest import NetTestLoader
+from ooni.utils.txscapy import ScapyFactory
 
 from ooni.utils import log, checkForRoot
 
@@ -111,14 +112,17 @@ def runWithDirector(logging=True, start_tor=True):
     config.set_paths()
     config.initialize_ooni_home()
     config.read_config_file()
+
     if global_options['verbose']:
         config.advanced.debug = True
+
     if not start_tor:
         config.advanced.start_tor = False
 
     if logging:
         log.start(global_options['logfile'])
 
+    config.scapyFactory = ScapyFactory(config.advanced.interface)
     if config.privacy.includepcap:
         try:
             checkForRoot()
diff --git a/ooni/settings.py b/ooni/settings.py
index 0d1275e..5bebc05 100644
--- a/ooni/settings.py
+++ b/ooni/settings.py
@@ -113,9 +113,4 @@ class OConfig(object):
                     pass
         self.set_paths()
 
-    def generate_pcap_filename(self, testDetails):
-        test_name, start_time = testDetails['test_name'], testDetails['start_time']
-        start_time = otime.epochToTimestamp(start_time)
-        return "report-%s-%s.%s" % (test_name, start_time, "pcap")
-
 config = OConfig()
diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py
index ee21508..d99ea3f 100644
--- a/ooni/templates/scapyt.py
+++ b/ooni/templates/scapyt.py
@@ -41,10 +41,6 @@ class BaseScapyTest(NetTestCase):
     def _setUp(self):
         super(BaseScapyTest, self)._setUp()
 
-        if not config.scapyFactory:
-            log.debug("Scapy factory not set, registering it.")
-            config.scapyFactory = ScapyFactory(config.advanced.interface)
-
         self.report['answer_flags'] = []
         if self.localOptions['ipsrc']:
             config.checkIPsrc = 0
@@ -93,12 +89,12 @@ class BaseScapyTest(NetTestCase):
             self.report['answered_packets'].append(received_packet)
         return packets
 
-    def sr(self, packets, *arg, **kw):
+    def sr(self, packets, timeout=None, *arg, **kw):
         """
         Wrapper around scapy.sendrecv.sr for sending and receiving of packets
         at layer 3.
         """
-        scapySender = ScapySender()
+        scapySender = ScapySender(timeout=timeout)
 
         config.scapyFactory.registerProtocol(scapySender)
         log.debug("Using sending with hash %s" % scapySender.__hash__)
diff --git a/ooni/tests/test_oonibclient.py b/ooni/tests/test_oonibclient.py
index d80b606..87ea8da 100644
--- a/ooni/tests/test_oonibclient.py
+++ b/ooni/tests/test_oonibclient.py
@@ -28,10 +28,8 @@ class TestOONIBClient(ConfigTestCase):
             data_dir = '/tmp/testooni'
             config.advanced.data_dir = data_dir
 
-            try:
+            if os.path.exists(data_dir):
                 shutil.rmtree(data_dir)
-            except:
-                pass
             os.mkdir(data_dir)
             os.mkdir(os.path.join(data_dir, 'inputs'))
             os.mkdir(os.path.join(data_dir, 'decks'))
@@ -101,21 +99,11 @@ class TestOONIBClient(ConfigTestCase):
 
     @defer.inlineCallbacks
     def test_input_descriptor_not_found(self):
-        try:
-            yield self.oonibclient.queryBackend('GET', '/input/' + 'a'*64)
-        except e.OONIBInputDescriptorNotFound:
-            pass
-        else:
-            assert False
+        yield self.assertFailure(self.oonibclient.queryBackend('GET', '/input/' + 'a'*64), e.OONIBInputDescriptorNotFound)
 
     @defer.inlineCallbacks
     def test_http_errors(self):
-        try:
-            yield self.oonibclient.queryBackend('PUT', '/policy/input')
-        except error.Error:
-            pass
-        else:
-            assert False
+        yield self.assertFailure(self.oonibclient.queryBackend('PUT', '/policy/input'), error.Error)
 
     @defer.inlineCallbacks
     def test_create_report(self):
diff --git a/ooni/utils/txscapy.py b/ooni/utils/txscapy.py
index bfd5d7f..89a0236 100644
--- a/ooni/utils/txscapy.py
+++ b/ooni/utils/txscapy.py
@@ -1,5 +1,3 @@
-import ipaddr
-import struct
 import socket
 import os
 import sys
@@ -9,12 +7,10 @@ import random
 from twisted.internet import protocol, base, fdesc
 from twisted.internet import reactor, threads, error
 from twisted.internet import defer, abstract
-from zope.interface import implements
 
 from scapy.config import conf
 from scapy.supersocket import L3RawSocket
-from scapy.all import RandShort, IP, IPerror, ICMP, ICMPerror
-from scapy.all import TCP, TCPerror, UDP, UDPerror
+from scapy.all import RandShort, IP, IPerror, ICMP, ICMPerror, TCP, TCPerror, UDP, UDPerror, read_routes
 
 from ooni.utils import log
 from ooni.settings import config
@@ -53,10 +49,8 @@ def pcapdnet_installed():
 
         config.pcap_dnet = True
 
-    except ImportError:
-        log.err("pypcap or dnet not installed. "
-                "Certain tests may not work.")
-
+    except ImportError as e:
+        log.err(e.message + ". Pypcap or dnet are not properly installed. Certain tests may not work.")
         config.pcap_dnet = False
         conf.use_pcap = False
         conf.use_dnet = False
@@ -320,6 +314,9 @@ class ScapySniffer(ScapyProtocol):
     def packetReceived(self, packet):
         self.pcapwriter.write(packet)
 
+    def close(self):
+        self.pcapwriter.close()
+
 
 class ParasiticTraceroute(ScapyProtocol):
     def __init__(self):
@@ -363,11 +360,7 @@ class ParasiticTraceroute(ScapyProtocol):
 
         def maxttl(packet=None):
             if packet:
-                return min(self.ttl_max,
-                           min(
-                               abs(64 - packet.ttl),
-                               abs(128 - packet.ttl),
-                               abs(256 - packet.ttl))) - 1
+                return min(self.ttl_max, *map(lambda x: x - packet.ttl, [64, 128, 256])) - 1
             else:
                 return self.ttl_max
 



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits