[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Update the tests to be consistent with the new data formats



commit 9bd24ee947194b0f5713b46e845106210e7163c9
Author: Arturo Filastò <arturo@xxxxxxxxxxx>
Date:   Thu Jan 28 13:09:36 2016 +0100

    Update the tests to be consistent with the new data formats
---
 ooni/nettests/blocking/dns_consistency.py | 29 +++++++----
 ooni/reporter.py                          | 50 ++++++++++---------
 ooni/templates/dnst.py                    | 83 ++++++++++++++++++++++---------
 ooni/templates/httpt.py                   | 41 +++++++++++----
 ooni/templates/scapyt.py                  | 12 +++--
 ooni/tests/test_trueheaders.py            | 12 ++---
 ooni/utils/__init__.py                    |  7 +++
 ooni/utils/trueheaders.py                 |  2 +-
 8 files changed, 157 insertions(+), 79 deletions(-)

diff --git a/ooni/nettests/blocking/dns_consistency.py b/ooni/nettests/blocking/dns_consistency.py
index 88e5d5e..93a2d65 100644
--- a/ooni/nettests/blocking/dns_consistency.py
+++ b/ooni/nettests/blocking/dns_consistency.py
@@ -39,7 +39,7 @@ class DNSConsistencyTest(dnst.DNSTest):
     name = "DNS Consistency"
     description = "Checks to see if the DNS responses from a "\
                   "set of DNS resolvers are consistent."
-    version = "0.6"
+    version = "0.7.0"
     authors = "Arturo Filastò, Isis Lovecruft"
 
     inputFile = ['file', 'f', None,
@@ -110,7 +110,11 @@ class DNSConsistencyTest(dnst.DNSTest):
         log.msg("Doing the test lookups on %s" % self.input)
         hostname = self.input
 
-        self.report['tampering'] = {}
+        self.report['successful'] = []
+        self.report['failures'] = []
+        self.report['inconsistent'] = []
+
+        self.report['errors'] = {}
 
         try:
             control_answers = yield self.performALookup(hostname,
@@ -121,11 +125,11 @@ class DNSConsistencyTest(dnst.DNSTest):
                     "Got no response from control DNS server %s:%d, "
                     "perhaps the DNS resolver is down?" %
                     self.control_dns_server)
-                self.report['tampering'][
+                self.report['errors'][
                     "%s:%d" %
                     self.control_dns_server] = 'no_answer'
         except:
-            self.report['tampering'][
+            self.report['errors'][
                 "%s:%d" %
                 self.control_dns_server] = 'error'
             control_answers = None
@@ -139,12 +143,14 @@ class DNSConsistencyTest(dnst.DNSTest):
                                                                test_dns_server)
             except Exception:
                 log.err("Problem performing the DNS lookup")
-                self.report['tampering'][test_resolver] = 'dns_lookup_error'
+                self.report['errors'][test_resolver] = 'dns_lookup_error'
+                self.report['failures'].append(test_resolver)
                 continue
 
             if not experiment_answers:
                 log.err("Got no response, perhaps the DNS resolver is down?")
-                self.report['tampering'][test_resolver] = 'no_answer'
+                self.report['errors'][test_resolver] = 'no_answer'
+                self.report['failures'].append(test_resolver)
                 continue
             else:
                 log.debug(
@@ -165,12 +171,13 @@ class DNSConsistencyTest(dnst.DNSTest):
 
             if not control_answers:
                 log.msg("Skipping control resolver comparison")
-                self.report['tampering'][test_resolver] = None
+                self.report['errors'][test_resolver] = None
 
             elif set(experiment_answers) & set(control_answers):
                 lookup_details()
                 log.msg("tampering: false")
-                self.report['tampering'][test_resolver] = False
+                self.report['errors'][test_resolver] = False
+                self.report['successful'].append(test_resolver)
             else:
                 log.msg("Trying to do reverse lookup")
                 experiment_reverse = yield self.performPTRLookup(experiment_answers[0],
@@ -182,12 +189,14 @@ class DNSConsistencyTest(dnst.DNSTest):
                     log.msg("Further testing has eliminated false positives")
                     lookup_details()
                     log.msg("tampering: reverse_match")
-                    self.report['tampering'][test_resolver] = 'reverse_match'
+                    self.report['errors'][test_resolver] = 'reverse_match'
+                    self.report['successful'].append(test_resolver)
                 else:
                     log.msg("Reverse lookups do not match")
                     lookup_details()
                     log.msg("tampering: true")
-                    self.report['tampering'][test_resolver] = True
+                    self.report['errors'][test_resolver] = True
+                    self.report['inconsistent'].append(test_resolver)
 
     def inputProcessor(self, filename=None):
         """
diff --git a/ooni/reporter.py b/ooni/reporter.py
index db6f7c1..3ea1b10 100644
--- a/ooni/reporter.py
+++ b/ooni/reporter.py
@@ -236,6 +236,7 @@ class OONIBReporter(OReporter):
         self.validateCollectorAddress()
 
         self.reportID = None
+        self.supportedFormats = ["yaml"]
 
         if self.collectorAddress.startswith('https://'):
             # not sure if there's something else it needs.  Seems to work.
@@ -257,27 +258,26 @@ class OONIBReporter(OReporter):
         if not re.match(regexp, self.collectorAddress):
             raise errors.InvalidOONIBCollectorAddress
 
-    def serializeEntry(self, entry):
-        if config.report.format == "json":
+    def serializeEntry(self, entry, serialisation_format="yaml"):
+        if serialisation_format == "json":
             if isinstance(entry, Measurement):
                 report_entry = entry.testInstance.report
             elif isinstance(entry, Failure):
-                report_entry = entry.value
+                report_entry = {'failure': entry.value}
             elif isinstance(entry, dict):
                 report_entry = entry
-            report_entry["record_type"] = "entry"
-            report_entry["report_id"] = self.reportID
-            content = json.dumps(report_entry, ensure_ascii=True) + "\n"
+            return report_entry
         else:
             content = '---\n'
             if isinstance(entry, Measurement):
-                content += safe_dump(entry.testInstance.report)
+                report_entry = entry.testInstance.report
             elif isinstance(entry, Failure):
-                content += entry.value
+                report_entry = {'failure': entry.value}
             elif isinstance(entry, dict):
-                content += safe_dump(entry)
+                report_entry = entry
+            content += safe_dump(report_entry)
             content += '...\n'
-        return content
+            return content
 
     @defer.inlineCallbacks
     def writeReportEntry(self, entry):
@@ -285,8 +285,13 @@ class OONIBReporter(OReporter):
 
         url = self.collectorAddress + '/report/' + self.reportID
 
-        request = {'format': config.report.format,
-                   'content': self.serializeEntry(entry)}
+        if "json" in self.supportedFormats:
+            serialisation_format = 'json'
+        else:
+            serialisation_format = 'yaml'
+
+        request = {'format': serialisation_format,
+                   'content': self.serializeEntry(entry, serialisation_format)}
 
         log.debug("Updating report with id %s (%s)" % (self.reportID, url))
         request_json = json.dumps(request)
@@ -295,12 +300,11 @@ class OONIBReporter(OReporter):
         bodyProducer = StringProducer(request_json)
 
         try:
-            yield self.agent.request("PUT", url,
+            yield self.agent.request("PUT", str(url),
                                      bodyProducer=bodyProducer)
-        except:
-            # XXX we must trap this in the runner and make sure to report the
-            # data later.
+        except Exception as exc:
             log.err("Error in writing report entry")
+            log.exception(exc)
             raise errors.OONIBReportUpdateError
 
     @defer.inlineCallbacks
@@ -324,21 +328,16 @@ class OONIBReporter(OReporter):
 
         url = self.collectorAddress + '/report'
 
-        content = '---\n'
-        content += safe_dump(self.testDetails)
-        content += '...\n'
-
         request = {
             'software_name': self.testDetails['software_name'],
             'software_version': self.testDetails['software_version'],
             'probe_asn': self.testDetails['probe_asn'],
+            'probe_cc': self.testDetails['probe_cc'],
             'test_name': self.testDetails['test_name'],
             'test_version': self.testDetails['test_version'],
+            'start_time': self.testDetails['start_time'],
             'input_hashes': self.testDetails['input_hashes'],
-            # XXX there is a bunch of redundancy in the arguments getting sent
-            # to the backend. This may need to get changed in the client and
-            # the backend.
-            'content': content
+            'format': 'json'
         }
         # import values from the environment
         request.update([(k.lower(),v) for (k,v) in os.environ.iteritems()
@@ -395,6 +394,9 @@ class OONIBReporter(OReporter):
 
         self.reportID = parsed_response['report_id']
         self.backendVersion = parsed_response['backend_version']
+
+        self.supportedFormats = parsed_response.get('supported_formats', ["yaml"])
+
         log.debug("Created report with id %s" % parsed_response['report_id'])
         defer.returnValue(parsed_response['report_id'])
 
diff --git a/ooni/templates/dnst.py b/ooni/templates/dnst.py
index 7c9626c..e8ae189 100644
--- a/ooni/templates/dnst.py
+++ b/ooni/templates/dnst.py
@@ -58,13 +58,36 @@ def connectionLost(self, reason=None):
 udp.Port.connectionLost = connectionLost
 
 def representAnswer(answer):
-    # We store the resource record and the answer payload in a
-    # tuple
-    return (repr(answer), repr(answer.payload))
+    answer_types = {
+        dns.SOA: 'SOA',
+        dns.NS: 'NS',
+        dns.PTR: 'PTR',
+        dns.A: 'A',
+        dns.CNAME: 'CNAME',
+        dns.MX: 'MX'
+    }
+    answer_type = answer_types.get(answer.type, 'unknown')
+    represented_answer = {
+        "answer_type": answer_type
+    }
+    if answer_type is 'SOA':
+        represented_answer['ttl'] = answer.payload.ttl
+        represented_answer['hostname'] = answer.payload.mname
+        represented_answer['responsible_name'] = answer.payload.rname
+        represented_answer['serial_number'] = answer.payload.serial
+        represented_answer['refresh_interval'] = answer.payload.refresh
+        represented_answer['retry_interval'] = answer.payload.retry
+        represented_answer['minimum_ttl'] = answer.payload.minimum
+        represented_answer['expiration_limit'] = answer.payload.expire
+    elif answer_type in ['NS', 'PTR', 'CNAME']:
+        represented_answer['hostname'] = answer.payload.name.name
+    elif answer_type is 'A':
+        represented_answer['ipv4'] = answer.payload.dottedQuad()
+    return represented_answer
 
 class DNSTest(NetTestCase):
     name = "Base DNS Test"
-    version = 0.1
+    version = "0.2.0"
 
     requiresRoot = False
     queryTimeout = [1]
@@ -137,18 +160,25 @@ class DNSTest(NetTestCase):
         :dns_server: is the dns_server that should be used for the lookup as a
                      tuple of ip port (ex. ("127.0.0.1", 53))
         """
-        types={'NS':dns.NS,'A':dns.A,'SOA':dns.SOA,'PTR':dns.PTR}
-        dnsType=types[dns_type]
+        types = {
+            'NS': dns.NS,
+            'A': dns.A,
+            'SOA': dns.SOA,
+            'PTR': dns.PTR
+        }
+        dnsType = types[dns_type]
         query = [dns.Query(hostname, dnsType, dns.IN)]
         def gotResponse(message):
-            log.debug(dns_type+" Lookup successful")
+            log.debug(dns_type + " Lookup successful")
             log.debug(str(message))
-            addrs = []
-            answers = []
+
             if dns_server:
                 msg = message.answers
             else:
                 msg = message[0]
+
+            answers = []
+            addrs = []
             for answer in msg:
                 if answer.type is dnsType:
                     if dnsType is dns.SOA:
@@ -159,46 +189,53 @@ class DNSTest(NetTestCase):
                         addr = answer.payload.dottedQuad()
                     else:
                         addr = None
-                    addrs.append(addr)
+                        addrs.append(addr)
                 answers.append(representAnswer(answer))
 
-            DNSTest.addToReport(self, query, resolver=dns_server, query_type=dns_type,
-                        answers=answers, addrs=addrs)
+            if dns_type == 'SOA':
+                for authority in message.authority:
+                    answers.append(representAnswer(authority))
+
+            DNSTest.addToReport(self, query, resolver=dns_server,
+                                query_type=dns_type, answers=answers)
             return addrs
 
         def gotError(failure):
             failure.trap(gaierror, TimeoutError)
-            DNSTest.addToReport(self, query, resolver=dns_server, query_type=dns_type,
-                        failure=failure)
+            DNSTest.addToReport(self, query, resolver=dns_server,
+                                query_type=dns_type, failure=failure)
             return failure
 
         if dns_server:
             resolver = Resolver(servers=[dns_server])
             d = resolver.queryUDP(query, timeout=self.queryTimeout)
         else:
-            lookupFunction={'NS':client.lookupNameservers, 'SOA':client.lookupAuthority, 'A':client.lookupAddress, 'PTR':client.lookupPointer}
+            lookupFunction = {
+                'NS': client.lookupNameservers,
+                'SOA': client.lookupAuthority,
+                'A': client.lookupAddress,
+                'PTR': client.lookupPointer
+            }
             d = lookupFunction[dns_type](hostname)
 
         d.addCallback(gotResponse)
         d.addErrback(gotError)
         return d
 
-
     def addToReport(self, query, resolver=None, query_type=None,
-                    answers=None, name=None, addrs=None, failure=None):
+                    answers=None, failure=None):
         log.debug("Adding %s to report)" % query)
         result = {}
-        result['resolver'] = resolver
+        result['resolver_hostname'] = resolver[0]
+        result['resolver_port'] = resolver[1]
         result['query_type'] = query_type
-        result['query'] = repr(query)
+        result['hostname'] = str(query[0].name)
+        result['failure'] = None
         if failure:
             result['failure'] = failureToString(failure)
 
+        result['answers'] = []
         if answers:
             result['answers'] = answers
-            if name:
-                result['name'] = name
-            if addrs:
-                result['addrs'] = addrs
 
         self.report['queries'].append(result)
diff --git a/ooni/templates/httpt.py b/ooni/templates/httpt.py
index 3702e86..15630d5 100644
--- a/ooni/templates/httpt.py
+++ b/ooni/templates/httpt.py
@@ -9,7 +9,7 @@ from twisted.internet.endpoints import TCP4ClientEndpoint
 from ooni.utils.trueheaders import TrueHeadersAgent, TrueHeadersSOCKS5Agent
 
 from ooni.nettest import NetTestCase
-from ooni.utils import log
+from ooni.utils import log, base64Dict
 from ooni.settings import config
 
 from ooni.utils.net import BodyReceiver, StringProducer, userAgents
@@ -122,11 +122,28 @@ class HTTPTest(NetTestCase):
 
             failure (instance): An instance of :class:twisted.internet.failure.Failure
         """
+        def _representHeaders(headers):
+            represented_headers = {}
+            for name, value in headers.getAllRawHeaders():
+                represented_headers[name] = value[0]
+            return represented_headers
+
+        def _representBody(body):
+            # XXX perhaps add support for decoding gzip in the future.
+            try:
+                body.replace('\0', '')
+                body = unicode(body, 'ascii')
+            except UnicodeDecodeError:
+                try:
+                    body = unicode(body, 'utf-8')
+                except UnicodeDecodeError:
+                    body = base64Dict(body)
+
         log.debug("Adding %s to report" % request)
         request_headers = TrueHeaders(request['headers'])
-        request_response = {
+        session = {
             'request': {
-                'headers': list(request_headers.getAllRawHeaders()),
+                'headers': _representHeaders(request_headers),
                 'body': request['body'],
                 'url': request['url'],
                 'method': request['method'],
@@ -134,15 +151,16 @@ class HTTPTest(NetTestCase):
             }
         }
         if response:
-            request_response['response'] = {
-                'headers': list(response.headers.getAllRawHeaders()),
-                'body': response_body if self.localOptions.get('withoutbody', 0) == 0 else '',
+            session['response'] = {
+                'headers': _representHeaders(response.headers),
+                'body': _representBody(response_body),
                 'code': response.code
-            }
+        }
+        session['failure'] = None
         if failure_string:
-            request_response['failure'] = failure_string
+            session['failure'] = failure_string
 
-        self.report['requests'].append(request_response)
+        self.report['requests'].append(session)
 
     def _processResponseBody(self, response_body, request, response, body_processor):
         log.debug("Processing response body")
@@ -302,7 +320,10 @@ class HTTPTest(NetTestCase):
         request['url'] = url
         request['headers'] = headers
         request['body'] = body
-        request['tor'] = {}
+        request['tor'] = {
+            'exit_ip': None,
+            'exit_name': None
+        }
         if use_tor:
             request['tor']['is_tor'] = True
         else:
diff --git a/ooni/templates/scapyt.py b/ooni/templates/scapyt.py
index 35dbf5b..f4b6280 100644
--- a/ooni/templates/scapyt.py
+++ b/ooni/templates/scapyt.py
@@ -1,11 +1,17 @@
 from ooni.nettest import NetTestCase
-from ooni.utils import log
+from ooni.utils import log, base64Dict
 from ooni.settings import config
 from ooni.utils.net import hasRawSocketPermission
 
 from ooni.utils.txscapy import ScapySender, ScapyFactory
 
 
+def _representPacket(packet):
+    return {
+        "raw_packet": base64Dict(str(packet)),
+        "summary": repr(packet)
+    }
+
 class BaseScapyTest(NetTestCase):
 
     """
@@ -88,8 +94,8 @@ class BaseScapyTest(NetTestCase):
                 sent_packet.src = '127.0.0.1'
                 received_packet.dst = '127.0.0.1'
 
-            self.report['sent_packets'].append(sent_packet)
-            self.report['answered_packets'].append(received_packet)
+            self.report['sent_packets'].append(_representPacket(sent_packet))
+            self.report['answered_packets'].append(_representPacket(received_packet))
         return packets
 
     def sr(self, packets, timeout=None, *arg, **kw):
diff --git a/ooni/tests/test_trueheaders.py b/ooni/tests/test_trueheaders.py
index 8e9f053..f6c7812 100644
--- a/ooni/tests/test_trueheaders.py
+++ b/ooni/tests/test_trueheaders.py
@@ -23,19 +23,15 @@ dummy_headers_dict3 = {
 class TestTrueHeaders(unittest.TestCase):
     def test_names_match(self):
         th = TrueHeaders(dummy_headers_dict)
-        self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict)), set())
+        self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict)), [])
 
     def test_names_not_match(self):
         th = TrueHeaders(dummy_headers_dict)
-        self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), set(['Header3']))
+        self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), ['Header3'])
 
         th = TrueHeaders(dummy_headers_dict3)
-        self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), set(['Header3', 'Header4']))
+        self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), ['Header3', 'Header4'])
 
     def test_names_match_expect_ignore(self):
         th = TrueHeaders(dummy_headers_dict)
-        self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2), ignore=['Header3']), set())
-
-
-
-
+        self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2), ignore=['Header3']), [])
diff --git a/ooni/utils/__init__.py b/ooni/utils/__init__.py
index 02904cc..5e9616b 100644
--- a/ooni/utils/__init__.py
+++ b/ooni/utils/__init__.py
@@ -5,6 +5,7 @@ import glob
 import os
 
 import gzip
+from base64 import b64encode
 from zipfile import ZipFile
 
 from ooni import otime
@@ -176,3 +177,9 @@ def gunzip(filename, dst):
 def get_ooni_root():
     script = os.path.join(__file__, '..')
     return os.path.dirname(os.path.realpath(script))
+
+def base64Dict(data):
+    return {
+        'format': 'base64',
+        'data': b64encode(data)
+    }
diff --git a/ooni/utils/trueheaders.py b/ooni/utils/trueheaders.py
index 149b440..b7671b5 100644
--- a/ooni/utils/trueheaders.py
+++ b/ooni/utils/trueheaders.py
@@ -80,7 +80,7 @@ class TrueHeaders(http_headers.Headers):
                 pass
             else:
                 diff.add(name)
-        return diff
+        return list(diff)
 
     def getAllRawHeaders(self):
         for k, v in self._rawHeaders.iteritems():



_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits