[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [ooni-probe/master] Start porting tests to new twisted based pattern
commit 7a60f4cd618540ff2ae64244867d9d94798616db
Author: Arturo Filastò <hellais@xxxxxxxxxxxxxx>
Date: Wed May 23 21:45:38 2012 +0200
Start porting tests to new twisted based pattern
---
lib/Makefile | 14 ++
lib/traceroute.py | 32 ----
lib/txtraceroute.py | 389 --------------------------------------------------
oonicli.py | 3 -
plugins/bridget.py | 36 +++++
plugins/dropin.cache | 28 ++++
plugins/skel.py | 3 +-
plugoo/tests.py | 32 ++++-
tests/bridget.py | 9 +-
9 files changed, 110 insertions(+), 436 deletions(-)
diff --git a/lib/Makefile b/lib/Makefile
new file mode 100644
index 0000000..dfb2449
--- /dev/null
+++ b/lib/Makefile
@@ -0,0 +1,14 @@
+all: txtorcon txtraceroute
+
+txtraceroute:
+ echo "Processing dependency txtraceroute..."
+ git clone https://github.com/hellais/txtraceroute.git txtraceroute.git
+ mv txtraceroute.git/txtraceroute.py txtraceroute.py
+ rm -rf txtraceroute.git
+
+txtorcon:
+ echo "Processing dependency txtorcon..."
+ git clone https://github.com/meejah/txtorcon.git txtorcon.git
+ mv txtorcon.git/txtorcon txtorcon
+ rm -rf txtorcon.git
+
diff --git a/lib/traceroute.py b/lib/traceroute.py
deleted file mode 100644
index c02722a..0000000
--- a/lib/traceroute.py
+++ /dev/null
@@ -1,32 +0,0 @@
-import sys
-import socket
-from twisted.internet import defer
-from twisted.internet import reactor
-from twisted.internet import threads
-
-from txtraceroute import traceroute
-
-def run(target, src_port, dst_port):
- res = []
- @defer.inlineCallbacks
- def start_trace(target, **settings):
- hops = yield traceroute(target, **settings)
- for hop in hops:
- res.append(hop.get())
- reactor.stop()
-
- settings = dict(hop_callback=None,
- timeout=2,
- max_tries=3,
- max_hops=30, proto="tcp")
- try:
- target = socket.gethostbyname(target)
- except Exception, e:
- print("could not resolve '%s': %s" % (target, str(e)))
- sys.exit(1)
-
- reactor.callWhenRunning(start_trace, target, **settings)
- reactor.run()
- return res
-
-print run("8.8.8.8", 80, 80)
diff --git a/lib/txtraceroute.py b/lib/txtraceroute.py
deleted file mode 100644
index e18b558..0000000
--- a/lib/txtraceroute.py
+++ /dev/null
@@ -1,389 +0,0 @@
-#!/usr/bin/env python
-# coding: utf-8
-#
-# Copyright (c) 2012 Alexandre Fiori
-#
-# Permission is hereby granted, free of charge, to any person obtaining
-# a copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish,
-# distribute, sublicense, and/or sell copies of the Software, and to
-# permit persons to whom the Software is furnished to do so, subject to
-# the following conditions:
-#
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-import json
-import operator
-import os
-import socket
-import struct
-import sys
-import time
-
-from twisted.internet import defer
-from twisted.internet import reactor
-from twisted.internet import threads
-from twisted.python import usage
-from twisted.web.client import getPage
-
-
-class iphdr(object):
- def __init__(self, proto=socket.IPPROTO_ICMP, src="0.0.0.0", dst=None):
- self.version = 4
- self.hlen = 5
- self.tos = 0
- self.length = 20
- self.id = os.getpid()
- self.frag = 0
- self.ttl = 255
- self.proto = proto
- self.cksum = 0
- self.src = src
- self.saddr = socket.inet_aton(src)
- self.dst = dst or "0.0.0.0"
- self.daddr = socket.inet_aton(self.dst)
- self.data = ""
-
- def assemble(self):
- header = struct.pack('BBHHHBB',
- (self.version & 0x0f) << 4 | (self.hlen & 0x0f),
- self.tos, self.length + len(self.data),
- socket.htons(self.id), self.frag,
- self.ttl, self.proto)
- return header + "\000\000" + self.saddr + self.daddr + self.data
-
- @classmethod
- def disassemble(self, data):
- ip = iphdr()
- pkt = struct.unpack('!BBHHHBBH', data[:12])
- ip.version = (pkt[0] >> 4 & 0x0f)
- ip.hlen = (pkt[0] & 0x0f)
- ip.tos, ip.length, ip.id, ip.frag, ip.ttl, ip.proto, ip.cksum = pkt[1:]
- ip.saddr = data[12:16]
- ip.daddr = data[16:20]
- ip.src = socket.inet_ntoa(ip.saddr)
- ip.dst = socket.inet_ntoa(ip.daddr)
- return ip
-
- def __repr__(self):
- return "IP (tos %s, ttl %s, id %s, frag %s, proto %s, length %s) " \
- "%s -> %s" % \
- (self.tos, self.ttl, self.id, self.frag, self.proto,
- self.length, self.src, self.dst)
-
-class tcphdr(object):
- def __init__(self, data="", dport=4242, sport=4242):
- self.seq = 123132
- self.hlen = 44
- self.flags = 2
- self.wsize = 200
- self.cksum = 123
- self.options = 0
- self.mss = 1460
- self.dport = dport
- self.sport = sport
-
- def assemble(self):
- header = struct.pack("!HHL", self.sport, self.dport, self.seq)
- header += '\00\00\00\00'
- header += struct.pack("!HHH", (self.hlen & 0xff) << 10 | (self.flags &
- 0xff), self.wsize, self.cksum)
- # XXX There is something wrong here fixme
- options = struct.pack("!LBBBBBB", self.mss, 1, 3, 3, 1, 1, 1)
- options += struct.pack("!BBL", 8, 10, 1209452188)
- options += '\00'*4
- options += struct.pack("!BB", 4, 2)
- options += '\00'
- return header+options
-
- @classmethod
- def checksum(self, data):
- pass
-
- def disassemble(self, data):
- tcp = tcphdr()
- pkt = struct.unpack("!HHLH", data[:20])
- tcp.sport, tcp.dport, tcp.seq = pkt[:3]
- tcp.hlen = (pkt[4] >> 10 ) & 0xff
- tcp.flags = pkf[4] & 0xff
- tcp.wsize, tcp.cksum = struct.unpack("!HH", data[20:28])
- return tcp
-
-class udphdr(object):
- def __init__(self, data="", dport=4242, sport=4242):
- self.dport = dport
- self.sport = sport
- self.cksum = 0
- self.length = 0
- self.data = data
-
- def assemble(self):
- self.length = len(self.data) + 8
- part1 = struct.pack("!HHH", self.sport, self.dport, self.length)
- cksum = self.checksum(self.data)
- cksum = struct.pack("!H", cksum)
- return part1 + cksum + self.data
-
- @classmethod
- def checksum(self, data):
- # XXX implement proper checksum
- cksum = 0
- return cksum
-
- def disassemble(self, data):
- udp = udphdr()
- pkt = struct.unpack("!HHHH", data)
- udp.src_port, udp.dst_port, udp.length, udp.cksum = pkt
- return udp
-
-class icmphdr(object):
- def __init__(self, data=""):
- self.type = 8
- self.code = 0
- self.cksum = 0
- self.id = os.getpid()
- self.sequence = 0
- self.data = data
-
- def assemble(self):
- part1 = struct.pack("BB", self.type, self.code)
- part2 = struct.pack("!HH", self.id, self.sequence)
- cksum = self.checksum(part1 + "\000\000" + part2 + self.data)
- cksum = struct.pack("!H", cksum)
- return part1 + cksum + part2 + self.data
-
- @classmethod
- def checksum(self, data):
- if len(data) & 1:
- data += "\0"
- cksum = reduce(operator.add,
- struct.unpack('!%dH' % (len(data) >> 1), data))
- cksum = (cksum >> 16) + (cksum & 0xffff)
- cksum += (cksum >> 16)
- cksum = (cksum & 0xffff) ^ 0xffff
- return cksum
-
- @classmethod
- def disassemble(self, data):
- icmp = icmphdr()
- pkt = struct.unpack("!BBHHH", data)
- icmp.type, icmp.code, icmp.cksum, icmp.id, icmp.sequence = pkt
- return icmp
-
- def __repr__(self):
- return "ICMP (type %s, code %s, id %s, sequence %s)" % \
- (self.type, self.code, self.id, self.sequence)
-
-
-@xxxxxxxxxxxxxxxxxxxxx
-def geoip_lookup(ip):
- try:
- r = yield getPage("http://freegeoip.net/json/%s"; % ip)
- d = json.loads(r)
- items = [d["country_name"], d["region_name"], d["city"]]
- text = ", ".join([s for s in items if s])
- defer.returnValue(text.encode("utf-8"))
- except Exception:
- defer.returnValue("Unknown location")
-
-
-@xxxxxxxxxxxxxxxxxxxxx
-def reverse_lookup(ip):
- try:
- r = yield threads.deferToThread(socket.gethostbyaddr, ip)
- defer.returnValue(r[0])
- except Exception:
- defer.returnValue(None)
-
-
-class Hop(object):
- def __init__(self, target, ttl, proto="icmp"):
- self.proto = proto
- self.found = False
- self.tries = 0
- self.last_try = 0
- self.remote_ip = None
- self.remote_icmp = None
- self.remote_host = None
- self.location = ""
-
- self.ttl = ttl
- self.ip = iphdr(dst=target)
- self.ip.ttl = ttl
- self.ip.id += ttl
-
- if proto is "icmp":
- self.icmp = icmphdr("traceroute")
- self.icmp.id = self.ip.id
- self.ip.data = self.icmp.assemble()
- elif proto is "udp":
- self.udp = udphdr("blabla")
- self.ip.data = self.udp.assemble()
- self.ip.proto = socket.IPPROTO_UDP
- elif proto is "tcp":
- self.tcp = tcphdr()
- self.ip.data = self.tcp.assemble()
- self.ip.proto = socket.IPPROTO_TCP
-
- self._pkt = self.ip.assemble()
-
- @property
- def pkt(self):
- self.tries += 1
- self.last_try = time.time()
- return self._pkt
-
- def get(self):
- if self.found:
- if self.remote_host:
- ip = self.remote_host
- else:
- ip = self.remote_ip.src
- ping = self.found - self.last_try
- else:
- ip = None
- ping = None
-
- location = self.location if self.location else None
- return {'ttl': self.ttl, 'ping': ping, 'ip': ip, 'location': location}
-
- def __repr__(self):
- if self.found:
- if self.remote_host:
- ip = ":: %s" % self.remote_host
- else:
- ip = ":: %s" % self.remote_ip.src
- ping = "%0.3fs" % (self.found - self.last_try)
- else:
- ip = "??"
- ping = "-"
-
- location = ":: %s" % self.location if self.location else ""
- return "%02d. %s %s %s" % (self.ttl, ping, ip, location)
-
-
-class TracerouteProtocol(object):
- def __init__(self, target, **settings):
- self.target = target
- self.settings = settings
- self.fd = socket.socket(socket.AF_INET, socket.SOCK_RAW,
- socket.IPPROTO_ICMP)
- self.fd.setsockopt(socket.IPPROTO_IP, socket.IP_HDRINCL, 1)
-
- self.hops = []
- self.out_queue = []
- self.waiting = True
- self.deferred = defer.Deferred()
-
- reactor.addReader(self)
- reactor.addWriter(self)
-
- # send 1st probe packet
- self.out_queue.append(Hop(self.target, 1, settings.get("proto")))
-
- def logPrefix(self):
- return "TracerouteProtocol(%s)" % self.target
-
- def fileno(self):
- return self.fd.fileno()
-
- @defer.inlineCallbacks
- def hopFound(self, hop, ip, icmp):
- hop.remote_ip = ip
- hop.remote_icmp = icmp
-
- if (ip and icmp):
- hop.found = time.time()
- if self.settings.get("geoip_lookup") is True:
- hop.location = yield geoip_lookup(ip.src)
-
- if self.settings.get("reverse_lookup") is True:
- hop.remote_host = yield reverse_lookup(ip.src)
-
- ttl = hop.ttl + 1
- last = self.hops[-2:]
- if len(last) == 2 and last[0].remote_ip == ip or \
- (ttl > (self.settings.get("max_hops", 30) + 1)):
- done = True
- else:
- done = False
-
- if not done:
- cb = self.settings.get("hop_callback")
- if callable(cb):
- yield defer.maybeDeferred(cb, hop)
-
- if not self.waiting:
- if self.deferred:
- self.deferred.callback(self.hops)
- self.deferred = None
- else:
- self.out_queue.append(Hop(self.target, ttl, self.settings.get("proto")))
-
- def doRead(self):
- if not self.waiting or not self.hops:
- return
-
- pkt = self.fd.recv(4096)
-
- # disassemble ip header
- ip = iphdr.disassemble(pkt[:20])
- if ip.proto != socket.IPPROTO_ICMP:
- return
-
- found = False
-
- # disassemble icmp header
- icmp = icmphdr.disassemble(pkt[20:28])
- if icmp.type == 0 and icmp.id == self.hops[-1].icmp.id:
- found = True
- elif icmp.type == 11:
- # disassemble referenced ip header
- ref = iphdr.disassemble(pkt[28:48])
- if ref.dst == self.target:
- found = True
-
- if ip.src == self.target:
- self.waiting = False
-
- if found:
- self.hopFound(self.hops[-1], ip, icmp)
-
- def hopTimeout(self, *ign):
- hop = self.hops[-1]
- if not hop.found:
- if hop.tries < self.settings.get("max_tries", 3):
- # retry
- self.out_queue.append(hop)
- else:
- # give up and move forward
- self.hopFound(hop, None, None)
-
- def doWrite(self):
- if self.waiting and self.out_queue:
- hop = self.out_queue.pop(0)
- pkt = hop.pkt
- if not self.hops or (self.hops and hop.ttl != self.hops[-1].ttl):
- self.hops.append(hop)
- self.fd.sendto(pkt, (hop.ip.dst, 0))
-
- timeout = self.settings.get("timeout", 1)
- reactor.callLater(timeout, self.hopTimeout)
-
- def connectionLost(self, why):
- pass
-
-
-def traceroute(target, **settings):
- tr = TracerouteProtocol(target, **settings)
- return tr.deferred
diff --git a/oonicli.py b/oonicli.py
index 68384f8..d290e68 100755
--- a/oonicli.py
+++ b/oonicli.py
@@ -40,9 +40,6 @@ def retrieve_plugoo():
print "Plugin Broken"
print bi
error = True
- except BrokenMethodImplementation, bmi:
- print "Plugin Broken"
- error = True
if error != False:
print "Plugin Loaded!"
return d
diff --git a/plugins/bridget.py b/plugins/bridget.py
new file mode 100644
index 0000000..7271468
--- /dev/null
+++ b/plugins/bridget.py
@@ -0,0 +1,36 @@
+from zope.interface import implements
+from twisted.python import usage
+from twisted.plugin import IPlugin
+from plugoo.tests import ITest, TwistedTest
+from twisted.internet import threads
+
+from tests.bridget import BridgeT as BridgeTlegacy
+from tests.bridget import BridgeTAsset as BridgeTAsset
+from ooniprobe import ooni
+
+o = ooni()
+
+class BridgetArgs(usage.Options):
+ optParameters = [['asset', 'a', None, 'Asset file'],
+ ['resume', 'r', 0, 'Resume at this index'],
+ ['bridge', 'b', None, 'Specify a single bridge']]
+
+class BridgeT(TwistedTest):
+ implements(IPlugin, ITest)
+
+ shortName = "bridget"
+ description = "Bridget plugin"
+ requirements = None
+ options = BridgetArgs
+
+ def experiment(self):
+ bridget = BridgeTlegacy(o)
+ o.logger.info("Starting bridget test")
+ print "ASSET:%s " % self.asset
+ d = threads.deferToThread(bridget.connect, self.asset)
+ d.addCallback(self.d_experiment.callback, None)
+ return d
+
+# We need to instantiate it otherwise getPlugins does not detect it
+# XXX Find a way to load plugins without instantiating them.
+bridget = BridgeT(None, None)
diff --git a/plugins/dropin.cache b/plugins/dropin.cache
index 7bc3990..9258b18 100755
--- a/plugins/dropin.cache
+++ b/plugins/dropin.cache
@@ -45,4 +45,32 @@ p21
S'skel'
p22
sg10
+NsbasbsS'bridget'
+p23
+g3
+(g4
+g5
+NtRp24
+(dp25
+g8
+S'plugins.bridget'
+p26
+sg10
+Nsg11
+(lp27
+g3
+(g13
+g5
+NtRp28
+(dp29
+g16
+(lp30
+g18
+ag19
+asg20
+g24
+sg21
+S'bridget'
+p31
+sg10
Nsbasbs.
\ No newline at end of file
diff --git a/plugins/skel.py b/plugins/skel.py
index 0427239..93bc1bb 100644
--- a/plugins/skel.py
+++ b/plugins/skel.py
@@ -5,7 +5,8 @@ from plugoo.tests import ITest, TwistedTest
class SkelArgs(usage.Options):
optParameters = [['asset', 'a', None, 'Asset file'],
- ['resume', 'r', 0, 'Resume at this index']]
+ ['resume', 'r', 0, 'Resume at this index'],
+ ['other', 'o', None, 'Other arguments']]
class SkelTest(TwistedTest):
implements(IPlugin, ITest)
diff --git a/plugoo/tests.py b/plugoo/tests.py
index e74586c..39adfa2 100644
--- a/plugoo/tests.py
+++ b/plugoo/tests.py
@@ -81,7 +81,7 @@ class Test:
if assets:
self.logger.debug("Running through tests")
- if extradata['index']:
+ if extradata and 'index' in extradata:
index = extradata['index']
else:
index = None
@@ -168,11 +168,17 @@ class TwistedTest(object):
self.asset = asset
self.arguments = arguments
self.start_time = datetime.now()
+ self._parse_arguments()
#self.ooninet = ooninet
def __repr__(self):
return "<TwistedTest %s %s>" % (self.arguments, self.asset)
+ def _parse_arguments(self):
+ print self.arguments
+ if self.arguments and 'test' in self.arguments:
+ self.test = self.arguments['test']
+
def finished(self, result):
#self.ooninet.report(result)
print "FINIHSED"
@@ -182,12 +188,30 @@ class TwistedTest(object):
result['run_time'] = self.end_time - self.start_time
return self.d.callback(result)
+ def _do_experiment(self):
+ self.d_experiment = defer.Deferred()
+ self.d_experiment.addCallback(self._do_control)
+ self.experiment()
+ return self.d
+
+ def _do_control(self, exp):
+ self.control(exp)
+ self.finished(dict())
+
+ def control(self, exp):
+ print "Doing control..."
+
+ def experiment(self):
+ print "Doing experiment"
+ self.d_experiment.callback(None)
+
def startTest(self):
- print "Starting test"
+ print "Starting test %s" % repr(self)
self.d = defer.Deferred()
result = {}
- reactor.callLater(2.0, self.finished, result)
- return self.d
+ #reactor.callLater(2.0, self.finished, result)
+ # Start experiment
+ return self._do_experiment()
class TwistedTestFactory(object):
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/bridget.py b/tests/bridget.py
index da014fd..a613f61 100644
--- a/tests/bridget.py
+++ b/tests/bridget.py
@@ -241,13 +241,6 @@ Log info file %s
except:
self.logger.error("Error in starting Tor (do you have tor installed?)")
- # XXX this only works on UNIX (do we care?)
- # Make file reading non blocking
- try:
- fcntl.fcntl(p.stdout, fcntl.F_SETFL, os.O_NONBLOCK)
- except:
- self.logger.error("Unable to set file descriptor to non blocking")
-
self.logger.info("Testing bridge: %s" % bridge)
while True:
o = ""
@@ -308,6 +301,7 @@ Log info file %s
self.logger.error("Error IOError: EAGAIN")
raise
sys.exc_clear()
+ print "In this exception 1"
try:
# Set the timeout for the socket wait
@@ -376,3 +370,4 @@ def run(ooni, assets=None):
bridget.print_failures()
bridget.clean()
ooni.logger.info("Testing completed!")
+
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits