[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[tor-commits] [ooni-probe/master] Fixed imports.



commit cf5e13c241389f8192a587c0601a8cf561329ae7
Author: Isis Lovecruft <isis@xxxxxxxxxxxxxx>
Date:   Fri Sep 14 02:20:52 2012 +0000

    Fixed imports.
---
 ooni/lib/Makefile          |   30 ++
 ooni/lib/__init__.py       |   43 +---
 ooni/lib/txscapy           |    1 -
 ooni/lib/txscapy.py        |  348 ++++++++++++++++++++
 ooni/lib/txtraceroute      |    1 -
 ooni/lib/txtraceroute.py   |  752 ++++++++++++++++++++++++++++++++++++++++++++
 ooni/ooniprobe.py          |   10 +-
 ooni/plugins/blocking.py   |    2 +-
 ooni/plugins/dnstamper.py  |    6 +-
 ooni/plugins/httphost.py   |    2 +-
 ooni/plugins/tcpconnect.py |    5 +-
 ooni/plugoo/nodes.py       |    6 +-
 ooni/plugoo/reports.py     |    2 +-
 ooni/plugoo/tests.py       |    3 +-
 ooni/utils/log.py          |    1 -
 15 files changed, 1152 insertions(+), 60 deletions(-)

diff --git a/ooni/lib/Makefile b/ooni/lib/Makefile
new file mode 100644
index 0000000..3b0c922
--- /dev/null
+++ b/ooni/lib/Makefile
@@ -0,0 +1,30 @@
+all: txtorcon txtraceroute
+
+txtraceroute:
+	echo "Processing dependency txtraceroute..."
+	git clone https://github.com/hellais/txtraceroute.git  txtraceroute.git
+	mv txtraceroute.git/txtraceroute.py txtraceroute.py
+	rm -rf txtraceroute.git
+
+txtorcon:
+	echo "Processing dependency txtorcon..."
+	git clone https://github.com/meejah/txtorcon.git txtorcon.git
+	mv txtorcon.git/txtorcon txtorcon
+	rm -rf txtorcon.git
+
+clean:
+	rm -rf txtorcon
+	rm -rf txtraceroute.py
+
+#txscapy:
+#	echo "Processing dependency txscapy"
+#	git clone https://github.com/hellais/txscapy.git txscapy.git
+#	mv txscapy.git/txscapy.py txscapy.py
+#	rm -rf txscapy.git
+
+#rfc3339:
+#	echo "Processing RFC3339 dependency"
+#	hg clone https://bitbucket.org/henry/rfc3339 rfc3339
+#	mv rfc3339/rfc3339.py rfc3339.py
+#	rm -rf rfc3339
+
diff --git a/ooni/lib/__init__.py b/ooni/lib/__init__.py
index 0fd36c5..611d50c 100644
--- a/ooni/lib/__init__.py
+++ b/ooni/lib/__init__.py
@@ -1,40 +1,5 @@
-import pkgutil
-import sys
-from os import listdir, path
+from sys import path as syspath
+from os  import path as ospath
 
-__all__ = ['txtorcon', 'txscapy', 'txtraceroute']
-
-__sub_modules__ = [ ]
-
-def callback(arg, directory, files):
-    for file in listdir(directory):
-        fullpath = path.abspath(file)
-        if path.isdir(fullpath) and not path.islink(fullpath):
-            __sub_modules__.append(fullpath)
-            sys.path.append(fullpath)
-
-path.walk(".", callback, None)
-
-def load_submodules(init, list):
-    for subdir in list:
-        contents=[x for x in pkgutil.iter_modules(path=subdir, 
-                                                  prefix='ooni.lib.')]
-        for loader, module_name, ispkg in contents:
-            init_dot_module = init + "." + module_name
-            if init_dot_module in sys.modules:
-                module = sys.modules[module_name]
-            else:
-                if module_name in __all__:
-                    grep = loader.find_module(module_name)
-                    module = grep.load_module(module_name)
-                else:
-                    module = None
-
-            if module is not None:
-                globals()[module_name] = module
-
-load_submodules(__name__, __sub_modules__)
-
-print "system paths are: %s" % sys.path
-print "globals are: %s" % globals()
-print "system modules are: %s" % sys.modules
+pwd = ospath.dirname(__file__)
+syspath.append(pwd)
diff --git a/ooni/lib/txscapy b/ooni/lib/txscapy
deleted file mode 160000
index 19fb281..0000000
--- a/ooni/lib/txscapy
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 19fb28150c0b31f16a1ae2bc0aadeb6fd3c259bf
diff --git a/ooni/lib/txscapy.py b/ooni/lib/txscapy.py
new file mode 100644
index 0000000..4d83dce
--- /dev/null
+++ b/ooni/lib/txscapy.py
@@ -0,0 +1,348 @@
+# -*- coding:utf8 -*-
+"""
+    txscapy
+    ******
+    (c) 2012 Arturo Filastò
+    a twisted wrapper for scapys send and receive functions.
+
+    This software has been written to be part of OONI, the Open Observatory of
+    Network Interference. More information on that here: http://ooni.nu/
+
+"""
+
+import struct
+import socket
+import os
+import sys
+import time
+
+from twisted.internet import protocol, base, fdesc, error, defer
+from twisted.internet import reactor, threads
+from twisted.python import log
+from zope.interface import implements
+
+from scapy.all import Gen
+from scapy.all import SetGen
+
+LINUX=sys.platform.startswith("linux")
+OPENBSD=sys.platform.startswith("openbsd")
+FREEBSD=sys.platform.startswith("freebsd")
+NETBSD=sys.platform.startswith("netbsd")
+DARWIN=sys.platform.startswith("darwin")
+SOLARIS=sys.platform.startswith("sunos")
+WINDOWS=sys.platform.startswith("win32")
+
+from scapy.all import RawPcapWriter, MTU, BasePacketList, conf
+class PcapWriter(RawPcapWriter):
+    def __init__(self, filename, linktype=None, gz=False, endianness="",
+                 append=False, sync=False):
+        RawPcapWriter.__init__(self, filename, linktype=None, gz=False,
+                               endianness="", append=False, sync=False)
+        fdesc.setNonBlocking(self.f)
+
+    def _write_header(self, pkt):
+        if self.linktype == None:
+            if type(pkt) is list or type(pkt) is tuple or isinstance(pkt, BasePacketList):
+                pkt = pkt[0]
+            try:
+                self.linktype = conf.l2types[pkt.__class__]
+            except KeyError:
+                self.linktype = 1
+        RawPcapWriter._write_header(self, pkt)
+
+    def _write_packet(self, packet):
+        sec = int(packet.time)
+        usec = int(round((packet.time-sec)*1000000))
+        s = str(packet)
+        caplen = len(s)
+        RawPcapWriter._write_packet(self, s, sec, usec, caplen, caplen)
+
+class ScapySocket(object):
+    MTU = 1500
+    def __init__(self, filter=None, iface=None, nofilter=None):
+        from scapy.all import conf
+        self.ssocket = conf.L3socket(filter=filter, iface=iface, nofilter=nofilter)
+
+    def fileno(self):
+        return self.ssocket.ins.fileno()
+
+    def send(self, data):
+        return self.ssocket.send(data)
+
+    def recv(self):
+        if FREEBSD or DARWIN:
+            return self.ssocket.nonblock_recv()
+        else:
+            return self.ssocket.recv(self.MTU)
+
+class Scapy(object):
+    """
+    A twisted based wrapper for scapy send and receive functionality.
+
+    It sends packets inside of a threadpool and receives packets using the
+    libdnet receive non blocking file descriptor.
+    """
+    min = 2
+    max = 6
+    debug = True
+    write_only_answers = False
+    pcapwriter = None
+    recv = False
+
+    def __init__(self, pkts=None, maxPacketSize=8192, reactor=None, filter=None,
+            iface=None, nofilter=None, pcapfile=None):
+
+        if self.debug:
+            log.startLogging(sys.stdout)
+
+        self.maxPacketSize = maxPacketSize
+        if not reactor:
+            from twisted.internet import reactor
+
+        self._reactor = reactor
+
+        if pkts:
+            self._buildPacketQueues(pkts)
+            self._buildSocket()
+
+        self.cthreads = 0
+        self.mthreads = 80
+
+        self.running = False
+        self.done = False
+        self.finished = False
+
+        import thread
+        from twisted.python import threadpool
+        self.threadID = thread.get_ident
+        self.threadpool = threadpool.ThreadPool(self.min, self.max)
+        self.startID = self._reactor.callWhenRunning(self._start)
+
+        self.deferred = defer.Deferred()
+
+        if pcapfile:
+            self.pcapwriter = PcapWriter(pcapfile)
+
+    def _buildSocket(self, filter=None, iface=None, nofilter=None):
+        self.socket = ScapySocket(filter, iface, nofilter)
+        if self.recv:
+            self._reactor.addReader(self)
+
+    def _buildPacketQueues(self, pkts):
+        """
+        Converts the list of packets to a Scapy generator and sets up all the
+        necessary attributes for understanding if all the needed responses have
+        been received.
+        """
+        if not isinstance(pkts, Gen):
+            self.pkts = SetGen(pkts)
+
+        self.outqueue = [p for p in pkts]
+
+        self.total_count = len(self.outqueue)
+        self.answer_count = 0
+        self.out_count = 0
+
+        self.hsent = {}
+        for p in self.outqueue:
+            h = p.hashret()
+            if h in self.hsent:
+                self.hsent[h].append(p)
+            else:
+                self.hsent[h] = [p]
+
+
+    def gotAnswer(self, answer, question):
+        """
+        Got a packet that has been identified as an answer to one of the sent
+        out packets.
+
+        If the answer count matches the sent count the finish callback is
+        fired.
+
+        @param answer: the packet received on the wire.
+
+        @param question: the sent packet that matches that response.
+
+        """
+
+        if self.pcapwriter and self.write_only_answers:
+            self.pcapwriter.write(question)
+            self.pcapwriter.write(answer)
+        self.answer_count += 1
+        if self.answer_count >= self.total_count:
+            print "Got all the answers I need"
+            self.deferred.callback(None)
+
+    def processAnswer(self, pkt, hlst):
+        """
+        Checks if the potential answer is in fact an answer to one of the
+        matched sent packets. Uses the scapy .answers() function to verify
+        this.
+
+        @param pkt: The packet to be tested if is the answer to a sent packet.
+
+        @param hlst: a list of packets that match the hash for an answer to
+                     pkt.
+        """
+        for i in range(len(hlst)):
+            if pkt.answers(hlst[i]):
+                self.gotAnswer(pkt, hlst[i])
+
+    def fileno(self):
+        """
+        Returns a fileno for use by twisteds Reader.
+        """
+        return self.socket.fileno()
+
+    def processPacket(self, pkt):
+        """
+        Override this method to process your packets.
+
+        @param pkt: the packet that has been received.
+        """
+        #pkt.show()
+
+
+    def doRead(self):
+        """
+        There is something to be read on the wire. Do all the processing on the
+        received packet.
+        """
+        pkt = self.socket.recv()
+        if self.pcapwriter and not self.write_only_answers:
+            self.pcapwriter.write(pkt)
+        self.processPacket(pkt)
+
+        h = pkt.hashret()
+        if h in self.hsent:
+            hlst = self.hsent[h]
+            self.processAnswer(pkt, hlst)
+
+    def logPrefix(self):
+        """
+        The prefix to be prepended in logging.
+        """
+        return "txScapy"
+
+    def _start(self):
+        """
+        Start the twisted thread pool.
+        """
+        self.startID = None
+        return self.start()
+
+    def start(self):
+        """
+        Actually start the thread pool.
+        """
+        if not self.running:
+            self.threadpool.start()
+            self.shutdownID = self._reactor.addSystemEventTrigger(
+                    'during', 'shutdown', self.finalClose)
+            self.running = True
+
+    def sendPkt(self, pkt):
+        """
+        Send a packet to the wire.
+
+        @param pkt: The packet to be sent.
+        """
+        self.socket.send(pkt)
+
+    def sr(self, pkts, filter=None, iface=None, nofilter=0, *args, **kw):
+        """
+        Wraps the scapy sr function.
+
+        @param nofilter: put 1 to avoid use of bpf filters
+
+        @param retry:    if positive, how many times to resend unanswered packets
+                         if negative, how many times to retry when no more packets are
+                         answered (XXX to be implemented)
+
+        @param timeout:  how much time to wait after the last packet has
+                         been sent (XXX to be implemented)
+
+        @param multi:    whether to accept multiple answers for the same
+                         stimulus (XXX to be implemented)
+
+        @param filter:   provide a BPF filter
+        @param iface:    listen answers only on the given interface
+        """
+        self.recv = True
+        self._sendrcv(pkts, filter=filter, iface=iface, nofilter=nofilter)
+
+    def send(self, pkts, filter=None, iface=None, nofilter=0, *args, **kw):
+        """
+        Wraps the scapy send function. Its the same as send and receive, except
+        it does not receive. Who would have ever guessed? ;)
+
+        @param nofilter: put 1 to avoid use of bpf filters
+
+        @param retry:    if positive, how many times to resend unanswered packets
+                         if negative, how many times to retry when no more packets are
+                         answered (XXX to be implemented)
+
+        @param timeout:  how much time to wait after the last packet has
+                         been sent (XXX to be implemented)
+
+        @param multi:    whether to accept multiple answers for the same
+                         stimulus (XXX to be implemented)
+
+        @param filter:   provide a BPF filter
+        @param iface:    listen answers only on the given interface
+        """
+        self.recv = False
+        self._sendrcv(pkts, filter=filter, iface=iface, nofilter=nofilter)
+
+    def _sendrcv(self, pkts, filter=None, iface=None, nofilter=0):
+        self._buildSocket(filter, iface, nofilter)
+        self._buildPacketQueues(pkts)
+        def sent(cb):
+            if self.cthreads < self.mthreads and not self.done:
+                pkt = None
+                try:
+                    pkt = self.outqueue.pop()
+                except:
+                    self.done = True
+                    if not self.recv:
+                        self.deferred.callback(None)
+                    return
+                d = threads.deferToThreadPool(reactor, self.threadpool,
+                                    self.sendPkt, pkt)
+                d.addCallback(sent)
+                return d
+
+        for x in range(self.mthreads):
+            try:
+                pkt = self.outqueue.pop()
+            except:
+                self.done = True
+                return
+            if self.cthreads >= self.mthreads and self.done:
+                return
+            d = threads.deferToThreadPool(reactor, self.threadpool,
+                                self.sendPkt, pkt)
+            d.addCallback(sent)
+            return d
+
+    def connectionLost(self, why):
+        pass
+
+    def finalClose(self):
+        """
+        Clean all the thread related stuff up.
+        """
+        self.shutdownID = None
+        self.threadpool.stop()
+        self.running = False
+
+def txsr(*args, **kw):
+    tr = Scapy(*args, **kw)
+    tr.sr(*args, **kw)
+    return tr.deferred
+
+def txsend(*arg, **kw):
+    tr = Scapy(*arg, **kw)
+    tr.send(*arg, **kw)
+    return tr.deferred
diff --git a/ooni/lib/txtraceroute b/ooni/lib/txtraceroute
deleted file mode 160000
index 067a260..0000000
--- a/ooni/lib/txtraceroute
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 067a2609390e77bf9187275638cf8786efef7e13
diff --git a/ooni/lib/txtraceroute.py b/ooni/lib/txtraceroute.py
new file mode 100644
index 0000000..8182b34
--- /dev/null
+++ b/ooni/lib/txtraceroute.py
@@ -0,0 +1,752 @@
+#!/usr/bin/env python
+# coding: utf-8
+#
+# Copyright (c) 2012 Alexandre Fiori
+#                    Arturo Filastò
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import json
+import operator
+import os
+import socket
+import struct
+import sys
+import time
+import random
+import itertools
+from pprint import pprint
+
+from twisted.internet import defer
+from twisted.internet import reactor
+from twisted.internet import threads
+from twisted.python import usage
+from twisted.web.client import getPage
+
+class iphdr(object):
+    """
+    This represents an IP packet header.
+
+    XXX enable IP_TIMESTAMP in setsockopt
+        to get the timestamp of when the router says it has gotten an ICMP
+        timeout.
+
+    @assemble packages the packet
+    @disassemble disassembles the packet
+    """
+    def __init__(self, proto=socket.IPPROTO_ICMP, src="0.0.0.0", dst=None):
+        self.version = 4
+        self.hlen = 5
+        self.tos = 0
+        self.length = 20
+        self.id = os.getpid()
+        self.frag = 0
+        self.ttl = 255
+        self.proto = proto
+        self.cksum = 0
+        self.src = src
+        self.saddr = socket.inet_aton(src)
+        self.dst = dst or "0.0.0.0"
+        self.daddr = socket.inet_aton(self.dst)
+        self.data = ""
+
+    def assemble(self):
+        header = struct.pack('BBHHHBB',
+                             (self.version & 0x0f) << 4 | (self.hlen & 0x0f),
+                             self.tos, self.length + len(self.data),
+                             socket.htons(self.id), self.frag,
+                             self.ttl, self.proto)
+        self._raw = header + "\x00\x00" + self.saddr + self.daddr + self.data
+        return self._raw
+
+    @classmethod
+    def disassemble(self, data):
+        self._raw = data
+        ip = iphdr()
+        pkt = struct.unpack('!BBHHHBBH', data[:12])
+        ip.version = (pkt[0] >> 4 & 0x0f)
+        ip.hlen = (pkt[0] & 0x0f)
+        ip.tos, ip.length, ip.id, ip.frag, ip.ttl, ip.proto, ip.cksum = pkt[1:]
+        ip.saddr = data[12:16]
+        ip.daddr = data[16:20]
+        ip.src = socket.inet_ntoa(ip.saddr)
+        ip.dst = socket.inet_ntoa(ip.daddr)
+        return ip
+
+    def __repr__(self):
+        return "IP (tos %s, ttl %s, id %s, frag %s, proto %s, length %s) " \
+               "%s -> %s" % \
+               (self.tos, self.ttl, self.id, self.frag, self.proto,
+                self.length, self.src, self.dst)
+
+class tcphdr(object):
+    def __init__(self, data="", sport=4242, dport=4242):
+        self.seq = 0
+        self.hlen = 44
+        self.flags = 2
+        self.wsize = 200
+        self.cksum = 123
+        self.options = 0
+        self.mss = 1460
+        self.dport = dport
+        self.sport = sport
+
+    def assemble(self):
+        header = struct.pack("!HHL", self.sport, self.dport, self.seq)
+        header += '\x00\x00\x00\x00'
+        header += struct.pack("!HHH", (self.hlen & 0xff) << 10 | (self.flags &
+            0xff), self.wsize, self.cksum)
+        header += "\x00\x00"
+        options = '\x02\x04\x05\xb4\x01\x03\x03\x01\x01\x01\x08\x0a'
+        options += '\x4d\xcf\x52\x33\x00\x00\x00\x00\x04\x02\x00\x00'
+        # XXX There is something wrong here fixme
+        # options = struct.pack("!LBBBBBB", self.mss, 1, 3, 3, 1, 1, 1)
+        # options += struct.pack("!BBL", 8, 10, 1209452188)
+        # options += '\00'*4
+        # options += struct.pack("!BB", 4, 2)
+        # options += '\00'
+        self._raw = header+options
+        return self._raw
+
+    @classmethod
+    def checksum(self, data):
+        pass
+
+    def __repr__(self):
+        return "<TCPPacket (sport: %s dport: %s seq: %s) " %\
+               (self.sport, self.dport, self.seq)
+
+    @classmethod
+    def disassemble(self, data):
+        self._raw = data
+        tcp = tcphdr()
+        pkt = struct.unpack("!HHL", data[:8])
+        tcp.sport, tcp.dport, tcp.seq = pkt
+        if len(data) > 10:
+            pkt = struct.unpack("!H", data[8:10])
+            tcp.hlen = (pkt[0] >> 10 ) & 0xff
+            tcp.flags = pkt[0] & 0xff
+            tcp.wsize, tcp.cksum = struct.unpack("!HH", data[20:24])
+        return tcp
+
+class udphdr(object):
+    def __init__(self, data="", sport=4242, dport=4242):
+        self.dport = dport
+        self.sport = sport
+        self.cksum = 0
+        self.length = 0
+        self.data = data
+
+    def assemble(self):
+        self.length = len(self.data) + 8
+        part1 = struct.pack("!HHH", self.sport, self.dport, self.length)
+        cksum = self.checksum(self.data)
+        cksum = struct.pack("!H", cksum)
+
+        self._raw = part1 + cksum + self.data
+        return self._raw
+
+    @classmethod
+    def checksum(self, data):
+        # XXX implement proper checksum
+        cksum = 0
+        return cksum
+
+    def __repr__(self):
+        return "<UDPPacket (sport %s, dport %s, length %s, data %s)>" % \
+               (self.sport, self.dport, self.length, self.data)
+
+    @classmethod
+    def disassemble(self, data):
+        self._raw = data
+        udp = udphdr()
+        pkt = struct.unpack("!HHHH", data[:8])
+        udp.sport, udp.dport, udp.length, udp.cksum = pkt
+        udp.data = data[8:]
+        return udp
+
+class icmphdr(object):
+    def __init__(self, data=""):
+        self.type = 8
+        self.code = 0
+        self.cksum = 0
+        self.id = os.getpid()
+        self.sequence = 0
+        self.data = data
+
+    def assemble(self):
+        part1 = struct.pack("BB", self.type, self.code)
+        part2 = struct.pack("!HH", self.id, self.sequence)
+        cksum = self.checksum(part1 + "\x00\x00" + part2 + self.data)
+        cksum = struct.pack("!H", cksum)
+        self._raw = part1 + cksum + part2 + self.data
+        return self._raw
+
+    @classmethod
+    def checksum(self, data):
+        if len(data) & 1:
+            data += "\x00"
+        cksum = reduce(operator.add,
+                       struct.unpack('!%dH' % (len(data) >> 1), data))
+        cksum = (cksum >> 16) + (cksum & 0xffff)
+        cksum += (cksum >> 16)
+        cksum = (cksum & 0xffff) ^ 0xffff
+        return cksum
+
+    @classmethod
+    def disassemble(self, data):
+        self._raw = data
+        icmp = icmphdr()
+        pkt = struct.unpack("!BBHHH", data)
+        icmp.type, icmp.code, icmp.cksum, icmp.id, icmp.sequence = pkt
+        return icmp
+
+    def __repr__(self):
+        return "ICMP (type %s, code %s, id %s, sequence %s)" % \
+               (self.type, self.code, self.id, self.sequence)
+
+
+def pprintp(packet):
+    """
+    Used to pretty print packets.
+    """
+    lines = []
+    line = []
+    for i, byte in enumerate(packet):
+        line.append(("%.2x" % ord(byte), byte))
+        if (i + 1) % 8 == 0:
+            lines.append(line)
+            line = []
+
+    lines.append(line)
+
+    for row in lines:
+        left = ""
+        right = "   " * (8 - len(row))
+        for y in row:
+            left += "%s " % y[0]
+            right += "%s" % y[1]
+
+        print left + "     " + right
+
+@xxxxxxxxxxxxxxxxxxxxx
+def geoip_lookup(ip):
+    try:
+        r = yield getPage("http://freegeoip.net/json/%s"; % ip)
+        d = json.loads(r)
+        items = [d["country_name"], d["region_name"], d["city"]]
+        text = ", ".join([s for s in items if s])
+        defer.returnValue(text.encode("utf-8"))
+    except Exception:
+        defer.returnValue("Unknown location")
+
+
+@xxxxxxxxxxxxxxxxxxxxx
+def reverse_lookup(ip):
+    try:
+        r = yield threads.deferToThread(socket.gethostbyaddr, ip)
+        defer.returnValue(r[0])
+    except Exception:
+        defer.returnValue(None)
+
+
+class Hop(object):
+    def __init__(self, target, ttl, proto, sport=None, dport=None):
+        self.proto = proto
+        self.dport = dport
+        self.sport = sport
+
+        self.found = False
+        self.tries = 0
+        self.last_try = 0
+        self.remote_ip = None
+        self.remote_icmp = None
+        self.remote_host = None
+        self.location = ""
+
+        self.ttl = ttl
+        self.ip = iphdr(dst=target)
+        self.ip.ttl = ttl
+        self.ip.id += ttl
+        if self.proto == "icmp":
+            self.icmp = icmphdr('\x00'*20)
+            self.icmp.id = self.ip.id
+            self.ip.data = self.icmp.assemble()
+        elif self.proto == "udp":
+            self.udp = udphdr('\x00'*20, self.sport, self.dport)
+            self.ip.data = self.udp.assemble()
+            self.ip.proto = socket.IPPROTO_UDP
+        else:
+            self.tcp = tcphdr('\x42'*20, self.sport, self.dport)
+            self.ip.data = self.tcp.assemble()
+            self.ip.proto = socket.IPPROTO_TCP
+
+        self._pkt = self.ip.assemble()
+
+    @property
+    def pkt(self):
+        self.tries += 1
+        self.last_try = time.time()
+        return self._pkt
+
+    def get(self):
+        if self.found:
+            if self.remote_host:
+                ip = self.remote_host
+            else:
+                ip = self.remote_ip.src
+            ping = self.found - self.last_try
+        else:
+            ip = None
+            ping = None
+
+        location = self.location if self.location else None
+        return {'ttl': self.ttl, 'ping': ping, 'ip': ip, 'location': location,
+                'proto': self.proto, 'dport': self.dport, 'sport': self.sport}
+
+    def __repr__(self):
+        if self.found:
+            if self.remote_host:
+                ip = ":: %s" % self.remote_host
+            else:
+                ip = ":: %s" % self.remote_ip.src
+            ping = "%0.3fs" % (self.found - self.last_try)
+        else:
+            ip = "??"
+            ping = "-"
+
+        location = ":: %s" % self.location if self.location else ""
+        return "%02d. %s %s %s (%s, sport: %s dport: %s)" % (self.ttl, ping, ip, location, self.proto, self.sport, self.dport)
+
+class TracerouteResult(object):
+    """
+    Used to store the results of a Traceroute.
+    """
+    #src_ports = [0, 9090]
+    #dst_ports = [0, 21, 123, 80, 443]
+    src_ports = [0, 80]
+    dst_ports = [0, 80]
+    hops = []
+    done = False
+
+    def __init__(self, protocol):
+        self.protocol = protocol
+        self.probes = {}
+
+        if protocol == "icmp":
+            self.current = None
+        else:
+            self.current = {}
+            for src, dst in itertools.product(self.src_ports,
+                                              self.dst_ports):
+                if src not in self.probes:
+                    self.probes[src] = {}
+                self.probes[src][dst] = []
+
+                if src not in self.current:
+                    self.current[src] = {}
+                self.current[src][dst] = None
+
+    def get_current_probes(self):
+        if self.protocol == "icmp":
+            return self.current
+
+    def add_to_current_probes(self, probe):
+        if self.protocol == "icmp":
+            self.current = probe
+        else:
+            self.current[probe.sport][probe.dport] = probe
+
+    def is_in_progress(self):
+        if self.protocol == "icmp":
+            progress = self.current
+        else:
+            progress = None
+            for x in self.current:
+                for y in self.current[x]:
+                    if self.current[x][y] != None:
+                        progress = True
+        if progress is None:
+            return False
+        else:
+            return True
+
+    def get(self, src=None, dst=None):
+        if self.protocol == "icmp":
+            return self.probes
+        else:
+            return self.probes[src][dst]
+
+    def append(self, probe, src=None, dst=None):
+        if self.protocol == "icmp":
+            self.probes.append(hop)
+        else:
+            self.probes[src][dst].append(probe)
+
+    def pop(self, src=None, dst=None):
+        if self.protocol == "icmp":
+            hop = self.current
+            self.current = None
+            return hop
+
+        elif (dst != None) and (src != None):
+            hop = self.current[src][dst]
+            self.current[src][dst] = None
+            return hop
+
+        else:
+            raise Exception("Did not specify dst and src ports")
+
+    @classmethod
+    def hops(self, target, ttl):
+        """
+        Generates a set of ooni probes for traceroute based network tampering
+        detection.
+
+        We send in one round a set of packets with same TTL but on all protocols
+        and with all possible source and destination ports.
+        """
+        hops = []
+        for src, dst in itertools.product(self.src_ports, self.dst_ports):
+            hops.append(Hop(target, ttl,
+                            "tcp", src, dst))
+            hops.append(Hop(target, ttl,
+                            "udp", src, dst))
+        hops.append(Hop(target, ttl, "icmp", 0, 0))
+        return hops
+
+class TracerouteProtocol(object):
+    def __init__(self, target, **settings):
+
+        self.target = target
+        self.settings = settings
+        self.verbose = settings.get("verbose")
+        self.proto = settings.get("proto")
+        self.rfd = socket.socket(socket.AF_INET, socket.SOCK_RAW,
+                                 socket.IPPROTO_ICMP)
+        self.sfd = {}
+
+        # Create the data structures to contain the test results
+        self.traceroute = {}
+        self.traceroute["tcp"] = TracerouteResult("tcp")
+        self.traceroute["udp"] = TracerouteResult("udp")
+        self.traceroute["icmp"] = TracerouteResult("icmp")
+
+        if self.settings.get("ooni"):
+            self.sfd["tcp"] = socket.socket(socket.AF_INET, socket.SOCK_RAW,
+                                    socket.IPPROTO_TCP)
+            self.sfd["icmp"] = socket.socket(socket.AF_INET, socket.SOCK_RAW,
+                                    socket.IPPROTO_ICMP)
+            self.sfd["udp"] = socket.socket(socket.AF_INET, socket.SOCK_RAW,
+                                    socket.IPPROTO_UDP)
+        elif self.proto == "icmp":
+            self.sfd["icmp"] = socket.socket(socket.AF_INET, socket.SOCK_RAW,
+                                    socket.IPPROTO_ICMP)
+        elif self.proto == "udp":
+            self.sfd["udp"] = socket.socket(socket.AF_INET, socket.SOCK_RAW,
+                                    socket.IPPROTO_UDP)
+        elif self.proto == "tcp":
+            self.sfd["tcp"] = socket.socket(socket.AF_INET, socket.SOCK_RAW,
+                                    socket.IPPROTO_TCP)
+
+        # Let me add IP Headers myself, just give me a socket!
+        self.rfd.setsockopt(socket.IPPROTO_IP, socket.IP_HDRINCL, 1)
+        for fd in self.sfd:
+            self.sfd[fd].setsockopt(socket.IPPROTO_IP, socket.IP_HDRINCL, 1)
+
+        self.out_queue = []
+        self.waiting = True
+        self.deferred = defer.Deferred()
+
+        reactor.addReader(self)
+        reactor.addWriter(self)
+
+        # send 1st probe packet(s)
+        if self.settings.get("ooni"):
+            hops = list(TracerouteResult.hops(self.target, 1))
+        else:
+            hops = [Hop(self.target, 1,
+                        settings.get("proto"),
+                        self.settings.get("sport"),
+                        self.settings.get("dport"))]
+        for hop in hops:
+            # Store the to be completed items inside of a dictionary
+            self.traceroute[hop.proto].add_to_current_probes(hop)
+            self.out_queue.append(hop)
+
+    def logPrefix(self):
+        return "TracerouteProtocol(%s)" % self.target
+
+    def fileno(self):
+        return self.rfd.fileno()
+
+    @defer.inlineCallbacks
+    def hopFound(self, hop, ip, icmp, ref, subref):
+        hop.remote_ip = ip
+        hop.remote_icmp = icmp
+
+        if (ip and icmp):
+            hop.found = time.time()
+            if self.settings.get("geoip_lookup") is True:
+                hop.location = yield geoip_lookup(ip.src)
+
+            if self.settings.get("reverse_lookup") is True:
+                hop.remote_host = yield reverse_lookup(ip.src)
+
+        ttl = hop.ttl + 1
+
+        if (ttl > (self.settings.get("max_hops", 30) + 1)):
+            done = True
+        else:
+            done = False
+
+        if not done:
+            cb = self.settings.get("hop_callback")
+            if callable(cb):
+                yield defer.maybeDeferred(cb, hop)
+
+        if not self.waiting:
+            if self.deferred:
+                self.deferred.callback(self.hops)
+                self.deferred = None
+        else:
+            hops = []
+            if self.settings.get("ooni"):
+                if not (self.traceroute["icmp"].is_in_progress() or
+                        self.traceroute["tcp"].is_in_progress() or
+                        self.traceroute["udp"].is_in_progress()):
+                    # Add hops only if we are not in progress
+                    hops = list(TracerouteResult.hops(self.target, ttl))
+            else:
+                hops = [Hop(self.target, ttl,
+                            settings.get("proto"),
+                            self.settings.get("sport"),
+                            self.settings.get("dport"))]
+
+            for hop in hops:
+                # Store the to be completed items inside of a dictionary
+                self.traceroute[hop.proto].add_to_current_probes(hop)
+                self.out_queue.append(hop)
+
+    def doRead(self):
+        if not self.waiting:
+            return
+
+        pkt = self.rfd.recv(4096)
+        # disassemble ip header
+        ip = iphdr.disassemble(pkt[:20])
+
+        if self.verbose:
+            print "Got this packet:"
+            print "src %s" % ip.src
+            pprintp(pkt)
+
+        # Not interested in non ICMP packets.
+        if ip.proto != socket.IPPROTO_ICMP:
+            return
+
+        found = False
+        foundHop = None
+
+        # disassemble icmp header
+        icmp = icmphdr.disassemble(pkt[20:28])
+
+        if self.verbose:
+            print icmp
+
+        # If it's an ICMP Echo reply then our ICMP probe has hit destination
+        if icmp.type == 0 and icmp.id == self.current_hop["icmp"][1].icmp.id:
+            foundHop = self.traceroute["icmp"].pop()
+            found = True
+
+        elif icmp.type == 11:
+            # disassemble referenced ip header
+            ref = iphdr.disassemble(pkt[28:48])
+            subref = None
+
+            if self.verbose:
+                print ref
+
+            if ref.dst == self.target:
+                found = True
+
+            if ref.proto == socket.IPPROTO_UDP:
+                subref = udphdr.disassemble(pkt[48:])
+                proto = "udp"
+
+            elif ref.proto == socket.IPPROTO_TCP:
+                subref = tcphdr.disassemble(pkt[48:])
+                proto = "tcp"
+
+            else:
+                proto = "icmp"
+
+            if subref:
+                sport = subref.sport
+                dport = subref.dport
+            else:
+                sport = None
+                dport = None
+            # Remove completed hops
+            foundHop = self.traceroute[proto].pop(sport,
+                                                  dport)
+
+        if ip.src == self.target:
+            self.waiting = False
+
+        if found:
+            self.hopFound(foundHop, ip, icmp, ref, subref)
+        elif foundHop:
+            self.hopFound(foundHop, ip, icmp, ref, subref)
+
+    def hopTimeout(self, hop):
+        if not hop.found:
+            if hop.tries < self.settings.get("max_tries", 3):
+                # retry
+                hop.tries += 1
+                self.out_queue.append(hop)
+            else:
+                # give up and move forward
+                self.traceroute[hop.proto].pop(hop.dport,
+                                               hop.sport)
+                self.hopFound(hop, None, None, None, None)
+
+    def doWrite(self):
+        if self.waiting and self.out_queue:
+            hop = self.out_queue.pop(0)
+            pkt = hop.pkt
+            if self.verbose:
+                print "Sending this packet:"
+                pprintp(pkt)
+                print hop
+
+            self.sfd[hop.proto].sendto(pkt, (hop.ip.dst, 0))
+
+            self.traceroute[hop.proto].add_to_current_probes(hop)
+
+            timeout = self.settings.get("timeout", 1)
+            reactor.callLater(timeout, self.hopTimeout, hop)
+
+    def connectionLost(self, why):
+        pass
+
+
+def traceroute(target, **settings):
+    tr = TracerouteProtocol(target, **settings)
+    return tr.deferred
+
+
+@xxxxxxxxxxxxxxxxxxxxx
+def start_trace(target, **settings):
+    hops = yield traceroute(target, **settings)
+    if settings["hop_callback"] is None:
+        for hop in hops:
+            print hop
+    reactor.stop()
+
+class Options(usage.Options):
+    optFlags = [
+        ["quiet", "q", "Only print results at the end."],
+        ["no-dns", "n", "Show numeric IPs only, not their host names."],
+        ["no-geoip", "g", "Do not collect and show GeoIP information"],
+        ["verbose", "v", "Be more verbose"],
+        ["ooni", "o", "Run the ooni common port multiprotocol traceroute"],
+        ["help", "h", "Show this help"],
+    ]
+    optParameters = [
+        ["timeout", "t", 2, "Timeout for probe packets"],
+        ["tries", "r", 3, "How many tries before give up probing a hop"],
+        ["proto", "p", "icmp", "What protocol to use (tcp, udp, icmp)"],
+        ["dport", "d", random.randint(2**10, 2**16), "Destination port (TCP and UDP only)"],
+        ["sport", "s", random.randint(2**10, 2**16), "Source port (TCP and UDP only)"],
+        ["max_hops", "m", 30, "Max number of hops to probe"]
+    ]
+
+def main():
+    def show(hop):
+        print hop
+
+    defaults = dict(hop_callback=show,
+                    reverse_lookup=True,
+                    geoip_lookup=True,
+                    timeout=2,
+                    proto="icmp",
+                    dport=None,
+                    sport=None,
+                    verbose=False,
+                    ooni=False,
+                    max_tries=3,
+                    max_hops=30)
+
+    if len(sys.argv) < 2:
+        print("Usage: %s [options] host" % (sys.argv[0]))
+        print("%s: Try --help for usage details." % (sys.argv[0]))
+        sys.exit(1)
+
+    target = sys.argv.pop(-1) if sys.argv[-1][0] != "-" else ""
+    config = Options()
+    try:
+        config.parseOptions()
+        if not target:
+            raise
+    except usage.UsageError, e:
+        print("%s: %s" % (sys.argv[0], e))
+        print("%s: Try --help for usage details." % (sys.argv[0]))
+        sys.exit(1)
+
+    settings = defaults.copy()
+    if config.get("silent"):
+        settings["hop_callback"] = None
+    if config.get("no-dns"):
+        settings["reverse_lookup"] = False
+    if config.get("no-geoip"):
+        settings["geoip_lookup"] = False
+    if config.get("verbose"):
+        settings["verbose"] = True
+    if config.get("ooni"):
+        settings["ooni"] = True
+    if "timeout" in config:
+        settings["timeout"] = config["timeout"]
+    if "tries" in config:
+        settings["max_tries"] = config["tries"]
+    if "proto" in config:
+        settings["proto"] = config["proto"]
+    if "max_hops" in config:
+        settings["max_hops"] = config["max_hops"]
+    if "dport" in config:
+        settings["dport"] = int(config["dport"])
+    if "sport" in config:
+        settings["sport"] = int(config["sport"])
+
+    if os.getuid() != 0:
+        print("traceroute needs root privileges for the raw socket")
+        sys.exit(1)
+    try:
+        target = socket.gethostbyname(target)
+    except Exception, e:
+        print("could not resolve '%s': %s" % (target, str(e)))
+        sys.exit(1)
+
+    reactor.callWhenRunning(start_trace, target, **settings)
+    reactor.run()
+
+if __name__ == "__main__":
+    main()
+
diff --git a/ooni/ooniprobe.py b/ooni/ooniprobe.py
index 95b5a18..539c2ac 100755
--- a/ooni/ooniprobe.py
+++ b/ooni/ooniprobe.py
@@ -27,9 +27,11 @@ from zope.interface.verify import verifyObject
 from zope.interface.exceptions import BrokenImplementation
 from zope.interface.exceptions import BrokenMethodImplementation
 
-from ooni.plugoo import tests, work, assets, reports
-from ooni.logo import getlogo
-from ooni import plugins, log
+from plugoo import tests, work, assets, reports
+from plugoo.interface import ITest
+from utils.logo import getlogo
+from utils import log
+import plugins
 
 __version__ = "0.0.1-prealpha"
 
@@ -38,7 +40,7 @@ def retrieve_plugoo():
     Get all the plugins that implement the ITest interface and get the data
     associated to them into a dict.
     """
-    interface = tests.ITest
+    interface = ITest
     d = {}
     error = False
     for p in getPlugins(interface, plugins):
diff --git a/ooni/plugins/blocking.py b/ooni/plugins/blocking.py
index 72fd49f..f3c20e1 100644
--- a/ooni/plugins/blocking.py
+++ b/ooni/plugins/blocking.py
@@ -3,7 +3,7 @@ from twisted.python import usage
 from twisted.plugin import IPlugin
 
 from plugoo.assets import Asset
-from plugoo.tests import ITest, TwistedTest
+from plugoo.tests import ITest, OONITest
 
 class BlockingArgs(usage.Options):
     optParameters = [['asset', 'a', None, 'Asset file'],
diff --git a/ooni/plugins/dnstamper.py b/ooni/plugins/dnstamper.py
index 54cfbdf..3d373ce 100644
--- a/ooni/plugins/dnstamper.py
+++ b/ooni/plugins/dnstamper.py
@@ -38,9 +38,9 @@ from twisted.python import usage
 from twisted.plugin import IPlugin
 from zope.interface import implements
 
-from ooni.plugoo.assets import Asset
-from ooni.plugoo.tests import ITest, OONITest
-from ooni import log
+from plugoo.assets import Asset
+from plugoo.tests import ITest, OONITest
+from utils import log
 
 class AlexaAsset(Asset):
     """
diff --git a/ooni/plugins/httphost.py b/ooni/plugins/httphost.py
index e8808bd..7c783a1 100644
--- a/ooni/plugins/httphost.py
+++ b/ooni/plugins/httphost.py
@@ -21,7 +21,7 @@ from twisted.python import usage
 from twisted.plugin import IPlugin
 
 from plugoo.assets import Asset
-from plugoo.tests import ITest, TwistedTest
+from plugoo.tests import ITest, OONITest
 
 class HTTPHostArgs(usage.Options):
     optParameters = [['asset', 'a', None, 'Asset file'],
diff --git a/ooni/plugins/tcpconnect.py b/ooni/plugins/tcpconnect.py
index db3d969..bbf62a5 100644
--- a/ooni/plugins/tcpconnect.py
+++ b/ooni/plugins/tcpconnect.py
@@ -9,8 +9,9 @@ from twisted.plugin import IPlugin
 from twisted.internet.protocol import Factory, Protocol
 from twisted.internet.endpoints import TCP4ClientEndpoint
 
-from ooni.plugoo.tests import ITest, OONITest
-from ooni.plugoo.assets import Asset
+from plugoo.interface import ITest
+from plugoo.tests import OONITest
+from plugoo.assets import Asset
 from ooni.utils import log
 
 class tcpconnectArgs(usage.Options):
diff --git a/ooni/plugoo/nodes.py b/ooni/plugoo/nodes.py
index 0d01348..155f183 100644
--- a/ooni/plugoo/nodes.py
+++ b/ooni/plugoo/nodes.py
@@ -7,7 +7,7 @@
     This contains all the code related to Nodes
     both network and code execution.
 
-    :copyright: (c) 2012 by Arturo Filastò.
+    :copyright: (c) 2012 by Arturo Filastò, Isis Lovecruft
     :license: see LICENSE for more details.
 
 """
@@ -20,10 +20,6 @@ try:
 except:
     print "Error: module paramiko is not installed."
 from pprint import pprint
-try:
-    import pyXMLRPCssh
-except:
-    print "Error: module pyXMLRPCssh is not installed."
 import sys
 import socks
 import xmlrpclib
diff --git a/ooni/plugoo/reports.py b/ooni/plugoo/reports.py
index ef15a04..d0d9af3 100644
--- a/ooni/plugoo/reports.py
+++ b/ooni/plugoo/reports.py
@@ -4,7 +4,7 @@ import os
 import yaml
 
 import itertools
-import log
+from utils import log, date, net
 
 class Report:
     """This is the ooni-probe reporting mechanism. It allows
diff --git a/ooni/plugoo/tests.py b/ooni/plugoo/tests.py
index 19d42b2..42f9542 100644
--- a/ooni/plugoo/tests.py
+++ b/ooni/plugoo/tests.py
@@ -8,9 +8,10 @@ from twisted.internet import reactor, defer, threads
 ## XXX why is this imported and not used?
 from twisted.python import failure
 
-from ooni import log
+from utils import log, date
 from plugoo import assets, work
 from plugoo.reports import Report
+from plugoo.interface import ITest
 
 
 class OONITest(object):
diff --git a/ooni/utils/log.py b/ooni/utils/log.py
index cf57186..dd5cf13 100644
--- a/ooni/utils/log.py
+++ b/ooni/utils/log.py
@@ -93,7 +93,6 @@ def debug(message, level="debug", **kw):
 def msg(message, level="info", **kw):
     log.msg(message, logLevel=level, **kw)
 
-## XXX fixme log.err messages get printed to stdout twice
 def err(message, level="err", **kw):
     log.err(message, logLevel=level, **kw)
 

_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits