[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[tor-commits] [flashproxy/master] move fac.py and associated tests to flashproxy-common
commit 5d3ae963fa6b514c28e5ad37e60a165d82526831
Author: Ximin Luo <infinity0@xxxxxxx>
Date: Wed Nov 13 16:34:31 2013 +0000
move fac.py and associated tests to flashproxy-common
- use the facilitator's parse_addr_spec, and fix client uses to set defhost = ""
- the old client side accepted an empty host when defhost = None.
- the facilitator side requires setting defhost = "" to accept an empty host
- behaviour is otherwise equivalent
---
Makefile | 2 +-
facilitator/HACKING | 26 ++
facilitator/INSTALL | 15 +-
facilitator/Makefile.am | 10 +-
facilitator/doc/facilitator-design.txt | 3 -
facilitator/fac.py | 404 --------------------------------
facilitator/facilitator | 22 +-
facilitator/facilitator-email-poller | 10 +-
facilitator/facilitator-reg-daemon | 16 +-
facilitator/facilitator-test.py | 143 +----------
facilitator/facilitator.cgi | 2 +-
flashproxy-client | 12 +-
flashproxy-reg-appspot | 2 +-
flashproxy-reg-email | 2 +-
flashproxy-reg-url | 2 +-
flashproxy/fac.py | 224 ++++++++++++++++++
flashproxy/proc.py | 47 ++++
flashproxy/reg.py | 31 +++
flashproxy/test/test_fac.py | 93 ++++++++
flashproxy/test/test_reg.py | 23 ++
flashproxy/test/test_util.py | 35 +++
flashproxy/util.py | 92 ++++++--
22 files changed, 618 insertions(+), 598 deletions(-)
diff --git a/Makefile b/Makefile
index 9e8d2dd..93287f6 100644
--- a/Makefile
+++ b/Makefile
@@ -84,7 +84,7 @@ test-full: test
cd facilitator && \
{ test -x ./config.status && ./config.status || \
{ test -x ./configure || ./autogen.sh; } && ./configure; } \
- && make && make check
+ && make && PYTHONPATH=.. make check
cd proxy && make test
force-dist:
diff --git a/facilitator/HACKING b/facilitator/HACKING
new file mode 100644
index 0000000..f102880
--- /dev/null
+++ b/facilitator/HACKING
@@ -0,0 +1,26 @@
+== Running from source checkout
+
+In order to run the code directly from a source checkout, you must make sure it
+can find the flashproxy module, located in the top-level directory of the
+source checkout, which is probably the parent directory. You have two options:
+
+1. Install it in "development mode", see [1]
+
+ flashproxy# python setup-common.py develop
+
+This process is reversible too:
+
+ flashproxy# python setup-common.py develop --uninstall
+
+The disadvantage is that other programs (such as a system-installed flashproxy,
+or other checkouts in another directory) will see this development copy, rather
+than a more appropriate copy.
+
+2. Export PYTHONPATH when you need to run
+
+ $ export PYTHONPATH=..
+ $ make check
+
+The disadvantage is that you need to do this every shell session.
+
+[1] http://pythonhosted.org/distribute/setuptools.html#development-mode
diff --git a/facilitator/INSTALL b/facilitator/INSTALL
index 6325e53..7716ccc 100644
--- a/facilitator/INSTALL
+++ b/facilitator/INSTALL
@@ -1,12 +1,21 @@
Install the dependencies.
- $ apt-get install make openssl python-m2crypto
- $ apt-get install automake autoconf # if running from git
- $ apt-get install apache2
+ # apt-get install make openssl python-m2crypto
+ # apt-get install automake autoconf # if running from git
+
+ # apt-get install apache2
You may use a different webserver, but currently we only provide an apache2 site
config example, so you will need to adapt this to the correct syntax.
+ # apt-get install flashproxy-common
+
+If your distro does not have flashproxy-common, you can install it
+directly from the top-level source directory:
+
+ flashproxy# python setup-common.py install --record install.log \
+ --single-version-externally-managed
+
Configure and install.
$ ./autogen.sh # if running from git or ./configure doesn't otherwise exist
diff --git a/facilitator/Makefile.am b/facilitator/Makefile.am
index 2d315b9..435956d 100644
--- a/facilitator/Makefile.am
+++ b/facilitator/Makefile.am
@@ -2,9 +2,7 @@
fpfacilitatoruser = @fpfacilitatoruser@
initconfdir = @initconfdir@
-# TODO(infinity0): switch this to @cgibindir@ once we replace fac.py with
-# flashproxy-common, so that we install facilitator.cgi in the right place
-cgibindir = @bindir@
+cgibindir = @cgibindir@
# unfortunately sysvinit does not support having initscripts in /usr/local/etc
# yet, so we have to hard code a path here. :(
@@ -16,7 +14,7 @@ appengineconfdir = $(pkgconfdir)/reg-appspot
# automake PLVs
-dist_bin_SCRIPTS = facilitator facilitator-email-poller facilitator-reg-daemon facilitator-reg fac.py
+dist_bin_SCRIPTS = facilitator facilitator-email-poller facilitator-reg-daemon facilitator-reg
dist_cgibin_SCRIPTS = facilitator.cgi
if DO_INITSCRIPTS
initscript_SCRIPTS = init.d/facilitator init.d/facilitator-email-poller init.d/facilitator-reg-daemon
@@ -29,13 +27,13 @@ pkgconf_DATA = examples/facilitator-relays
dist_appengine_DATA = appengine/app.yaml appengine/config.go appengine/fp-reg.go
appengineconf_DATA = appengine/config.go
CLEANFILES = examples/fp-facilitator.conf
-EXTRA_DIST = examples/fp-facilitator.conf.in $(TESTS)
+EXTRA_DIST = examples/fp-facilitator.conf.in HACKING $(TESTS)
TESTS = facilitator-test.py
# see http://www.gnu.org/software/automake/manual/html_node/Parallel-Test-Harness.html#index-TEST_005fEXTENSIONS
TEST_EXTENSIONS = .py
PY_LOG_COMPILER = $(PYTHON)
-AM_TESTS_ENVIRONMENT = PYTHONPATH='$(srcdir)'; export PYTHONPATH;
+AM_TESTS_ENVIRONMENT = PYTHONPATH='$(srcdir):$(PYTHONPATH)'; export PYTHONPATH;
AM_PY_LOG_FLAGS =
# AC_CONFIG_FILES doesn't fully-expand directory variables
diff --git a/facilitator/doc/facilitator-design.txt b/facilitator/doc/facilitator-design.txt
index 0d84da3..20f9c0a 100644
--- a/facilitator/doc/facilitator-design.txt
+++ b/facilitator/doc/facilitator-design.txt
@@ -39,6 +39,3 @@ the HTTP method, either yours or that of another facilitator. It takes
advantage of the fact that a censor cannot distinguish between a TLS
connection to appspot.com or google.com, since the IPs are the same,
and it is highly unlikely that anyone will try to block the latter.
-
-fac.py is a Python module containing code common to the various
-facilitator programs.
diff --git a/facilitator/fac.py b/facilitator/fac.py
deleted file mode 100644
index 00c103c..0000000
--- a/facilitator/fac.py
+++ /dev/null
@@ -1,404 +0,0 @@
-import errno
-import os
-import re
-import socket
-import stat
-import subprocess
-import pwd
-import urlparse
-from collections import namedtuple
-
-DEFAULT_CLIENT_TRANSPORT = "websocket"
-
-# Return true iff the given fd is readable, writable, and executable only by its
-# owner.
-def check_perms(fd):
- mode = os.fstat(fd)[0]
- return (mode & (stat.S_IRWXG | stat.S_IRWXO)) == 0
-
-# Drop privileges by switching ID to that of the given user.
-# http://stackoverflow.com/questions/2699907/dropping-root-permissions-in-python/2699996#2699996
-# https://www.securecoding.cert.org/confluence/display/seccode/POS36-C.+Observe+correct+revocation+order+while+relinquishing+privileges
-# https://www.securecoding.cert.org/confluence/display/seccode/POS37-C.+Ensure+that+privilege+relinquishment+is+successful
-def drop_privs(username):
- uid = pwd.getpwnam(username).pw_uid
- gid = pwd.getpwnam(username).pw_gid
- os.setgroups([])
- os.setgid(gid)
- os.setuid(uid)
- try:
- os.setuid(0)
- except OSError:
- pass
- else:
- raise AssertionError("setuid(0) succeeded after attempting to drop privileges")
-
-# A decorator to ignore "broken pipe" errors.
-def catch_epipe(fn):
- def ret(self, *args):
- try:
- return fn(self, *args)
- except socket.error, e:
- try:
- err_num = e.errno
- except AttributeError:
- # Before Python 2.6, exception can be a pair.
- err_num, errstr = e
- except:
- raise
- if err_num != errno.EPIPE:
- raise
- return ret
-
-def parse_addr_spec(spec, defhost = None, defport = None):
- """Parse a host:port specification and return a 2-tuple ("host", port) as
- understood by the Python socket functions.
- >>> parse_addr_spec("192.168.0.1:9999")
- ('192.168.0.1', 9999)
-
- If defhost or defport are given, those parts of the specification may be
- omitted; if so, they will be filled in with defaults.
- >>> parse_addr_spec("192.168.0.2:8888", defhost="192.168.0.1", defport=9999)
- ('192.168.0.2', 8888)
- >>> parse_addr_spec(":8888", defhost="192.168.0.1", defport=9999)
- ('192.168.0.1', 8888)
- >>> parse_addr_spec("192.168.0.2", defhost="192.168.0.1", defport=9999)
- ('192.168.0.2', 9999)
- >>> parse_addr_spec("192.168.0.2:", defhost="192.168.0.1", defport=9999)
- ('192.168.0.2', 9999)
- >>> parse_addr_spec(":", defhost="192.168.0.1", defport=9999)
- ('192.168.0.1', 9999)
- >>> parse_addr_spec("", defhost="192.168.0.1", defport=9999)
- ('192.168.0.1', 9999)
- IPv6 addresses must be enclosed in square brackets."""
- host = None
- port = None
- af = 0
- m = None
- # IPv6 syntax.
- if not m:
- m = re.match(ur'^\[(.+)\]:(\d*)$', spec)
- if m:
- host, port = m.groups()
- af = socket.AF_INET6
- if not m:
- m = re.match(ur'^\[(.+)\]$', spec)
- if m:
- host, = m.groups()
- af = socket.AF_INET6
- # IPv4/hostname/port-only syntax.
- if not m:
- try:
- host, port = spec.split(":", 1)
- except ValueError:
- host = spec
- if re.match(ur'^[\d.]+$', host):
- af = socket.AF_INET
- else:
- af = 0
- host = host or defhost
- port = port or defport
- if host is None or port is None:
- raise ValueError("Bad address specification \"%s\"" % spec)
- return host, int(port)
-
-def resolve_to_ip(host, port, af=0, gai_flags=0):
- """Resolves a host string to an IP address in canonical format.
-
- Note: in many cases this is not necessary since the consumer of the address
- can probably accept host names directly.
-
- :param: host string to resolve; may be a DNS name or an IP address.
- :param: port of the host
- :param: af address family, default unspecified. set to socket.AF_INET or
- socket.AF_INET6 to force IPv4 or IPv6 name resolution.
- :returns: (IP address in canonical format, port)
- """
- # Forward-resolve the name into an addrinfo struct. Real DNS resolution is
- # done only if resolve is true; otherwise the address must be numeric.
- try:
- addrs = socket.getaddrinfo(host, port, af, 0, 0, gai_flags)
- except socket.gaierror, e:
- raise ValueError("Bad host or port: \"%s\" \"%s\": %s" % (host, port, str(e)))
- if not addrs:
- raise ValueError("Bad host or port: \"%s\" \"%s\"" % (host, port))
-
- # Convert the result of socket.getaddrinfo (which is a 2-tuple for IPv4 and
- # a 4-tuple for IPv6) into a (host, port) 2-tuple.
- host, port = socket.getnameinfo(addrs[0][4], socket.NI_NUMERICHOST | socket.NI_NUMERICSERV)
- return host, int(port)
-
-def canonical_ip(host, port, af=0):
- """Convert an IP address to a canonical format. Identical to resolve_to_ip,
- except that the host param must already be an IP address."""
- return resolve_to_ip(host, port, af, gai_flags=socket.AI_NUMERICHOST)
-
-def format_addr(addr):
- host, port = addr
- host_str = u""
- port_str = u""
- if host is not None:
- # Numeric IPv6 address?
- try:
- addrs = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM, socket.IPPROTO_TCP, socket.AI_NUMERICHOST)
- af = addrs[0][0]
- except socket.gaierror, e:
- af = 0
- if af == socket.AF_INET6:
- host_str = u"[%s]" % host
- else:
- host_str = u"%s" % host
- if port is not None:
- if not (0 < port <= 65535):
- raise ValueError("port must be between 1 and 65535 (is %d)" % port)
- port_str = u":%d" % port
-
- if not host_str and not port_str:
- raise ValueError("host and port may not both be None")
- return u"%s%s" % (host_str, port_str)
-
-def read_client_registrations(body, defhost=None, defport=None):
- """Yield client registrations (as Endpoints) from an encoded registration
- message body. The message format is one registration per line, with each
- line being encoded as application/x-www-form-urlencoded. The key "client" is
- required and contains the client address and port (perhaps filled in by
- defhost and defport). The key "client-transport" is optional and defaults to
- "websocket".
- Example:
- client=1.2.3.4:9000&client-transport=websocket
- client=1.2.3.4:9090&client-transport=obfs3|websocket
- """
- for line in body.splitlines():
- qs = urlparse.parse_qs(line, keep_blank_values=True, strict_parsing=True)
- # Get the unique value associated with the given key in qs. If the key
- # is absent or appears more than once, raise ValueError.
- def get_unique(key, default=None):
- try:
- vals = qs[key]
- except KeyError:
- if default is None:
- raise ValueError("missing %r key" % key)
- vals = (default,)
- if len(vals) != 1:
- raise ValueError("more than one %r key" % key)
- return vals[0]
- addr = parse_addr_spec(get_unique("client"), defhost, defport)
- transport = get_unique("client-transport", DEFAULT_CLIENT_TRANSPORT)
- yield Endpoint(addr, transport)
-
-
-class Transport(namedtuple("Transport", "inner outer")):
- @classmethod
- def parse(cls, transport):
- if isinstance(transport, cls):
- return transport
- elif type(transport) == str:
- if "|" in transport:
- inner, outer = transport.rsplit("|", 1)
- else:
- inner, outer = "", transport
- return cls(inner, outer)
- else:
- raise ValueError("could not parse transport: %s" % transport)
-
- def __init__(self, inner, outer):
- if not outer:
- raise ValueError("outer (proxy) part of transport must be non-empty: %s" % str(self))
-
- def __str__(self):
- return "%s|%s" % (self.inner, self.outer) if self.inner else self.outer
-
-
-class Endpoint(namedtuple("Endpoint", "addr transport")):
- @classmethod
- def parse(cls, spec, transport, defhost = None, defport = None):
- host, port = parse_addr_spec(spec, defhost, defport)
- return cls((host, port), Transport.parse(transport))
-
-
-def skip_space(pos, line):
- """Skip a (possibly empty) sequence of space characters (the ASCII character
- '\x20' exactly). Returns a pair (pos, num_skipped)."""
- begin = pos
- while pos < len(line) and line[pos] == "\x20":
- pos += 1
- return pos, pos - begin
-
-TOKEN_CHARS = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-")
-def get_token(pos, line):
- begin = pos
- while pos < len(line) and line[pos] in TOKEN_CHARS:
- pos += 1
- if begin == pos:
- raise ValueError("No token found at position %d" % pos)
- return pos, line[begin:pos]
-
-def get_quoted_string(pos, line):
- chars = []
- if not (pos < len(line) and line[pos] == '"'):
- raise ValueError("Expected '\"' at beginning of quoted string.")
- pos += 1
- while pos < len(line) and line[pos] != '"':
- if line[pos] == '\\':
- pos += 1
- if not (pos < len(line)):
- raise ValueError("End of line after backslash in quoted string")
- chars.append(line[pos])
- pos += 1
- if not (pos < len(line) and line[pos] == '"'):
- raise ValueError("Expected '\"' at end of quoted string.")
- pos += 1
- return pos, "".join(chars)
-
-def parse_transaction(line):
- """A transaction is a command followed by zero or more key-value pairs. Like so:
- COMMAND KEY="VALUE" KEY="\"ESCAPED\" VALUE"
- Values must be quoted. Any byte value may be escaped with a backslash.
- Returns a pair: (COMMAND, ((KEY1, VALUE1), (KEY2, VALUE2), ...)).
- """
- pos = 0
- pos, skipped = skip_space(pos, line)
- pos, command = get_token(pos, line)
-
- pairs = []
- while True:
- pos, skipped = skip_space(pos, line)
- if not (pos < len(line)):
- break
- if skipped == 0:
- raise ValueError("Expected space before key-value pair")
- pos, key = get_token(pos, line)
- if not (pos < len(line) and line[pos] == '='):
- raise ValueError("No '=' found after key")
- pos += 1
- pos, value = get_quoted_string(pos, line)
- pairs.append((key, value))
- return command, tuple(pairs)
-
-def param_first(key, params):
- """Search 'params' for 'key' and return the first value that
- occurs. If 'key' was not found, return None."""
- for k, v in params:
- if key == k:
- return v
- return None
-
-def param_getlist(key, params):
- """Search 'params' for 'key' and return a list with its values. If
- 'key' did not appear in 'params', return the empty list."""
- result = []
- for k, v in params:
- if key == k:
- result.append(v)
- return result
-
-def quote_string(s):
- chars = []
- for c in s:
- if c == "\\":
- c = "\\\\"
- elif c == "\"":
- c = "\\\""
- chars.append(c)
- return "\"" + "".join(chars) + "\""
-
-def render_transaction(command, *params):
- parts = [command]
- for key, value in params:
- parts.append("%s=%s" % (key, quote_string(value)))
- return " ".join(parts)
-
-def fac_socket(facilitator_addr):
- return socket.create_connection(facilitator_addr, 1.0).makefile()
-
-def transact(f, command, *params):
- transaction = render_transaction(command, *params)
- print >> f, transaction
- f.flush()
- line = f.readline()
- if not (len(line) > 0 and line[-1] == '\n'):
- raise ValueError("No newline at end of string returned by facilitator")
- return parse_transaction(line[:-1])
-
-def put_reg(facilitator_addr, client_addr, transport):
- """Send a registration to the facilitator using a one-time socket. Returns
- true iff the command was successful. transport is a transport string such as
- "websocket" or "obfs3|websocket"."""
- f = fac_socket(facilitator_addr)
- params = [("CLIENT", format_addr(client_addr))]
- params.append(("TRANSPORT", transport))
- try:
- command, params = transact(f, "PUT", *params)
- finally:
- f.close()
- return command == "OK"
-
-def get_reg(facilitator_addr, proxy_addr, proxy_transport_list):
- """
- Get a client registration for proxy proxy_addr from the
- facilitator at facilitator_addr using a one-time
- socket. proxy_transport_list is a list containing the transport names that
- the flashproxy supports.
-
- Returns a dict with keys "client", "client-transport", "relay",
- and "relay-transport" if successful, or a dict with the key "client"
- mapped to the value "" if there are no registrations available for
- proxy_addr. Raises an exception otherwise."""
- f = fac_socket(facilitator_addr)
-
- # Form a list (in transact() format) with the transports that we
- # should send to the facilitator. Then pass that list to the
- # transact() function.
- # For example, PROXY-TRANSPORT=obfs2 PROXY-TRANSPORT=obfs3.
- transports = [("PROXY-TRANSPORT", tp) for tp in proxy_transport_list]
-
- try:
- command, params = transact(f, "GET", ("FROM", format_addr(proxy_addr)), *transports)
- finally:
- f.close()
- response = {}
- check_back_in = param_first("CHECK-BACK-IN", params)
- if check_back_in is not None:
- try:
- float(check_back_in)
- except ValueError:
- raise ValueError("Facilitator returned non-numeric polling interval.")
- response["check-back-in"] = check_back_in
- if command == "NONE":
- response["client"] = ""
- return response
- elif command == "OK":
- client_spec = param_first("CLIENT", params)
- client_transport = param_first("CLIENT-TRANSPORT", params)
- relay_spec = param_first("RELAY", params)
- relay_transport = param_first("RELAY-TRANSPORT", params)
- if not client_spec:
- raise ValueError("Facilitator did not return CLIENT")
- if not client_transport:
- raise ValueError("Facilitator did not return CLIENT-TRANSPORT")
- if not relay_spec:
- raise ValueError("Facilitator did not return RELAY")
- if not relay_transport:
- raise ValueError("Facilitator did not return RELAY-TRANSPORT")
- # Check the syntax returned by the facilitator.
- client = parse_addr_spec(client_spec)
- relay = parse_addr_spec(relay_spec)
- response["client"] = format_addr(client)
- response["client-transport"] = client_transport
- response["relay"] = format_addr(relay)
- response["relay-transport"] = relay_transport
- return response
- else:
- raise ValueError("Facilitator response was not \"OK\"")
-
-def put_reg_base64(b64):
- """Attempt to add a registration by running a facilitator-reg program
- locally."""
- # Padding is optional, but the python base64 functions can't
- # handle lack of padding. Add it here. Assumes correct encoding.
- mod = len(b64) % 4
- if mod != 0:
- b64 += (4 - mod) * "="
- p = subprocess.Popen(["facilitator-reg"], stdin=subprocess.PIPE)
- stdout, stderr = p.communicate(b64)
- return p.returncode == 0
diff --git a/facilitator/facilitator b/facilitator/facilitator
index a2dd56a..39ffe2a 100755
--- a/facilitator/facilitator
+++ b/facilitator/facilitator
@@ -9,8 +9,10 @@ import threading
import time
from collections import defaultdict
-import fac
-from fac import Transport, Endpoint
+from flashproxy import fac
+from flashproxy import proc
+from flashproxy.reg import Transport, Endpoint
+from flashproxy.util import parse_addr_spec, format_addr, canonical_ip
LISTEN_ADDRESS = "127.0.0.1"
DEFAULT_LISTEN_PORT = 9002
@@ -241,7 +243,7 @@ class Handler(SocketServer.StreamRequestHandler):
if buflen >= READLINE_MAX_LENGTH:
raise socket.error("readline: refusing to buffer %d bytes (last read was %d bytes)" % (buflen, len(data)))
- @fac.catch_epipe
+ @proc.catch_epipe
def handle(self):
num_lines = 0
while True:
@@ -290,7 +292,7 @@ class Handler(SocketServer.StreamRequestHandler):
if proxy_spec is None:
return self.error(u"GET missing FROM param")
try:
- proxy_addr = fac.canonical_ip(*fac.parse_addr_spec(proxy_spec, defport=0))
+ proxy_addr = canonical_ip(*parse_addr_spec(proxy_spec, defport=0))
except ValueError, e:
return self.error(u"syntax error in proxy address %s: %s" % (safe_str(repr(proxy_spec)), safe_str(repr(str(e)))))
@@ -309,9 +311,9 @@ class Handler(SocketServer.StreamRequestHandler):
log(u"proxy (%s) gets client '%s' (supported transports: %s) (num relays: %s) (remaining regs: %d/%d)" %
(safe_str(repr(proxy_spec)), safe_str(repr(client_reg.addr)), transport_list, num_relays(), num_unhandled_regs(), num_regs()))
print >> self.wfile, fac.render_transaction("OK",
- ("CLIENT", fac.format_addr(client_reg.addr)),
+ ("CLIENT", format_addr(client_reg.addr)),
("CLIENT-TRANSPORT", client_reg.transport.outer),
- ("RELAY", fac.format_addr(relay_reg.addr)),
+ ("RELAY", format_addr(relay_reg.addr)),
("RELAY-TRANSPORT", relay_reg.transport.outer),
("CHECK-BACK-IN", str(check_back_in)))
else:
@@ -356,7 +358,7 @@ class Handler(SocketServer.StreamRequestHandler):
self.send_ok()
return True
- finish = fac.catch_epipe(SocketServer.StreamRequestHandler.finish)
+ finish = proc.catch_epipe(SocketServer.StreamRequestHandler.finish)
class Server(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
allow_reuse_address = True
@@ -422,7 +424,7 @@ def parse_relay_file(servers, fp):
transport_spec, addr_spec = line.strip().split()
except ValueError, e:
raise ValueError("Wrong line format: %s." % repr(line))
- addr = fac.parse_addr_spec(addr_spec, defport=DEFAULT_RELAY_PORT)
+ addr = parse_addr_spec(addr_spec, defport=DEFAULT_RELAY_PORT)
transport = Transport.parse(transport_spec)
if transport.outer not in options.outer_transports:
raise ValueError(u"Unrecognized transport: %s" % transport)
@@ -495,7 +497,7 @@ obfs2|websocket 1.4.6.1:4123\
server = Server(addrinfo[4], Handler)
- log(u"start on %s" % fac.format_addr(addrinfo[4]))
+ log(u"start on %s" % format_addr(addrinfo[4]))
log(u"using IPv4 relays %s" % str(RELAYS[socket.AF_INET]._endpoints))
log(u"using IPv6 relays %s" % str(RELAYS[socket.AF_INET6]._endpoints))
@@ -512,7 +514,7 @@ obfs2|websocket 1.4.6.1:4123\
if options.privdrop_username is not None:
log(u"dropping privileges to those of user %s" % options.privdrop_username)
try:
- fac.drop_privs(options.privdrop_username)
+ proc.drop_privs(options.privdrop_username)
except BaseException, e:
print >> sys.stderr, "Can't drop privileges:", str(e)
sys.exit(1)
diff --git a/facilitator/facilitator-email-poller b/facilitator/facilitator-email-poller
index 2efceec..0eef115 100755
--- a/facilitator/facilitator-email-poller
+++ b/facilitator/facilitator-email-poller
@@ -16,7 +16,9 @@ import sys
import tempfile
import time
-import fac
+from flashproxy import fac
+from flashproxy import proc
+from flashproxy.util import parse_addr_spec
from hashlib import sha1
from M2Crypto import SSL
@@ -209,7 +211,7 @@ Failed to open password file "%s": %s.\
""" % (options.password_filename, str(e))
sys.exit(1)
try:
- if not fac.check_perms(password_file.fileno()):
+ if not proc.check_perms(password_file.fileno()):
print >> sys.stderr, "Refusing to run with group- or world-readable password file. Try"
print >> sys.stderr, "\tchmod 600 %s" % options.password_filename
sys.exit(1)
@@ -221,7 +223,7 @@ try:
if not res:
raise ValueError("could not find email or password on line %s" % (lineno0+1))
(imap_addr_spec, email_addr, email_password) = res.groups()
- imap_addr = fac.parse_addr_spec(
+ imap_addr = parse_addr_spec(
imap_addr_spec or "", DEFAULT_IMAP_HOST, DEFAULT_IMAP_PORT)
break
else:
@@ -255,7 +257,7 @@ if options.daemonize:
if options.privdrop_username is not None:
log(u"dropping privileges to those of user %s" % options.privdrop_username)
try:
- fac.drop_privs(options.privdrop_username)
+ proc.drop_privs(options.privdrop_username)
except BaseException, e:
print >> sys.stderr, "Can't drop privileges:", str(e)
sys.exit(1)
diff --git a/facilitator/facilitator-reg-daemon b/facilitator/facilitator-reg-daemon
index cab7403..f5e592f 100755
--- a/facilitator/facilitator-reg-daemon
+++ b/facilitator/facilitator-reg-daemon
@@ -8,7 +8,9 @@ import sys
import threading
import time
-import fac
+from flashproxy import fac
+from flashproxy import proc
+from flashproxy.util import format_addr
from M2Crypto import RSA
@@ -98,7 +100,7 @@ class Handler(SocketServer.StreamRequestHandler):
raise socket.error("refusing to buffer %d bytes (last read was %d bytes)" % (buflen, len(data)))
return self.buffer
- @fac.catch_epipe
+ @proc.catch_epipe
def handle(self):
try:
b64_ciphertext = self.read_input()
@@ -109,7 +111,7 @@ class Handler(SocketServer.StreamRequestHandler):
ciphertext = b64_ciphertext.decode("base64")
plaintext = rsa.private_decrypt(ciphertext, RSA.pkcs1_oaep_padding)
for client_reg in fac.read_client_registrations(plaintext):
- log(u"registering %s" % safe_str(fac.format_addr(client_reg.addr)))
+ log(u"registering %s" % safe_str(format_addr(client_reg.addr)))
if not fac.put_reg(FACILITATOR_ADDR, client_reg.addr, client_reg.transport):
print >> self.wfile, "FAIL"
break
@@ -120,7 +122,7 @@ class Handler(SocketServer.StreamRequestHandler):
print >> self.wfile, "FAIL"
raise
- finish = fac.catch_epipe(SocketServer.StreamRequestHandler.finish)
+ finish = proc.catch_epipe(SocketServer.StreamRequestHandler.finish)
class Server(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
allow_reuse_address = True
@@ -164,7 +166,7 @@ def main():
print >> sys.stderr, "Failed to open private key file \"%s\": %s." % (options.key_filename, str(e))
sys.exit(1)
try:
- if not fac.check_perms(key_file.fileno()):
+ if not proc.check_perms(key_file.fileno()):
print >> sys.stderr, "Refusing to run with group- or world-readable private key file. Try"
print >> sys.stderr, "\tchmod 600 %s" % options.key_filename
sys.exit(1)
@@ -183,7 +185,7 @@ def main():
server = Server(addrinfo[4], Handler)
- log(u"start on %s" % fac.format_addr(addrinfo[4]))
+ log(u"start on %s" % format_addr(addrinfo[4]))
if options.daemonize:
log(u"daemonizing")
@@ -198,7 +200,7 @@ def main():
if options.privdrop_username is not None:
log(u"dropping privileges to those of user %s" % options.privdrop_username)
try:
- fac.drop_privs(options.privdrop_username)
+ proc.drop_privs(options.privdrop_username)
except BaseException, e:
print >> sys.stderr, "Can't drop privileges:", str(e)
sys.exit(1)
diff --git a/facilitator/facilitator-test.py b/facilitator/facilitator-test.py
index d63e0b5..fd4ac88 100755
--- a/facilitator/facilitator-test.py
+++ b/facilitator/facilitator-test.py
@@ -9,8 +9,9 @@ import sys
import time
import unittest
-import fac
-from fac import Transport, Endpoint
+from flashproxy import fac
+from flashproxy.reg import Transport, Endpoint
+from flashproxy.util import format_addr
# Import the facilitator program as a module.
import imp
@@ -178,19 +179,6 @@ class EndpointsTest(unittest.TestCase):
class FacilitatorTest(unittest.TestCase):
- def test_transport_parse(self):
- self.assertEquals(Transport.parse("a"), Transport("", "a"))
- self.assertEquals(Transport.parse("|a"), Transport("", "a"))
- self.assertEquals(Transport.parse("a|b|c"), Transport("a|b","c"))
- self.assertEquals(Transport.parse(Transport("a|b","c")), Transport("a|b","c"))
- self.assertRaises(ValueError, Transport, "", "")
- self.assertRaises(ValueError, Transport, "a", "")
- self.assertRaises(ValueError, Transport.parse, "")
- self.assertRaises(ValueError, Transport.parse, "|")
- self.assertRaises(ValueError, Transport.parse, "a|")
- self.assertRaises(ValueError, Transport.parse, ["a"])
- self.assertRaises(ValueError, Transport.parse, [Transport("a", "b")])
-
def test_parse_relay_file(self):
fp = StringIO()
fp.write("websocket 0.0.1.0:1\n")
@@ -201,6 +189,7 @@ class FacilitatorTest(unittest.TestCase):
parse_relay_file(servers, fp)
self.assertEquals(servers[af]._endpoints, {('0.0.1.0', 1): Transport('', 'websocket')})
+
class FacilitatorProcTest(unittest.TestCase):
IPV4_CLIENT_ADDR = ("1.1.1.1", 9000)
IPV6_CLIENT_ADDR = ("[11::11]", 9000)
@@ -214,8 +203,8 @@ class FacilitatorProcTest(unittest.TestCase):
def setUp(self):
self.relay_file = tempfile.NamedTemporaryFile()
- self.relay_file.write("%s %s\n" % (RELAY_TP, fac.format_addr(self.IPV4_RELAY_ADDR)))
- self.relay_file.write("%s %s\n" % (RELAY_TP, fac.format_addr(self.IPV6_RELAY_ADDR)))
+ self.relay_file.write("%s %s\n" % (RELAY_TP, format_addr(self.IPV4_RELAY_ADDR)))
+ self.relay_file.write("%s %s\n" % (RELAY_TP, format_addr(self.IPV6_RELAY_ADDR)))
self.relay_file.flush()
self.relay_file.seek(0)
fn = os.path.join(os.path.dirname(__file__), "./facilitator")
@@ -261,7 +250,7 @@ class FacilitatorProcTest(unittest.TestCase):
fac.put_reg(FACILITATOR_ADDR, self.IPV4_CLIENT_ADDR, CLIENT_TP)
fac.put_reg(FACILITATOR_ADDR, self.IPV6_CLIENT_ADDR, CLIENT_TP)
reg = fac.get_reg(FACILITATOR_ADDR, self.IPV4_PROXY_ADDR, PROXY_TPS)
- self.assertEqual(reg["client"], fac.format_addr(self.IPV4_CLIENT_ADDR))
+ self.assertEqual(reg["client"], format_addr(self.IPV4_CLIENT_ADDR))
def test_af_v4_v6(self):
"""Test that IPv4 proxies do not get IPv6 clients."""
@@ -280,15 +269,15 @@ class FacilitatorProcTest(unittest.TestCase):
fac.put_reg(FACILITATOR_ADDR, self.IPV4_CLIENT_ADDR, CLIENT_TP)
fac.put_reg(FACILITATOR_ADDR, self.IPV6_CLIENT_ADDR, CLIENT_TP)
reg = fac.get_reg(FACILITATOR_ADDR, self.IPV6_PROXY_ADDR, PROXY_TPS)
- self.assertEqual(reg["client"], fac.format_addr(self.IPV6_CLIENT_ADDR))
+ self.assertEqual(reg["client"], format_addr(self.IPV6_CLIENT_ADDR))
def test_fields(self):
"""Test that facilitator responses contain all the required fields."""
fac.put_reg(FACILITATOR_ADDR, self.IPV4_CLIENT_ADDR, CLIENT_TP)
reg = fac.get_reg(FACILITATOR_ADDR, self.IPV4_PROXY_ADDR, PROXY_TPS)
- self.assertEqual(reg["client"], fac.format_addr(self.IPV4_CLIENT_ADDR))
+ self.assertEqual(reg["client"], format_addr(self.IPV4_CLIENT_ADDR))
self.assertEqual(reg["client-transport"], CLIENT_TP)
- self.assertEqual(reg["relay"], fac.format_addr(self.IPV4_RELAY_ADDR))
+ self.assertEqual(reg["relay"], format_addr(self.IPV4_RELAY_ADDR))
self.assertEqual(reg["relay-transport"], RELAY_TP)
self.assertGreater(int(reg["check-back-in"]), 0)
@@ -323,117 +312,5 @@ class FacilitatorProcTest(unittest.TestCase):
# """Test that the facilitator rejects hostnames."""
# self.fail()
-class ParseAddrSpecTest(unittest.TestCase):
- def test_ipv4(self):
- self.assertEqual(fac.parse_addr_spec("192.168.0.1:9999"), ("192.168.0.1", 9999))
-
- def test_ipv6(self):
- self.assertEqual(fac.parse_addr_spec("[12::34]:9999"), ("12::34", 9999))
-
- def test_defhost_defport_ipv4(self):
- self.assertEqual(fac.parse_addr_spec("192.168.0.2:8888", defhost="192.168.0.1", defport=9999), ("192.168.0.2", 8888))
- self.assertEqual(fac.parse_addr_spec("192.168.0.2:", defhost="192.168.0.1", defport=9999), ("192.168.0.2", 9999))
- self.assertEqual(fac.parse_addr_spec("192.168.0.2", defhost="192.168.0.1", defport=9999), ("192.168.0.2", 9999))
- self.assertEqual(fac.parse_addr_spec(":8888", defhost="192.168.0.1", defport=9999), ("192.168.0.1", 8888))
- self.assertEqual(fac.parse_addr_spec(":", defhost="192.168.0.1", defport=9999), ("192.168.0.1", 9999))
- self.assertEqual(fac.parse_addr_spec("", defhost="192.168.0.1", defport=9999), ("192.168.0.1", 9999))
-
- def test_defhost_defport_ipv6(self):
- self.assertEqual(fac.parse_addr_spec("[1234::2]:8888", defhost="1234::1", defport=9999), ("1234::2", 8888))
- self.assertEqual(fac.parse_addr_spec("[1234::2]:", defhost="1234::1", defport=9999), ("1234::2", 9999))
- self.assertEqual(fac.parse_addr_spec("[1234::2]", defhost="1234::1", defport=9999), ("1234::2", 9999))
- self.assertEqual(fac.parse_addr_spec(":8888", defhost="1234::1", defport=9999), ("1234::1", 8888))
- self.assertEqual(fac.parse_addr_spec(":", defhost="1234::1", defport=9999), ("1234::1", 9999))
- self.assertEqual(fac.parse_addr_spec("", defhost="1234::1", defport=9999), ("1234::1", 9999))
-
- def test_canonical_ip_noresolve(self):
- """Test that canonical_ip does not do DNS resolution by default."""
- self.assertRaises(ValueError, fac.canonical_ip, *fac.parse_addr_spec("example.com:80"))
-
-class ParseTransactionTest(unittest.TestCase):
- def test_empty_string(self):
- self.assertRaises(ValueError, fac.parse_transaction, "")
-
- def test_correct(self):
- self.assertEqual(fac.parse_transaction("COMMAND"), ("COMMAND", ()))
- self.assertEqual(fac.parse_transaction("COMMAND X=\"\""), ("COMMAND", (("X", ""),)))
- self.assertEqual(fac.parse_transaction("COMMAND X=\"ABC\""), ("COMMAND", (("X", "ABC"),)))
- self.assertEqual(fac.parse_transaction("COMMAND X=\"\\A\\B\\C\""), ("COMMAND", (("X", "ABC"),)))
- self.assertEqual(fac.parse_transaction("COMMAND X=\"\\\\\\\"\""), ("COMMAND", (("X", "\\\""),)))
- self.assertEqual(fac.parse_transaction("COMMAND X=\"ABC\" Y=\"DEF\""), ("COMMAND", (("X", "ABC"), ("Y", "DEF"))))
- self.assertEqual(fac.parse_transaction("COMMAND KEY-NAME=\"ABC\""), ("COMMAND", (("KEY-NAME", "ABC"),)))
- self.assertEqual(fac.parse_transaction("COMMAND KEY_NAME=\"ABC\""), ("COMMAND", (("KEY_NAME", "ABC"),)))
-
- def test_missing_command(self):
- self.assertRaises(ValueError, fac.parse_transaction, "X=\"ABC\"")
- self.assertRaises(ValueError, fac.parse_transaction, " X=\"ABC\"")
-
- def test_missing_space(self):
- self.assertRaises(ValueError, fac.parse_transaction, "COMMAND/X=\"ABC\"")
- self.assertRaises(ValueError, fac.parse_transaction, "COMMAND X=\"ABC\"Y=\"DEF\"")
-
- def test_bad_quotes(self):
- self.assertRaises(ValueError, fac.parse_transaction, "COMMAND X=\"")
- self.assertRaises(ValueError, fac.parse_transaction, "COMMAND X=\"ABC")
- self.assertRaises(ValueError, fac.parse_transaction, "COMMAND X=\"ABC\" Y=\"ABC")
- self.assertRaises(ValueError, fac.parse_transaction, "COMMAND X=\"ABC\\")
-
- def test_truncated(self):
- self.assertRaises(ValueError, fac.parse_transaction, "COMMAND X=")
-
- def test_newline(self):
- self.assertRaises(ValueError, fac.parse_transaction, "COMMAND X=\"ABC\" \nY=\"DEF\"")
-
-class ReadClientRegistrationsTest(unittest.TestCase):
- def testSingle(self):
- l = list(fac.read_client_registrations(""))
- self.assertEqual(len(l), 0)
- l = list(fac.read_client_registrations("client=1.2.3.4:1111"))
- self.assertEqual(len(l), 1)
- self.assertEqual(l[0].addr, ("1.2.3.4", 1111))
- l = list(fac.read_client_registrations("client=1.2.3.4:1111\n"))
- self.assertEqual(len(l), 1)
- self.assertEqual(l[0].addr, ("1.2.3.4", 1111))
- l = list(fac.read_client_registrations("foo=bar&client=1.2.3.4:1111&baz=quux"))
- self.assertEqual(len(l), 1)
- self.assertEqual(l[0].addr, ("1.2.3.4", 1111))
- l = list(fac.read_client_registrations("foo=b%3dar&client=1.2.3.4%3a1111"))
- self.assertEqual(len(l), 1)
- self.assertEqual(l[0].addr, ("1.2.3.4", 1111))
- l = list(fac.read_client_registrations("client=%5b1::2%5d:3333"))
- self.assertEqual(len(l), 1)
- self.assertEqual(l[0].addr, ("1::2", 3333))
-
- def testDefaultAddress(self):
- l = list(fac.read_client_registrations("client=:1111&transport=websocket", defhost="1.2.3.4"))
- self.assertEqual(l[0].addr, ("1.2.3.4", 1111))
- l = list(fac.read_client_registrations("client=1.2.3.4:&transport=websocket", defport=1111))
- self.assertEqual(l[0].addr, ("1.2.3.4", 1111))
-
- def testDefaultTransport(self):
- l = list(fac.read_client_registrations("client=1.2.3.4:1111"))
- self.assertEqual(l[0].transport, "websocket")
-
- def testMultiple(self):
- l = list(fac.read_client_registrations("client=1.2.3.4:1111&foo=bar\nfoo=bar&client=5.6.7.8:2222"))
- self.assertEqual(len(l), 2)
- self.assertEqual(l[0].addr, ("1.2.3.4", 1111))
- self.assertEqual(l[1].addr, ("5.6.7.8", 2222))
- l = list(fac.read_client_registrations("client=1.2.3.4:1111&foo=bar\nfoo=bar&client=%5b1::2%5d:3333"))
- self.assertEqual(len(l), 2)
- self.assertEqual(l[0].addr, ("1.2.3.4", 1111))
- self.assertEqual(l[1].addr, ("1::2", 3333))
-
- def testInvalid(self):
- # Missing "client".
- with self.assertRaises(ValueError):
- list(fac.read_client_registrations("foo=bar"))
- # More than one "client".
- with self.assertRaises(ValueError):
- list(fac.read_client_registrations("client=1.2.3.4:1111&foo=bar&client=5.6.7.8:2222"))
- # Single client with bad syntax.
- with self.assertRaises(ValueError):
- list(fac.read_client_registrations("client=1.2.3.4,1111"))
-
if __name__ == "__main__":
unittest.main()
diff --git a/facilitator/facilitator.cgi b/facilitator/facilitator.cgi
index 42a888e..addcaf4 100755
--- a/facilitator/facilitator.cgi
+++ b/facilitator/facilitator.cgi
@@ -6,7 +6,7 @@ import socket
import sys
import urllib
-import fac
+from flashproxy import fac
FACILITATOR_ADDR = ("127.0.0.1", 9002)
diff --git a/flashproxy-client b/flashproxy-client
index e54e554..f2cf04b 100755
--- a/flashproxy-client
+++ b/flashproxy-client
@@ -1129,14 +1129,14 @@ def main():
default_remote_port = DEFAULT_REMOTE_PORT
if len(args) == 0:
- local_addr = (None, default_local_port)
- remote_addr = (None, default_remote_port)
+ local_addr = ("", default_local_port)
+ remote_addr = ("", default_remote_port)
elif len(args) == 1:
- local_addr = parse_addr_spec(args[0], defport=default_local_port)
- remote_addr = (None, default_remote_port)
+ local_addr = parse_addr_spec(args[0], defhost="", defport=default_local_port)
+ remote_addr = ("", default_remote_port)
elif len(args) == 2:
- local_addr = parse_addr_spec(args[0], defport=default_local_port)
- remote_addr = parse_addr_spec(args[1], defport=default_remote_port)
+ local_addr = parse_addr_spec(args[0], defhost="", defport=default_local_port)
+ remote_addr = parse_addr_spec(args[1], defhost="", defport=default_remote_port)
else:
usage(sys.stderr)
sys.exit(1)
diff --git a/flashproxy-reg-appspot b/flashproxy-reg-appspot
index 516fcc9..7738e43 100755
--- a/flashproxy-reg-appspot
+++ b/flashproxy-reg-appspot
@@ -19,7 +19,7 @@ except ImportError:
# Defer the error reporting so that --help works even without M2Crypto.
SSL = None
-DEFAULT_REMOTE_ADDRESS = None
+DEFAULT_REMOTE_ADDRESS = ""
DEFAULT_REMOTE_PORT = 9000
DEFAULT_TRANSPORT = "websocket"
diff --git a/flashproxy-reg-email b/flashproxy-reg-email
index ab8d4cb..d286af8 100755
--- a/flashproxy-reg-email
+++ b/flashproxy-reg-email
@@ -20,7 +20,7 @@ except ImportError:
RSA = None
SSL = None
-DEFAULT_REMOTE_ADDRESS = None
+DEFAULT_REMOTE_ADDRESS = ""
DEFAULT_REMOTE_PORT = 9000
DEFAULT_EMAIL_ADDRESS = "flashproxyreg.a@xxxxxxxxx"
# dig MX gmail.com
diff --git a/flashproxy-reg-url b/flashproxy-reg-url
index c45bc26..0f92d44 100755
--- a/flashproxy-reg-url
+++ b/flashproxy-reg-url
@@ -15,7 +15,7 @@ except ImportError:
# Defer the error reporting so that --help works even without M2Crypto.
RSA = None
-DEFAULT_REMOTE_ADDRESS = None
+DEFAULT_REMOTE_ADDRESS = ""
DEFAULT_REMOTE_PORT = 9000
DEFAULT_FACILITATOR_URL = "https://fp-facilitator.org/";
DEFAULT_TRANSPORT = "websocket"
diff --git a/flashproxy/fac.py b/flashproxy/fac.py
new file mode 100644
index 0000000..0686f54
--- /dev/null
+++ b/flashproxy/fac.py
@@ -0,0 +1,224 @@
+import socket
+import subprocess
+import urlparse
+
+from flashproxy import reg
+from flashproxy.util import parse_addr_spec, format_addr
+
+DEFAULT_CLIENT_TRANSPORT = "websocket"
+
+def read_client_registrations(body, defhost=None, defport=None):
+ """Yield client registrations (as Endpoints) from an encoded registration
+ message body. The message format is one registration per line, with each
+ line being encoded as application/x-www-form-urlencoded. The key "client" is
+ required and contains the client address and port (perhaps filled in by
+ defhost and defport). The key "client-transport" is optional and defaults to
+ "websocket".
+ Example:
+ client=1.2.3.4:9000&client-transport=websocket
+ client=1.2.3.4:9090&client-transport=obfs3|websocket
+ """
+ for line in body.splitlines():
+ qs = urlparse.parse_qs(line, keep_blank_values=True, strict_parsing=True)
+ # Get the unique value associated with the given key in qs. If the key
+ # is absent or appears more than once, raise ValueError.
+ def get_unique(key, default=None):
+ try:
+ vals = qs[key]
+ except KeyError:
+ if default is None:
+ raise ValueError("missing %r key" % key)
+ vals = (default,)
+ if len(vals) != 1:
+ raise ValueError("more than one %r key" % key)
+ return vals[0]
+ addr = parse_addr_spec(get_unique("client"), defhost, defport)
+ transport = get_unique("client-transport", DEFAULT_CLIENT_TRANSPORT)
+ yield reg.Endpoint(addr, transport)
+
+def skip_space(pos, line):
+ """Skip a (possibly empty) sequence of space characters (the ASCII character
+ '\x20' exactly). Returns a pair (pos, num_skipped)."""
+ begin = pos
+ while pos < len(line) and line[pos] == "\x20":
+ pos += 1
+ return pos, pos - begin
+
+TOKEN_CHARS = set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-")
+def get_token(pos, line):
+ begin = pos
+ while pos < len(line) and line[pos] in TOKEN_CHARS:
+ pos += 1
+ if begin == pos:
+ raise ValueError("No token found at position %d" % pos)
+ return pos, line[begin:pos]
+
+def get_quoted_string(pos, line):
+ chars = []
+ if not (pos < len(line) and line[pos] == '"'):
+ raise ValueError("Expected '\"' at beginning of quoted string.")
+ pos += 1
+ while pos < len(line) and line[pos] != '"':
+ if line[pos] == '\\':
+ pos += 1
+ if not (pos < len(line)):
+ raise ValueError("End of line after backslash in quoted string")
+ chars.append(line[pos])
+ pos += 1
+ if not (pos < len(line) and line[pos] == '"'):
+ raise ValueError("Expected '\"' at end of quoted string.")
+ pos += 1
+ return pos, "".join(chars)
+
+def parse_transaction(line):
+ """A transaction is a command followed by zero or more key-value pairs. Like so:
+ COMMAND KEY="VALUE" KEY="\"ESCAPED\" VALUE"
+ Values must be quoted. Any byte value may be escaped with a backslash.
+ Returns a pair: (COMMAND, ((KEY1, VALUE1), (KEY2, VALUE2), ...)).
+ """
+ pos = 0
+ pos, skipped = skip_space(pos, line)
+ pos, command = get_token(pos, line)
+
+ pairs = []
+ while True:
+ pos, skipped = skip_space(pos, line)
+ if not (pos < len(line)):
+ break
+ if skipped == 0:
+ raise ValueError("Expected space before key-value pair")
+ pos, key = get_token(pos, line)
+ if not (pos < len(line) and line[pos] == '='):
+ raise ValueError("No '=' found after key")
+ pos += 1
+ pos, value = get_quoted_string(pos, line)
+ pairs.append((key, value))
+ return command, tuple(pairs)
+
+def param_first(key, params):
+ """Search 'params' for 'key' and return the first value that
+ occurs. If 'key' was not found, return None."""
+ for k, v in params:
+ if key == k:
+ return v
+ return None
+
+def param_getlist(key, params):
+ """Search 'params' for 'key' and return a list with its values. If
+ 'key' did not appear in 'params', return the empty list."""
+ result = []
+ for k, v in params:
+ if key == k:
+ result.append(v)
+ return result
+
+def quote_string(s):
+ chars = []
+ for c in s:
+ if c == "\\":
+ c = "\\\\"
+ elif c == "\"":
+ c = "\\\""
+ chars.append(c)
+ return "\"" + "".join(chars) + "\""
+
+def render_transaction(command, *params):
+ parts = [command]
+ for key, value in params:
+ parts.append("%s=%s" % (key, quote_string(value)))
+ return " ".join(parts)
+
+def fac_socket(facilitator_addr):
+ return socket.create_connection(facilitator_addr, 1.0).makefile()
+
+def transact(f, command, *params):
+ transaction = render_transaction(command, *params)
+ print >> f, transaction
+ f.flush()
+ line = f.readline()
+ if not (len(line) > 0 and line[-1] == '\n'):
+ raise ValueError("No newline at end of string returned by facilitator")
+ return parse_transaction(line[:-1])
+
+def put_reg(facilitator_addr, client_addr, transport):
+ """Send a registration to the facilitator using a one-time socket. Returns
+ true iff the command was successful. transport is a transport string such as
+ "websocket" or "obfs3|websocket"."""
+ f = fac_socket(facilitator_addr)
+ params = [("CLIENT", format_addr(client_addr))]
+ params.append(("TRANSPORT", transport))
+ try:
+ command, params = transact(f, "PUT", *params)
+ finally:
+ f.close()
+ return command == "OK"
+
+def get_reg(facilitator_addr, proxy_addr, proxy_transport_list):
+ """
+ Get a client registration for proxy proxy_addr from the
+ facilitator at facilitator_addr using a one-time
+ socket. proxy_transport_list is a list containing the transport names that
+ the flashproxy supports.
+
+ Returns a dict with keys "client", "client-transport", "relay",
+ and "relay-transport" if successful, or a dict with the key "client"
+ mapped to the value "" if there are no registrations available for
+ proxy_addr. Raises an exception otherwise."""
+ f = fac_socket(facilitator_addr)
+
+ # Form a list (in transact() format) with the transports that we
+ # should send to the facilitator. Then pass that list to the
+ # transact() function.
+ # For example, PROXY-TRANSPORT=obfs2 PROXY-TRANSPORT=obfs3.
+ transports = [("PROXY-TRANSPORT", tp) for tp in proxy_transport_list]
+
+ try:
+ command, params = transact(f, "GET", ("FROM", format_addr(proxy_addr)), *transports)
+ finally:
+ f.close()
+ response = {}
+ check_back_in = param_first("CHECK-BACK-IN", params)
+ if check_back_in is not None:
+ try:
+ float(check_back_in)
+ except ValueError:
+ raise ValueError("Facilitator returned non-numeric polling interval.")
+ response["check-back-in"] = check_back_in
+ if command == "NONE":
+ response["client"] = ""
+ return response
+ elif command == "OK":
+ client_spec = param_first("CLIENT", params)
+ client_transport = param_first("CLIENT-TRANSPORT", params)
+ relay_spec = param_first("RELAY", params)
+ relay_transport = param_first("RELAY-TRANSPORT", params)
+ if not client_spec:
+ raise ValueError("Facilitator did not return CLIENT")
+ if not client_transport:
+ raise ValueError("Facilitator did not return CLIENT-TRANSPORT")
+ if not relay_spec:
+ raise ValueError("Facilitator did not return RELAY")
+ if not relay_transport:
+ raise ValueError("Facilitator did not return RELAY-TRANSPORT")
+ # Check the syntax returned by the facilitator.
+ client = parse_addr_spec(client_spec)
+ relay = parse_addr_spec(relay_spec)
+ response["client"] = format_addr(client)
+ response["client-transport"] = client_transport
+ response["relay"] = format_addr(relay)
+ response["relay-transport"] = relay_transport
+ return response
+ else:
+ raise ValueError("Facilitator response was not \"OK\"")
+
+def put_reg_base64(b64):
+ """Attempt to add a registration by running a facilitator-reg program
+ locally."""
+ # Padding is optional, but the python base64 functions can't
+ # handle lack of padding. Add it here. Assumes correct encoding.
+ mod = len(b64) % 4
+ if mod != 0:
+ b64 += (4 - mod) * "="
+ p = subprocess.Popen(["facilitator-reg"], stdin=subprocess.PIPE)
+ stdout, stderr = p.communicate(b64)
+ return p.returncode == 0
diff --git a/flashproxy/proc.py b/flashproxy/proc.py
new file mode 100644
index 0000000..4a008b2
--- /dev/null
+++ b/flashproxy/proc.py
@@ -0,0 +1,47 @@
+import errno
+import os
+import socket
+import stat
+import pwd
+
+DEFAULT_CLIENT_TRANSPORT = "websocket"
+
+# Return true iff the given fd is readable, writable, and executable only by its
+# owner.
+def check_perms(fd):
+ mode = os.fstat(fd)[0]
+ return (mode & (stat.S_IRWXG | stat.S_IRWXO)) == 0
+
+# Drop privileges by switching ID to that of the given user.
+# http://stackoverflow.com/questions/2699907/dropping-root-permissions-in-python/2699996#2699996
+# https://www.securecoding.cert.org/confluence/display/seccode/POS36-C.+Observe+correct+revocation+order+while+relinquishing+privileges
+# https://www.securecoding.cert.org/confluence/display/seccode/POS37-C.+Ensure+that+privilege+relinquishment+is+successful
+def drop_privs(username):
+ uid = pwd.getpwnam(username).pw_uid
+ gid = pwd.getpwnam(username).pw_gid
+ os.setgroups([])
+ os.setgid(gid)
+ os.setuid(uid)
+ try:
+ os.setuid(0)
+ except OSError:
+ pass
+ else:
+ raise AssertionError("setuid(0) succeeded after attempting to drop privileges")
+
+# A decorator to ignore "broken pipe" errors.
+def catch_epipe(fn):
+ def ret(self, *args):
+ try:
+ return fn(self, *args)
+ except socket.error, e:
+ try:
+ err_num = e.errno
+ except AttributeError:
+ # Before Python 2.6, exception can be a pair.
+ err_num, errstr = e
+ except:
+ raise
+ if err_num != errno.EPIPE:
+ raise
+ return ret
diff --git a/flashproxy/reg.py b/flashproxy/reg.py
new file mode 100644
index 0000000..0551f06
--- /dev/null
+++ b/flashproxy/reg.py
@@ -0,0 +1,31 @@
+from collections import namedtuple
+
+from flashproxy.util import parse_addr_spec
+
+class Transport(namedtuple("Transport", "inner outer")):
+ @classmethod
+ def parse(cls, transport):
+ if isinstance(transport, cls):
+ return transport
+ elif type(transport) == str:
+ if "|" in transport:
+ inner, outer = transport.rsplit("|", 1)
+ else:
+ inner, outer = "", transport
+ return cls(inner, outer)
+ else:
+ raise ValueError("could not parse transport: %s" % transport)
+
+ def __init__(self, inner, outer):
+ if not outer:
+ raise ValueError("outer (proxy) part of transport must be non-empty: %s" % str(self))
+
+ def __str__(self):
+ return "%s|%s" % (self.inner, self.outer) if self.inner else self.outer
+
+
+class Endpoint(namedtuple("Endpoint", "addr transport")):
+ @classmethod
+ def parse(cls, spec, transport, defhost = None, defport = None):
+ host, port = parse_addr_spec(spec, defhost, defport)
+ return cls((host, port), Transport.parse(transport))
diff --git a/flashproxy/test/test_fac.py b/flashproxy/test/test_fac.py
new file mode 100644
index 0000000..e7dfa00
--- /dev/null
+++ b/flashproxy/test/test_fac.py
@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+
+import unittest
+
+from flashproxy.fac import parse_transaction, read_client_registrations
+
+class ParseTransactionTest(unittest.TestCase):
+ def test_empty_string(self):
+ self.assertRaises(ValueError, parse_transaction, "")
+
+ def test_correct(self):
+ self.assertEqual(parse_transaction("COMMAND"), ("COMMAND", ()))
+ self.assertEqual(parse_transaction("COMMAND X=\"\""), ("COMMAND", (("X", ""),)))
+ self.assertEqual(parse_transaction("COMMAND X=\"ABC\""), ("COMMAND", (("X", "ABC"),)))
+ self.assertEqual(parse_transaction("COMMAND X=\"\\A\\B\\C\""), ("COMMAND", (("X", "ABC"),)))
+ self.assertEqual(parse_transaction("COMMAND X=\"\\\\\\\"\""), ("COMMAND", (("X", "\\\""),)))
+ self.assertEqual(parse_transaction("COMMAND X=\"ABC\" Y=\"DEF\""), ("COMMAND", (("X", "ABC"), ("Y", "DEF"))))
+ self.assertEqual(parse_transaction("COMMAND KEY-NAME=\"ABC\""), ("COMMAND", (("KEY-NAME", "ABC"),)))
+ self.assertEqual(parse_transaction("COMMAND KEY_NAME=\"ABC\""), ("COMMAND", (("KEY_NAME", "ABC"),)))
+
+ def test_missing_command(self):
+ self.assertRaises(ValueError, parse_transaction, "X=\"ABC\"")
+ self.assertRaises(ValueError, parse_transaction, " X=\"ABC\"")
+
+ def test_missing_space(self):
+ self.assertRaises(ValueError, parse_transaction, "COMMAND/X=\"ABC\"")
+ self.assertRaises(ValueError, parse_transaction, "COMMAND X=\"ABC\"Y=\"DEF\"")
+
+ def test_bad_quotes(self):
+ self.assertRaises(ValueError, parse_transaction, "COMMAND X=\"")
+ self.assertRaises(ValueError, parse_transaction, "COMMAND X=\"ABC")
+ self.assertRaises(ValueError, parse_transaction, "COMMAND X=\"ABC\" Y=\"ABC")
+ self.assertRaises(ValueError, parse_transaction, "COMMAND X=\"ABC\\")
+
+ def test_truncated(self):
+ self.assertRaises(ValueError, parse_transaction, "COMMAND X=")
+
+ def test_newline(self):
+ self.assertRaises(ValueError, parse_transaction, "COMMAND X=\"ABC\" \nY=\"DEF\"")
+
+class ReadClientRegistrationsTest(unittest.TestCase):
+ def testSingle(self):
+ l = list(read_client_registrations(""))
+ self.assertEqual(len(l), 0)
+ l = list(read_client_registrations("client=1.2.3.4:1111"))
+ self.assertEqual(len(l), 1)
+ self.assertEqual(l[0].addr, ("1.2.3.4", 1111))
+ l = list(read_client_registrations("client=1.2.3.4:1111\n"))
+ self.assertEqual(len(l), 1)
+ self.assertEqual(l[0].addr, ("1.2.3.4", 1111))
+ l = list(read_client_registrations("foo=bar&client=1.2.3.4:1111&baz=quux"))
+ self.assertEqual(len(l), 1)
+ self.assertEqual(l[0].addr, ("1.2.3.4", 1111))
+ l = list(read_client_registrations("foo=b%3dar&client=1.2.3.4%3a1111"))
+ self.assertEqual(len(l), 1)
+ self.assertEqual(l[0].addr, ("1.2.3.4", 1111))
+ l = list(read_client_registrations("client=%5b1::2%5d:3333"))
+ self.assertEqual(len(l), 1)
+ self.assertEqual(l[0].addr, ("1::2", 3333))
+
+ def testDefaultAddress(self):
+ l = list(read_client_registrations("client=:1111&transport=websocket", defhost="1.2.3.4"))
+ self.assertEqual(l[0].addr, ("1.2.3.4", 1111))
+ l = list(read_client_registrations("client=1.2.3.4:&transport=websocket", defport=1111))
+ self.assertEqual(l[0].addr, ("1.2.3.4", 1111))
+
+ def testDefaultTransport(self):
+ l = list(read_client_registrations("client=1.2.3.4:1111"))
+ self.assertEqual(l[0].transport, "websocket")
+
+ def testMultiple(self):
+ l = list(read_client_registrations("client=1.2.3.4:1111&foo=bar\nfoo=bar&client=5.6.7.8:2222"))
+ self.assertEqual(len(l), 2)
+ self.assertEqual(l[0].addr, ("1.2.3.4", 1111))
+ self.assertEqual(l[1].addr, ("5.6.7.8", 2222))
+ l = list(read_client_registrations("client=1.2.3.4:1111&foo=bar\nfoo=bar&client=%5b1::2%5d:3333"))
+ self.assertEqual(len(l), 2)
+ self.assertEqual(l[0].addr, ("1.2.3.4", 1111))
+ self.assertEqual(l[1].addr, ("1::2", 3333))
+
+ def testInvalid(self):
+ # Missing "client".
+ with self.assertRaises(ValueError):
+ list(read_client_registrations("foo=bar"))
+ # More than one "client".
+ with self.assertRaises(ValueError):
+ list(read_client_registrations("client=1.2.3.4:1111&foo=bar&client=5.6.7.8:2222"))
+ # Single client with bad syntax.
+ with self.assertRaises(ValueError):
+ list(read_client_registrations("client=1.2.3.4,1111"))
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/flashproxy/test/test_reg.py b/flashproxy/test/test_reg.py
new file mode 100644
index 0000000..6b0e196
--- /dev/null
+++ b/flashproxy/test/test_reg.py
@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+
+import unittest
+
+from flashproxy.reg import Transport
+
+class TransportTest(unittest.TestCase):
+
+ def test_transport_parse(self):
+ self.assertEquals(Transport.parse("a"), Transport("", "a"))
+ self.assertEquals(Transport.parse("|a"), Transport("", "a"))
+ self.assertEquals(Transport.parse("a|b|c"), Transport("a|b","c"))
+ self.assertEquals(Transport.parse(Transport("a|b","c")), Transport("a|b","c"))
+ self.assertRaises(ValueError, Transport, "", "")
+ self.assertRaises(ValueError, Transport, "a", "")
+ self.assertRaises(ValueError, Transport.parse, "")
+ self.assertRaises(ValueError, Transport.parse, "|")
+ self.assertRaises(ValueError, Transport.parse, "a|")
+ self.assertRaises(ValueError, Transport.parse, ["a"])
+ self.assertRaises(ValueError, Transport.parse, [Transport("a", "b")])
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/flashproxy/test/test_util.py b/flashproxy/test/test_util.py
new file mode 100644
index 0000000..af4c2e6
--- /dev/null
+++ b/flashproxy/test/test_util.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+
+import unittest
+
+from flashproxy.util import parse_addr_spec, canonical_ip
+
+class ParseAddrSpecTest(unittest.TestCase):
+ def test_ipv4(self):
+ self.assertEqual(parse_addr_spec("192.168.0.1:9999"), ("192.168.0.1", 9999))
+
+ def test_ipv6(self):
+ self.assertEqual(parse_addr_spec("[12::34]:9999"), ("12::34", 9999))
+
+ def test_defhost_defport_ipv4(self):
+ self.assertEqual(parse_addr_spec("192.168.0.2:8888", defhost="192.168.0.1", defport=9999), ("192.168.0.2", 8888))
+ self.assertEqual(parse_addr_spec("192.168.0.2:", defhost="192.168.0.1", defport=9999), ("192.168.0.2", 9999))
+ self.assertEqual(parse_addr_spec("192.168.0.2", defhost="192.168.0.1", defport=9999), ("192.168.0.2", 9999))
+ self.assertEqual(parse_addr_spec(":8888", defhost="192.168.0.1", defport=9999), ("192.168.0.1", 8888))
+ self.assertEqual(parse_addr_spec(":", defhost="192.168.0.1", defport=9999), ("192.168.0.1", 9999))
+ self.assertEqual(parse_addr_spec("", defhost="192.168.0.1", defport=9999), ("192.168.0.1", 9999))
+
+ def test_defhost_defport_ipv6(self):
+ self.assertEqual(parse_addr_spec("[1234::2]:8888", defhost="1234::1", defport=9999), ("1234::2", 8888))
+ self.assertEqual(parse_addr_spec("[1234::2]:", defhost="1234::1", defport=9999), ("1234::2", 9999))
+ self.assertEqual(parse_addr_spec("[1234::2]", defhost="1234::1", defport=9999), ("1234::2", 9999))
+ self.assertEqual(parse_addr_spec(":8888", defhost="1234::1", defport=9999), ("1234::1", 8888))
+ self.assertEqual(parse_addr_spec(":", defhost="1234::1", defport=9999), ("1234::1", 9999))
+ self.assertEqual(parse_addr_spec("", defhost="1234::1", defport=9999), ("1234::1", 9999))
+
+ def test_canonical_ip_noresolve(self):
+ """Test that canonical_ip does not do DNS resolution by default."""
+ self.assertRaises(ValueError, canonical_ip, *parse_addr_spec("example.com:80"))
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/flashproxy/util.py b/flashproxy/util.py
index 47bd87a..b069bf7 100644
--- a/flashproxy/util.py
+++ b/flashproxy/util.py
@@ -2,6 +2,27 @@ import re
import socket
def parse_addr_spec(spec, defhost = None, defport = None):
+ """Parse a host:port specification and return a 2-tuple ("host", port) as
+ understood by the Python socket functions.
+ >>> parse_addr_spec("192.168.0.1:9999")
+ ('192.168.0.1', 9999)
+
+ If defhost or defport are given, those parts of the specification may be
+ omitted; if so, they will be filled in with defaults.
+ >>> parse_addr_spec("192.168.0.2:8888", defhost="192.168.0.1", defport=9999)
+ ('192.168.0.2', 8888)
+ >>> parse_addr_spec(":8888", defhost="192.168.0.1", defport=9999)
+ ('192.168.0.1', 8888)
+ >>> parse_addr_spec("192.168.0.2", defhost="192.168.0.1", defport=9999)
+ ('192.168.0.2', 9999)
+ >>> parse_addr_spec("192.168.0.2:", defhost="192.168.0.1", defport=9999)
+ ('192.168.0.2', 9999)
+ >>> parse_addr_spec(":", defhost="192.168.0.1", defport=9999)
+ ('192.168.0.1', 9999)
+ >>> parse_addr_spec("", defhost="192.168.0.1", defport=9999)
+ ('192.168.0.1', 9999)
+
+ IPv6 addresses must be enclosed in square brackets."""
host = None
port = None
af = 0
@@ -29,24 +50,61 @@ def parse_addr_spec(spec, defhost = None, defport = None):
af = 0
host = host or defhost
port = port or defport
- if port is not None:
- port = int(port)
- return host, port
+ if host is None or port is None:
+ raise ValueError("Bad address specification \"%s\"" % spec)
+ return host, int(port)
-def format_addr(addr):
- host, port = addr
- if not host:
- return u":%d" % port
- # Numeric IPv6 address?
+def resolve_to_ip(host, port, af=0, gai_flags=0):
+ """Resolves a host string to an IP address in canonical format.
+
+ Note: in many cases this is not necessary since the consumer of the address
+ can probably accept host names directly.
+
+ :param: host string to resolve; may be a DNS name or an IP address.
+ :param: port of the host
+ :param: af address family, default unspecified. set to socket.AF_INET or
+ socket.AF_INET6 to force IPv4 or IPv6 name resolution.
+ :returns: (IP address in canonical format, port)
+ """
+ # Forward-resolve the name into an addrinfo struct. Real DNS resolution is
+ # done only if resolve is true; otherwise the address must be numeric.
try:
- addrs = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM, socket.IPPROTO_TCP, socket.AI_NUMERICHOST)
- af = addrs[0][0]
+ addrs = socket.getaddrinfo(host, port, af, 0, 0, gai_flags)
except socket.gaierror, e:
- af = 0
- if af == socket.AF_INET6:
- result = u"[%s]" % host
- else:
- result = "%s" % host
+ raise ValueError("Bad host or port: \"%s\" \"%s\": %s" % (host, port, str(e)))
+ if not addrs:
+ raise ValueError("Bad host or port: \"%s\" \"%s\"" % (host, port))
+
+ # Convert the result of socket.getaddrinfo (which is a 2-tuple for IPv4 and
+ # a 4-tuple for IPv6) into a (host, port) 2-tuple.
+ host, port = socket.getnameinfo(addrs[0][4], socket.NI_NUMERICHOST | socket.NI_NUMERICSERV)
+ return host, int(port)
+
+def canonical_ip(host, port, af=0):
+ """Convert an IP address to a canonical format. Identical to resolve_to_ip,
+ except that the host param must already be an IP address."""
+ return resolve_to_ip(host, port, af, gai_flags=socket.AI_NUMERICHOST)
+
+def format_addr(addr):
+ host, port = addr
+ host_str = u""
+ port_str = u""
+ if host is not None:
+ # Numeric IPv6 address?
+ try:
+ addrs = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM, socket.IPPROTO_TCP, socket.AI_NUMERICHOST)
+ af = addrs[0][0]
+ except socket.gaierror, e:
+ af = 0
+ if af == socket.AF_INET6:
+ host_str = u"[%s]" % host
+ else:
+ host_str = u"%s" % host
if port is not None:
- result += u":%d" % port
- return result
+ if not (0 < port <= 65535):
+ raise ValueError("port must be between 1 and 65535 (is %d)" % port)
+ port_str = u":%d" % port
+
+ if not host_str and not port_str:
+ raise ValueError("host and port may not both be None")
+ return u"%s%s" % (host_str, port_str)
_______________________________________________
tor-commits mailing list
tor-commits@xxxxxxxxxxxxxxxxxxxx
https://lists.torproject.org/cgi-bin/mailman/listinfo/tor-commits