[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[or-cvs] r12726: 0.1 works okay with web.py's builtin web server. (in weather/tags: . 0.1)



Author: pde
Date: 2007-12-08 19:45:56 -0500 (Sat, 08 Dec 2007)
New Revision: 12726

Added:
   weather/tags/0.1/
   weather/tags/0.1/README
   weather/tags/0.1/config.py
   weather/tags/0.1/poll.py
   weather/tags/0.1/weather.py
Removed:
   weather/tags/0.1/README
   weather/tags/0.1/config.py
   weather/tags/0.1/poll.py
   weather/tags/0.1/weather.py
Log:
0.1 works okay with web.py's builtin web server.



Copied: weather/tags/0.1 (from rev 12325, weather/trunk)

Deleted: weather/tags/0.1/README
===================================================================
--- weather/trunk/README	2007-11-02 04:31:50 UTC (rev 12325)
+++ weather/tags/0.1/README	2007-12-09 00:45:56 UTC (rev 12726)
@@ -1,25 +0,0 @@
-This is the Tor Weather server.  It offers a service that allows users to sign
-up for email alerts in case a particular tor node becomes unreachable.
-
-The process runs a web server which allows users to sign up for these alerts.
-Subscription confirmations, and the email alerts themselves, are sent via SMTP
-on localhost:25.
-
-On debian systems, the following packages are required to run it:
-
-python2.5
-python-gdbm
-python-dns
-python-webpy
-tor
-
-/etc/tor/torrc should be configured to enable the control port and insist upon
-authentication.  Plaintext control port authentication information
-should be placed in config.py, along with a publicly addressable url prefix
-("http://server.domain.com:port";).
-
-Weather stores its records in a set of gdbm databases: requests.gdbm,
-subscriptions.gdbm, unsubscriptions.gdbm, and failures.gdbm.  For real usage,
-it is absolutely essential to backup subscriptions.gdbm properly, and
-unsubscriptions.gdbm is pretty important too (though the code could be modified
-to recover from its loss).

Copied: weather/tags/0.1/README (from rev 12326, weather/trunk/README)
===================================================================
--- weather/tags/0.1/README	                        (rev 0)
+++ weather/tags/0.1/README	2007-12-09 00:45:56 UTC (rev 12726)
@@ -0,0 +1,34 @@
+This is the Tor Weather server.  It offers a service that allows users to sign
+up for email alerts in case a particular tor node becomes unreachable.
+
+The process runs a web server which allows users to sign up for these alerts.
+Subscription confirmations, and the email alerts themselves, are sent via SMTP
+on localhost:25.
+
+On debian systems, the following packages are required to run it:
+
+python2.5
+python-gdbm
+python-dns
+python-webpy
+tor
+
+If you're running this in apache, you'll also need:
+python-flup
+
+/etc/tor/torrc should be configured to enable the control port and insist upon
+authentication.  Plaintext control port authentication information
+should be placed in config.py, along with a publicly addressable url prefix
+("http://server.domain.com:port";).
+
+Weather stores its records in a set of gdbm databases: requests.gdbm,
+subscriptions.gdbm, unsubscriptions.gdbm, and failures.gdbm.  For real usage,
+it is absolutely essential to backup subscriptions.gdbm properly, and
+unsubscriptions.gdbm is pretty important too (though the code could be modified
+to recover from its loss).
+
+By default these files are stored in /var/lib/torweather/
+
+Create this like so:
+mkdir -p /var/lib/torweather && chown www-data:www-data /var/lib/torweather
+

Deleted: weather/tags/0.1/config.py
===================================================================
--- weather/trunk/config.py	2007-11-02 04:31:50 UTC (rev 12325)
+++ weather/tags/0.1/config.py	2007-12-09 00:45:56 UTC (rev 12726)
@@ -1,17 +0,0 @@
-#!/usr/bin/env python2.5
-
-authenticator = ""  # customise this
-
-#URLbase = "http://weather.torproject.org";
-URLbase = "http://ip-adress:port";
-
-weather_email = "no-reply@xxxxxxxxxxxxxx"
-
-# these respond to pings (for now!) and are geographically dispersed
-
-ping_targets = ["google.com", "telstra.com.au", "yahoo.co.uk"]
-
-failure_threshold = 4 # this number of failures in a row counts as being
-                      # down
-
-poll_period = 1800 # try to wait this number of seconds in between polling

Copied: weather/tags/0.1/config.py (from rev 12326, weather/trunk/config.py)
===================================================================
--- weather/tags/0.1/config.py	                        (rev 0)
+++ weather/tags/0.1/config.py	2007-12-09 00:45:56 UTC (rev 12726)
@@ -0,0 +1,18 @@
+#!/usr/bin/env python2.5
+
+authenticator = "frogzify"  # customise this
+
+URLbase = "http://weather.torproject.org";
+
+weather_storage = "/var/lib/torweather/"
+
+weather_email = "tor-ops@xxxxxxxxxxxxxx"
+
+# these respond to pings (for now!) and are geographically dispersed
+
+ping_targets = ["google.com", "telstra.com.au", "yahoo.co.uk"]
+
+failure_threshold = 4 # this number of failures in a row counts as being
+                      # down
+
+poll_period = 1800 # try to wait this number of seconds in between polling

Deleted: weather/tags/0.1/poll.py
===================================================================
--- weather/trunk/poll.py	2007-11-02 04:31:50 UTC (rev 12325)
+++ weather/tags/0.1/poll.py	2007-12-09 00:45:56 UTC (rev 12726)
@@ -1,241 +0,0 @@
-#!/usr/bin/env python2.5
-import socket
-import sys
-import os
-import gdbm
-import re
-import time
-import threading
-from datetime import datetime
-from traceback import print_exception
-from subprocess import Popen, PIPE
-import TorCtl.TorCtl as TorCtl
-
-from config import authenticator, URLbase, weather_email, failure_threshold
-from config import poll_period, ping_targets
-from weather import parse_subscriptions
-
-debug = 0
-
-
-debugfile = open("torctl-debug","w")
-class TorPing:
-  "Check to see if various tor nodes respond to SSL hanshakes"
-  def __init__(self, control_host = "127.0.0.1", control_port = 9051):
-    "Keep the connection to the control port lying around"
-    self.control_host = control_host
-    self.control_port = control_port
-    self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 
-    self.sock.connect((control_host,control_port)) 
-    self.control = TorCtl.Connection(self.sock)
-    self.control.authenticate(authenticator)
-    self.control.debug(debugfile)
-
-  def __del__(self):
-    self.sock.close()
-    del self.sock
-    self.sock = None                # prevents double deletion exceptions
-
-    # it would be better to fix TorCtl!
-    try:
-      self.control.close()
-    except:
-      pass
-
-    del self.control
-    self.control = None
-
-  def ping(self, node_id):
-    "Let's see if this tor node is up."
-    string = "ns/id/" + node_id
-    info = self.control.get_info(string)
-    # info looks like this:
-    # {'ns/id/FFCB46DB1339DA84674C70D7CB586434C4370441': 'r moria1 /8tG2xM52oRnTHDXy1hkNMQ3BEE pavoLDqxMvw+T1VHR5hmmgpr9self 2007-10-10 21:12:08 128.31.0.34 9001 9031\ns Authority Fast Named Running Valid V2Dir\n'}
-    ip,port = info[string].split()[6:8]
-    # throw exceptions like confetti if this isn't legit
-    socket.inet_aton(ip)
-    # ensure port is a kosher string
-    assert 0 < int(port) < 65536
-
-    if debug: print "contacting node at %s:%s" % (ip,port)
-
-    # XXX check: could non-blocking io be used to make us safe from
-    # answer-very-slowly DOSes?  or do we need to spawn threads here?
-
-    cmd = ["openssl", "s_client",  "-connect", ip + ':' + port]
-    ssl_handshake = Popen( args = cmd, stdout = PIPE, stderr = PIPE, stdin=PIPE)
-    ssl_handshake.stdin.close()
-    safe_from_DOS = 10000    # moria1's response is ~1500 chars long
-    output = ssl_handshake.stdout.read(safe_from_DOS)
-    n = output.find("Server public key is 1024 bit")
-    if n > 0:
-      return True
-    else:
-      return False
-
-  def test(self):
-    "Check that the connection to the Tor Control port is still okay."
-    try:
-      self.control.get_info("version")
-      return True
-    except:
-      if debug: print "Respawning control port connection..."
-      self.__del__()
-      try:
-        self.__init__(self.control_host, self.control_port)
-        return True
-      except:
-        if debug: print "Respawn failed"
-        return False
-
-
-report_text = \
-"""This is a Tor Weather report.
-
-It appears that a tor node you elected to monitor,
-
-(node id: %s)
-
-has been uncontactable through the Tor network for a while.  You may wish 
-to look at it to see why.  The last error message from our code while trying to
-contact it is included below.  You may or may not find it helpful!
-
-(You can unsubscribe from these reports at any time by visiting the 
-following url:
-
-%s )
-
-The last error message was as follows:
---------------------------------------
-%s"""
-    
-class WeatherPoller(threading.Thread):
-  "This thread sits around, checking to see if tor nodes are up."
-
-  def __init__(self, subscriptions, lock):
-    #self.subscriptions = gdbm.open("subscriptions")
-    self.gdbm_lock = lock
-    self.subscriptions = subscriptions
-    self.failure_counts = gdbm.open("failures.gdbm", "cs") 
-    self.failure_counts.reorganize() # just in case
-    if debug:
-      print "failure counts"
-      for node in self.failure_counts.keys():
-        print node, self.failure_counts[node]
-    self.tp = TorPing()
-    threading.Thread.__init__(self)
-
-  def run(self):
-    "Keep polling nodes... forever."
-    while True:
-      stamp = time.time()
-      self.ping_all()
-      offset = time.time() - stamp
-      if offset < poll_period:
-        time.sleep(poll_period - offset)
-      
-  def ping_all(self):
-    if debug: print "starting a new round of polls"
-    #self.tp = TorPing()
-    if not self.tp.test():
-      return False
-    print 'Timestamp', datetime.now().isoformat('-')
-    self.gdbm_lock.acquire()
-    node = self.subscriptions.firstkey()
-    while node != None:
-      # nodes stay in the subscription db even if nobody is subscribed to them
-      # anymore
-      if self.subscriptions[node] != "":
-        self.gdbm_lock.release()
-        self.ping(node)       # this is time consuming ; don't hold the lock
-        self.gdbm_lock.acquire()
-
-      node = self.subscriptions.nextkey(node)
-    self.gdbm_lock.release()
-    #del self.tp   # this minimises the chance of confusion a local tor control
-                  # port crash with a remote node being down
-    if debug: print "Ping_all finished"
- 
-  def ping(self, node):
-    if debug: print "pinging", node
-    try: 
-      assert self.tp.ping(node)
-      # Okay we can see this node.  Zero its count, if it has one
-      if debug: print node, "is okay"
-      try:
-        if int(self.failure_counts[node]) != 0:
-          self.failure_counts[node] = "0"
-      except KeyError:
-        pass
-    except:
-      # for /some/ reason, we can't contact this tor node
-      ex1,ex2,ex3 = sys.exc_info()
-      if self.internet_looks_okay():
-        # But we can ping the net.  That's bad.
-        reason = print_exception(ex1,ex2,ex3)
-        if (debug):
-          print "logging a strike against node", node, "because of:"
-          print reason
-        self.strike_against(node, reason)
-      else:
-        print "I would have concluded that tor node", node, "was down;"
-        print "The problem looked like this:"
-        print print_exception(ex1,ex2,ex3)
-        print "But I couldn't ping %s!" % (self.ping_failure)
- 
-  good_ping = re.compile("0% packet loss")
-
-  def internet_looks_okay(self):
-    cmd = ["ping", "-c", "3", "x"]
-    pings = []
-    for host in ping_targets:
-      cmd[3] = host
-      pings.append((Popen(args=cmd,stdout=PIPE,stdin=PIPE,stderr=PIPE), host))
-    for ping,host in pings:
-      output = ping.stdout.read()
-      ping.stdin.close()
-      if not self.good_ping.search(output):
-        self.ping_failure = host
-        return False
-    return True
-      
-  def strike_against(self, node, reason):
-    "Increment the failure count for this node"
-    # gdbm is string based
-    if not self.failure_counts.has_key(node):
-      self.failure_counts[node] = "1"
-    else:
-      count = int(self.failure_counts[node]) + 1
-      self.failure_counts[node] = "%d" %(count)
-      if count == failure_threshold:
-        self.send_failure_email(node, reason)
-
-  def send_failure_email(self, node, reason):
-    import smtplib
-    from email.mime.text import MIMEText
-
-    # Send the message via our own SMTP server, but don"t include the
-    # envelope header.
-    s = smtplib.SMTP()
-    s.connect()
-    self.gdbm_lock.acquire()
-    list = parse_subscriptions(node,self.subscriptions)
-    self.gdbm_lock.release()
-    for address, unsub_token in list:
- 
-      unsub_url = URLbase+"/unsubscribe/" + unsub_token
-      msg= MIMEText(report_text % (node, unsub_url, reason))
-
-      sender = weather_email
-      msg["Subject"] = "Tor weather report"
-      msg["From"] = sender
-      msg["To"] = address
-      msg["List-Unsubscribe"] = unsub_url
-      s.sendmail(sender, [address], msg.as_string())
-    s.close()
-        
-def ping_test():
-  x = NodePoller()
-  print x.internet_looks_okay()
-  x.send_failure_email()
-

Copied: weather/tags/0.1/poll.py (from rev 12349, weather/trunk/poll.py)
===================================================================
--- weather/tags/0.1/poll.py	                        (rev 0)
+++ weather/tags/0.1/poll.py	2007-12-09 00:45:56 UTC (rev 12726)
@@ -0,0 +1,253 @@
+#!/usr/bin/env python2.5
+import socket
+import sys
+import os
+import gdbm
+import re
+import time
+import threading
+from datetime import datetime
+from traceback import format_exc
+from subprocess import Popen, PIPE
+import TorCtl.TorCtl as TorCtl
+
+from config import authenticator, URLbase, weather_email, failure_threshold
+from config import poll_period, ping_targets, weather_storage
+from weather import parse_subscriptions
+
+debug = 0
+
+class NodePollFailure(Exception):
+  pass
+
+debugfile = open(weather_storage + "/torctl-debug","w")
+class TorPing:
+  "Check to see if various tor nodes respond to SSL hanshakes"
+  def __init__(self, control_host = "127.0.0.1", control_port = 9051):
+    "Keep the connection to the control port lying around"
+    self.control_host = control_host
+    self.control_port = control_port
+    self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 
+    self.sock.connect((control_host,control_port)) 
+    self.control = TorCtl.Connection(self.sock)
+    self.control.authenticate(authenticator)
+    self.control.debug(debugfile)
+
+  def __del__(self):
+    self.sock.close()
+    del self.sock
+    self.sock = None                # prevents double deletion exceptions
+
+    # it would be better to fix TorCtl!
+    try:
+      self.control.close()
+    except:
+      pass
+
+    del self.control
+    self.control = None
+
+  def ping(self, node_id):
+    "Let's see if this tor node is up."
+    string = "ns/id/" + node_id
+    info = self.control.get_info(string)
+    # info looks like this:
+    # {'ns/id/FFCB46DB1339DA84674C70D7CB586434C4370441': 'r moria1 /8tG2xM52oRnTHDXy1hkNMQ3BEE pavoLDqxMvw+T1VHR5hmmgpr9self 2007-10-10 21:12:08 128.31.0.34 9001 9031\ns Authority Fast Named Running Valid V2Dir\n'}
+    try:
+      ip,port = info[string].split()[6:8]
+    except:
+      raise NodePollFailure, "Could not extract port and IP from tor client"
+    # throw exceptions like confetti if this isn't legit
+    try:
+      socket.inet_aton(ip)
+      # ensure port is a kosher string
+      assert 0 < int(port) < 65536
+    except:
+      raise NodePollFailure, "Tor client getinfo gave a non-kosher ip:port!"
+
+    if debug: print "contacting node at %s:%s" % (ip,port)
+
+    # XXX check: could non-blocking io be used to make us safe from
+    # answer-very-slowly DOSes?  or do we need to spawn threads here?
+
+    cmd = ["openssl", "s_client",  "-connect", ip + ':' + port]
+    ssl_handshake = Popen( args = cmd, stdout = PIPE, stderr = PIPE, stdin=PIPE)
+    ssl_handshake.stdin.close()
+    safe_from_DOS = 10000    # moria1's response is ~1500 chars long
+    output = ssl_handshake.stdout.read(safe_from_DOS)
+    n = output.find("Server public key is 1024 bit")
+    if n > 0:
+      return True
+    else:
+      raise NodePollFailure, "Cannot SSL handshake to node."
+      #return False
+
+  def test(self):
+    "Check that the connection to the Tor Control port is still okay."
+    try:
+      self.control.get_info("version")
+      return True
+    except:
+      if debug: print "Respawning control port connection..."
+      self.__del__()
+      try:
+        self.__init__(self.control_host, self.control_port)
+        return True
+      except:
+        if debug: print "Respawn failed"
+        return False
+
+
+report_text = \
+"""This is a Tor Weather report.
+
+It appears that a tor node you elected to monitor,
+
+(node id: %s)
+
+has been uncontactable through the Tor network for a while.  You may wish 
+to look at it to see why.  The last error message from our code while trying to
+contact it is included below.  You may or may not find it helpful!
+
+(You can unsubscribe from these reports at any time by visiting the 
+following url:
+
+%s )
+
+The last error message was as follows:
+--------------------------------------
+%s"""
+    
+class WeatherPoller(threading.Thread):
+  "This thread sits around, checking to see if tor nodes are up."
+
+  def __init__(self, subscriptions, lock):
+    #self.subscriptions = gdbm.open(weather_storage + "/subscriptions")
+    self.gdbm_lock = lock
+    self.subscriptions = subscriptions
+    self.failure_counts = gdbm.open(weather_storage + "/failures.gdbm", "cs") 
+    self.failure_counts.reorganize() # just in case
+    if debug:
+      print "failure counts"
+      for node in self.failure_counts.keys():
+        print node, self.failure_counts[node]
+    self.tp = TorPing()
+    threading.Thread.__init__(self)
+
+  def run(self):
+    "Keep polling nodes... forever."
+    while True:
+      stamp = time.time()
+      self.ping_all()
+      offset = time.time() - stamp
+      if offset < poll_period:
+        time.sleep(poll_period - offset)
+      
+  def ping_all(self):
+    if debug: print "starting a new round of polls"
+    #self.tp = TorPing()
+    if not self.tp.test():
+      return False
+    print 'Timestamp', datetime.now().isoformat('-')
+    self.gdbm_lock.acquire()
+    node = self.subscriptions.firstkey()
+    while node != None:
+      # nodes stay in the subscription db even if nobody is subscribed to them
+      # anymore
+      if self.subscriptions[node] != "":
+        self.gdbm_lock.release()
+        self.ping(node)       # this is time consuming ; don't hold the lock
+        self.gdbm_lock.acquire()
+
+      node = self.subscriptions.nextkey(node)
+    self.gdbm_lock.release()
+    #del self.tp   # this minimises the chance of confusion a local tor control
+                   # port crash with a remote node being down
+    if debug: print "Ping_all finished"
+
+ 
+  def ping(self, node):
+    "Is this node there and, to the best of our knowledge, being a tor node?"
+    if debug: print "pinging", node
+    try: 
+      assert self.tp.ping(node)
+      # Okay we can see this node.  Zero its count, if it has one
+      if debug: print node, "is okay"
+      try:
+        if int(self.failure_counts[node]) != 0:
+          self.failure_counts[node] = "0"
+      except KeyError:
+        pass
+    except:
+      # for /some/ reason, we can't contact this tor node
+      #ex1,ex2,ex3 = sys.exc_info()
+      if self.internet_looks_okay():
+        # But we can ping the net.  That's bad.
+        reason = format_exc(500)    # limit to 500 stack levels in emails!
+        if (debug):
+          print "logging a strike against node", node, "because of:"
+          print reason
+        self.strike_against(node, reason)
+      else:
+        print "I would have concluded that tor node", node, "was down;"
+        print "The problem looked like this:"
+        print format_exc()
+        print "But I couldn't ping %s!" % (self.ping_failure)
+ 
+  good_ping = re.compile("0% packet loss")
+
+  def internet_looks_okay(self):
+    "If none of the ping targets are dropping packets, the Internet looks okay."
+    cmd = ["ping", "-c", "3", "x"]
+    pings = []
+    for host in ping_targets:
+      cmd[3] = host
+      pings.append((Popen(args=cmd,stdout=PIPE,stdin=PIPE,stderr=PIPE), host))
+    for ping,host in pings:
+      output = ping.stdout.read()
+      ping.stdin.close()
+      if not self.good_ping.search(output):
+        self.ping_failure = host
+        return False
+    return True
+      
+  def strike_against(self, node, reason):
+    "Increment the failure count for this node"
+    # gdbm is string based
+    if not self.failure_counts.has_key(node):
+      self.failure_counts[node] = "1"
+    else:
+      count = int(self.failure_counts[node]) + 1
+      self.failure_counts[node] = "%d" %(count)
+      if count == failure_threshold:
+        self.send_failure_email(node, reason)
+
+  def send_failure_email(self, node, reason):
+    import smtplib
+    from email.mime.text import MIMEText
+
+    # Send the message via our own SMTP server, but don"t include the
+    # envelope header.
+    s = smtplib.SMTP()
+    s.connect()
+    self.gdbm_lock.acquire()
+    list = parse_subscriptions(node,self.subscriptions)
+    self.gdbm_lock.release()
+    for address, unsub_token in list:
+ 
+      unsub_url = URLbase+"/unsubscribe/" + unsub_token
+      msg= MIMEText(report_text % (node, unsub_url, reason))
+
+      sender = weather_email
+      msg["Subject"] = "Tor weather report"
+      msg["From"] = sender
+      msg["To"] = address
+      msg["List-Unsubscribe"] = unsub_url
+      s.sendmail(sender, [address], msg.as_string())
+    s.close()
+        
+def ping_test():
+  x = NodePoller()
+  print x.internet_looks_okay()
+  x.send_failure_email()
+

Deleted: weather/tags/0.1/weather.py
===================================================================
--- weather/trunk/weather.py	2007-11-02 04:31:50 UTC (rev 12325)
+++ weather/tags/0.1/weather.py	2007-12-09 00:45:56 UTC (rev 12726)
@@ -1,350 +0,0 @@
-#!/usr/bin/env python2.5
-import os
-import web
-import DNS
-import re
-import random
-import sys
-import gdbm
-import time
-import threading
-import signal # does this help with keyboard interrupts?
-import base64
-
-from config import URLbase, weather_email
-
-debug = 0
-dummy_testing = 0
-
-DNS.ParseResolvConf()
-
-urls = (
-'/subscribe', 'subscribe', 
-'/confirm-subscribe/(.*)', 'confirm',
-'/unsubscribe/(.*)', 'unsubscribe'
-)
-
-# Should do something more elegant with this!
-if __name__ == "__main__": 
-
-# This is a single lock for all the gdbm write rights, to ensure that
-# different web.py threads aren't trying to write at the same time.
-
-  gdbm_lock = threading.RLock()
-
-  requests = gdbm.open("requests.gdbm","cs")
-  print "requests:"
-  for s in requests.keys():
-    print s, requests[s]
-  subscriptions = gdbm.open("subscriptions.gdbm","cs")
-  print "subscriptions:"
-  for s in subscriptions.keys():
-    print s, '"'+subscriptions[s]+'"'
-  unsubscriptions = gdbm.open("unsubscriptions.gdbm","cs")
-  print "unsubscriptions:"
-  for s in unsubscriptions.keys():
-    print s, unsubscriptions[s]
-
-  antispam_lock = threading.RLock()
-  antispam = {}      # a dict mapping IP to the number of recent unanswered requests allowed
-                     # from that IP
-  antispam_min = 2
-  antispam_max = 10
-
-# these may or may not be better than storing pickles with gdbm
-
-class DatabaseError(Exception):
-  pass
-
-def parse_subscriptions(node, subs):
-  "Turn a string in the db back into a list of pairs"
-  words = subs[node].split()
-  try:
-    return [ (words[i], words[i+1]) for i in xrange(0, len(words), 2) ]
-  except IndexError:
-    raise DatabaseError, words
-
-def delete_sub(pair, sub, node):
-  "Craziness to delete pair from a string in the subscriptions db"
-  # regexps probably aren't easily made safe here
-  words = sub[node].split()
-  if (len(words) % 2 != 0):
-    raise DatabaseError, words
-  for n in xrange(len(words) / 2):
-    if pair[0] == words[n*2] and pair[1] == words[n*2 + 1]:
-      sub[node] = " ".join(words[:n*2] + words[n*2 + 2:])
-      break
-  else:
-    raise DatabaseError, pair
-  sub.sync()
-      
-def randstring():
-  "Produce a random alphanumeric string for authentication"
-  return base64.urlsafe_b64encode(os.urandom(18))[:-1]
-
-subscribe_text = \
-"""Dear human, this is the Tor Weather Report system.  
-
-Somebody (possibly you) has requested that status monitoring information about 
-a tor node (id: %s) 
-be sent to this email address.
-
-If you wish to confirm this request, please visit the following link:
-
-%s 
-
-If you do *not* wish to receive Tor Weather Reports, you do not need to do 
-anything."""
-
-class subscribe:
-  
-  def GET(self):
-    print open("subscribe.html").read()
-
-  whitespace = re.compile("\s*")
-  def POST(self):
-    i = web.input(node="none",email="none")
-    if not self.check_email(i.email):
-      print 'That email address looks fishy to our refined sensibilities!'
-      if debug: print "(" + self.email_error + ")"
-      return True # XXX temp
-
-    node_cleaned = self.whitespace.sub("", i.node) 
-    if not self.check_node_id(node_cleaned):
-      print "That doesn't look like a proper Tor node ID."
-      return True
-
-    if not self.allowed_to_subscribe(web.ctx.ip):
-      print "Sorry, too many recent unconfirmed subscriptions from your IP address."
-      return True
-
-    if not self.already_subscribed(i.email, node_cleaned):
-      self.send_confirmation_email(i.email, node_cleaned)
-    elif debug:
-      print "Sorry, I'm not subscribing you twice."
-    else:
-      # Leak no information about who is subscribed
-      print "Thankyou for using Tor Weather.  A confirmation request has been sent to", i.email + "."
-
-  # node ids are 40 digit hexidecimal numbers
-  node_okay = re.compile("(0x)?[a-fA-F0-9]{40}\Z")
-
-  def check_node_id(self, node):
-    if self.node_okay.match(node):
-      return True
-    else:
-      return False
-
-  random.seed()
-  def allowed_to_subscribe(self,ip):
-    "An antispam measure!"
-    antispam_lock.acquire()
-    if antispam.has_key(ip):
-      if antispam[ip] == 0:
-        antispam_lock.release()
-        return False
-      else:
-        antispam[ip] -= 1
-        antispam_lock.release()
-        return True
-    else:
-      # okay this is silly but leaks very slightly less information
-      antispam[ip] = random.randrange(antispam_min,antispam_max)
-      antispam_lock.release()
-      return True
-
-  def already_subscribed(self, address, node):
-    gdbm_lock.acquire()
-
-    try:
-      words = subscriptions[node].split()
-      if address in words:
-        already = True
-      else:
-        already = False
-    except KeyError:
-      already = False
-
-    gdbm_lock.release()
-    return already
-    
-  def send_confirmation_email(self, address, node):
-    authstring = randstring()
-
-    gdbm_lock.acquire()
-    requests[authstring] = address + " " + node
-    gdbm_lock.release()
-
-    if dummy_testing:
-      print "gotcha"
-      return True
-
-    #def f(s):
-    #  requests[authstring] = s
-    #gdbm_lock.lock(f, address + " " + node)
-
-    #url = web.ctx.homedomain + "/confirm-subscribe/" + authstring
-    url = URLbase + "/confirm-subscribe/" + authstring
-
-    import smtplib
-    from email.mime.text import MIMEText
-    msg= MIMEText(subscribe_text % (node, url))
-    s = smtplib.SMTP()
-    s.connect()
-    sender = weather_email
-    msg["Subject"] = "Tor weather subscription request"
-    msg["From"] = sender
-    msg["To"] = address
-    s.sendmail(sender, [address], msg.as_string())
-    s.close()
-
-    print "Thankyou for using Tor Weather.  A confirmation request has been sent to", address + "."
-    #print url
-
-  # Section 3.4.1 of RFC 2822 is much more liberal than this!
-  domain_okay = re.compile("[A-Za-z0-9\-\.']*")
-  local_okay = re.compile("[A-Za-z0-9\-_\.\+]*")
-  querinator=DNS.Request(qtype='mx')
-  email_error = None
-
-  def check_email(self, address):
-    "Just check that address is not something fruity"
-    # This is wrong (see http://www.ex-parrot.com/~pdw/Mail-RFC822-Address.html)
-    # but it should prevent crazy stuff from being accepted
-
-    if len(address) >= 80:
-      self.email_error = "We declare this address too long"
-      return False
-    atpos = address.find('@')
-    if atpos == -1:
-      self.email_error = "No @ symbol"
-      return False
-
-    if address[atpos:].find('.') == -1:
-      self.email_error = "No .s after @"
-      return False
-
-    local = address[:atpos]
-    domain = address[atpos + 1:]
-
-    if self.local_okay.match(local).end() != len(local):
-      self.email_error = "unauthorised chars in local part"
-      return False
-
-    for component in domain.split("."):
-      l = len(component)
-      if l == 0:
-        self.email_error = "empty domain segment"
-        return False
-      if self.domain_okay.match(component).end() != l:
-        self.email_error = "unauthorised chars in domain, " + component
-        return False
-
-    # XXX it's not clear yet what exception handling this should do:
-    try:
-      dnsquery = self.querinator.req(domain)
-    except DNS.DNSError, type:
-      if type == 'Timeout':
-        self.email_error = "Can't find a DNS server!"
-        return False
-      else:
-        raise 
-      
-
-    if not dnsquery.answers:
-      # No DNS MX records for this domain
-      self.email_error = "no MX records for domain"
-      return False
-
-    return True
-
-class confirm:
-  def GET(self,authstring):
-    
-    print "<html>"
-    if debug: print "checking confirmation..."
-    gdbm_lock.acquire()
-
-    if not requests.has_key(authstring):
-      print "Error in subscription request!"
-      gdbm_lock.release()
-      return 0
-
-    email, node = requests[authstring].split()
-
-    # We want a single link in every outgoing email that will unsubscribe that
-    # user.  But we don't want to generate a new database entry every time
-    # an email gets sent.  So do it now, and remember the token.
-    unsub_authstring = randstring()
-    subscription = email + " " + node + " " + unsub_authstring
-    unsubscriptions[unsub_authstring] = subscription
-    subscription2 = email + " " + unsub_authstring
-    if subscriptions.has_key(node):
-      subscriptions[node] += " " +subscription2
-    else:
-      subscriptions[node] = subscription2
-    url = web.ctx.homedomain + "/unsubscribe/" + unsub_authstring
-    print "Succesfully subscribed <tt>", email, 
-    print "</tt> to weather reports about Tor node", node
-    print "<p>You can unsubscribe at any time by clicking on the following link:"
-    print '<p><a href="' + url + '">' + url + '</a>'
-    print '<p>(you will be reminded of it in each weather report we send)'
-    
-    del(requests[authstring])
-    subscriptions.sync()
-    gdbm_lock.release()
-    # okay now slacken antispam watch
-    antispam_lock.acquire()
-    if antispam.has_key(web.ctx.ip):
-      antispam[web.ctx.ip] += 1
-      if antispam[web.ctx.ip] >= antispam_max:
-        del antispam[web.ctx.ip]
-    antispam_lock.release()
-
-    
-class unsubscribe:
-  def GET(self,authstring):
-
-    gdbm_lock.acquire()
-    if not unsubscriptions.has_key(authstring):
-      print "Invalid unsubscription request!"
-      print unsubscriptions
-      return 0
-
-    email, node, _ = unsubscriptions[authstring].split()
-
-    delete_sub ((email, authstring), subscriptions, node)
-    #subscriptions[node].remove((email,authstring))
-    print "<html>Succesfully unsubscribed <tt>", email, 
-    print "</tt> from weather reports about Tor node", node, "</html>"
-    del (unsubscriptions[authstring])
-    gdbm_lock.release()
-
-class AntispamRelaxer(threading.Thread):
-  "Prevent long term accretion of antispam counts."
-  timescale = 24 * 3600          # sleep for up to a day
-  def run(self):
-    while True:
-      time.sleep(random.randrange(0,self.timescale))
-      antispam_lock.acquire()
-      for ip in antispam.keys():
-        antispam[ip] += 1
-        if antispam[ip] == antispam_max:
-          del antispam[ip]
-      antispam_lock.release()
-
-def main():
-  from poll import WeatherPoller
-  weather_reports = WeatherPoller(subscriptions, gdbm_lock)
-  weather_reports.start()                 # this starts another thread
-
-  relaxant = AntispamRelaxer()
-  relaxant.start()
-
-  web.run(urls, globals())
-
-
-if __name__ == "__main__": 
-  main()
-  
-

Copied: weather/tags/0.1/weather.py (from rev 12326, weather/trunk/weather.py)
===================================================================
--- weather/tags/0.1/weather.py	                        (rev 0)
+++ weather/tags/0.1/weather.py	2007-12-09 00:45:56 UTC (rev 12726)
@@ -0,0 +1,350 @@
+#!/usr/bin/env python2.5
+import os
+import web
+import DNS
+import re
+import random
+import sys
+import gdbm
+import time
+import threading
+import signal # does this help with keyboard interrupts?
+import base64
+
+from config import URLbase, weather_email, weather_storage
+
+debug = 0
+dummy_testing = 0
+
+DNS.ParseResolvConf()
+
+urls = (
+'/subscribe', 'subscribe', 
+'/confirm-subscribe/(.*)', 'confirm',
+'/unsubscribe/(.*)', 'unsubscribe'
+)
+
+# Should do something more elegant with this!
+if __name__ == "__main__": 
+
+# This is a single lock for all the gdbm write rights, to ensure that
+# different web.py threads aren't trying to write at the same time.
+
+  gdbm_lock = threading.RLock()
+
+  requests = gdbm.open(weather_storage + "/requests.gdbm","cs")
+  print "requests:"
+  for s in requests.keys():
+    print s, requests[s]
+  subscriptions = gdbm.open(weather_storage + "/subscriptions.gdbm","cs")
+  print "subscriptions:"
+  for s in subscriptions.keys():
+    print s, '"'+subscriptions[s]+'"'
+  unsubscriptions = gdbm.open(weather_storage + "/unsubscriptions.gdbm","cs")
+  print "unsubscriptions:"
+  for s in unsubscriptions.keys():
+    print s, unsubscriptions[s]
+
+  antispam_lock = threading.RLock()
+  antispam = {}      # a dict mapping IP to the number of recent unanswered requests allowed
+                     # from that IP
+  antispam_min = 2
+  antispam_max = 10
+
+# these may or may not be better than storing pickles with gdbm
+
+class DatabaseError(Exception):
+  pass
+
+def parse_subscriptions(node, subs):
+  "Turn a string in the db back into a list of pairs"
+  words = subs[node].split()
+  try:
+    return [ (words[i], words[i+1]) for i in xrange(0, len(words), 2) ]
+  except IndexError:
+    raise DatabaseError, words
+
+def delete_sub(pair, sub, node):
+  "Craziness to delete pair from a string in the subscriptions db"
+  # regexps probably aren't easily made safe here
+  words = sub[node].split()
+  if (len(words) % 2 != 0):
+    raise DatabaseError, words
+  for n in xrange(len(words) / 2):
+    if pair[0] == words[n*2] and pair[1] == words[n*2 + 1]:
+      sub[node] = " ".join(words[:n*2] + words[n*2 + 2:])
+      break
+  else:
+    raise DatabaseError, pair
+  sub.sync()
+      
+def randstring():
+  "Produce a random alphanumeric string for authentication"
+  return base64.urlsafe_b64encode(os.urandom(18))[:-1]
+
+subscribe_text = \
+"""Dear human, this is the Tor Weather Report system.  
+
+Somebody (possibly you) has requested that status monitoring information about 
+a tor node (id: %s) 
+be sent to this email address.
+
+If you wish to confirm this request, please visit the following link:
+
+%s 
+
+If you do *not* wish to receive Tor Weather Reports, you do not need to do 
+anything."""
+
+class subscribe:
+  
+  def GET(self):
+    print open("subscribe.html").read()
+
+  whitespace = re.compile("\s*")
+  def POST(self):
+    i = web.input(node="none",email="none")
+    if not self.check_email(i.email):
+      print 'That email address looks fishy to our refined sensibilities!'
+      if debug: print "(" + self.email_error + ")"
+      return True # XXX temp
+
+    node_cleaned = self.whitespace.sub("", i.node) 
+    if not self.check_node_id(node_cleaned):
+      print "That doesn't look like a proper Tor node ID."
+      return True
+
+    if not self.allowed_to_subscribe(web.ctx.ip):
+      print "Sorry, too many recent unconfirmed subscriptions from your IP address."
+      return True
+
+    if not self.already_subscribed(i.email, node_cleaned):
+      self.send_confirmation_email(i.email, node_cleaned)
+    elif debug:
+      print "Sorry, I'm not subscribing you twice."
+    else:
+      # Leak no information about who is subscribed
+      print "Thank you for using Tor Weather.  A confirmation request has been sent to", i.email + "."
+
+  # node ids are 40 digit hexidecimal numbers
+  node_okay = re.compile("(0x)?[a-fA-F0-9]{40}\Z")
+
+  def check_node_id(self, node):
+    if self.node_okay.match(node):
+      return True
+    else:
+      return False
+
+  random.seed()
+  def allowed_to_subscribe(self,ip):
+    "An antispam measure!"
+    antispam_lock.acquire()
+    if antispam.has_key(ip):
+      if antispam[ip] == 0:
+        antispam_lock.release()
+        return False
+      else:
+        antispam[ip] -= 1
+        antispam_lock.release()
+        return True
+    else:
+      # okay this is silly but leaks very slightly less information
+      antispam[ip] = random.randrange(antispam_min,antispam_max)
+      antispam_lock.release()
+      return True
+
+  def already_subscribed(self, address, node):
+    gdbm_lock.acquire()
+
+    try:
+      words = subscriptions[node].split()
+      if address in words:
+        already = True
+      else:
+        already = False
+    except KeyError:
+      already = False
+
+    gdbm_lock.release()
+    return already
+    
+  def send_confirmation_email(self, address, node):
+    authstring = randstring()
+
+    gdbm_lock.acquire()
+    requests[authstring] = address + " " + node
+    gdbm_lock.release()
+
+    if dummy_testing:
+      print "gotcha"
+      return True
+
+    #def f(s):
+    #  requests[authstring] = s
+    #gdbm_lock.lock(f, address + " " + node)
+
+    #url = web.ctx.homedomain + "/confirm-subscribe/" + authstring
+    url = URLbase + "/confirm-subscribe/" + authstring
+
+    import smtplib
+    from email.mime.text import MIMEText
+    msg= MIMEText(subscribe_text % (node, url))
+    s = smtplib.SMTP()
+    s.connect()
+    sender = weather_email
+    msg["Subject"] = "Tor weather subscription request"
+    msg["From"] = sender
+    msg["To"] = address
+    s.sendmail(sender, [address], msg.as_string())
+    s.close()
+
+    print "Thankyou for using Tor Weather.  A confirmation request has been sent to", address + "."
+    #print url
+
+  # Section 3.4.1 of RFC 2822 is much more liberal than this!
+  domain_okay = re.compile("[A-Za-z0-9\-\.']*")
+  local_okay = re.compile("[A-Za-z0-9\-_\.\+]*")
+  querinator=DNS.Request(qtype='mx')
+  email_error = None
+
+  def check_email(self, address):
+    "Just check that address is not something fruity"
+    # This is wrong (see http://www.ex-parrot.com/~pdw/Mail-RFC822-Address.html)
+    # but it should prevent crazy stuff from being accepted
+
+    if len(address) >= 80:
+      self.email_error = "We declare this address too long"
+      return False
+    atpos = address.find('@')
+    if atpos == -1:
+      self.email_error = "No @ symbol"
+      return False
+
+    if address[atpos:].find('.') == -1:
+      self.email_error = "No .s after @"
+      return False
+
+    local = address[:atpos]
+    domain = address[atpos + 1:]
+
+    if self.local_okay.match(local).end() != len(local):
+      self.email_error = "unauthorised chars in local part"
+      return False
+
+    for component in domain.split("."):
+      l = len(component)
+      if l == 0:
+        self.email_error = "empty domain segment"
+        return False
+      if self.domain_okay.match(component).end() != l:
+        self.email_error = "unauthorised chars in domain, " + component
+        return False
+
+    # XXX it's not clear yet what exception handling this should do:
+    try:
+      dnsquery = self.querinator.req(domain)
+    except DNS.DNSError, type:
+      if type == 'Timeout':
+        self.email_error = "Can't find a DNS server!"
+        return False
+      else:
+        raise 
+      
+
+    if not dnsquery.answers:
+      # No DNS MX records for this domain
+      self.email_error = "no MX records for domain"
+      return False
+
+    return True
+
+class confirm:
+  def GET(self,authstring):
+    
+    print "<html>"
+    if debug: print "checking confirmation..."
+    gdbm_lock.acquire()
+
+    if not requests.has_key(authstring):
+      print "Error in subscription request!"
+      gdbm_lock.release()
+      return 0
+
+    email, node = requests[authstring].split()
+
+    # We want a single link in every outgoing email that will unsubscribe that
+    # user.  But we don't want to generate a new database entry every time
+    # an email gets sent.  So do it now, and remember the token.
+    unsub_authstring = randstring()
+    subscription = email + " " + node + " " + unsub_authstring
+    unsubscriptions[unsub_authstring] = subscription
+    subscription2 = email + " " + unsub_authstring
+    if subscriptions.has_key(node):
+      subscriptions[node] += " " +subscription2
+    else:
+      subscriptions[node] = subscription2
+    url = web.ctx.homedomain + "/unsubscribe/" + unsub_authstring
+    print "Succesfully subscribed <tt>", email, 
+    print "</tt> to weather reports about Tor node", node
+    print "<p>You can unsubscribe at any time by clicking on the following link:"
+    print '<p><a href="' + url + '">' + url + '</a>'
+    print '<p>(you will be reminded of it in each weather report we send)'
+    
+    del(requests[authstring])
+    subscriptions.sync()
+    gdbm_lock.release()
+    # okay now slacken antispam watch
+    antispam_lock.acquire()
+    if antispam.has_key(web.ctx.ip):
+      antispam[web.ctx.ip] += 1
+      if antispam[web.ctx.ip] >= antispam_max:
+        del antispam[web.ctx.ip]
+    antispam_lock.release()
+
+    
+class unsubscribe:
+  def GET(self,authstring):
+
+    gdbm_lock.acquire()
+    if not unsubscriptions.has_key(authstring):
+      print "Invalid unsubscription request!"
+      print unsubscriptions
+      return 0
+
+    email, node, _ = unsubscriptions[authstring].split()
+
+    delete_sub ((email, authstring), subscriptions, node)
+    #subscriptions[node].remove((email,authstring))
+    print "<html>Succesfully unsubscribed <tt>", email, 
+    print "</tt> from weather reports about Tor node", node, "</html>"
+    del (unsubscriptions[authstring])
+    gdbm_lock.release()
+
+class AntispamRelaxer(threading.Thread):
+  "Prevent long term accretion of antispam counts."
+  timescale = 24 * 3600          # sleep for up to a day
+  def run(self):
+    while True:
+      time.sleep(random.randrange(0,self.timescale))
+      antispam_lock.acquire()
+      for ip in antispam.keys():
+        antispam[ip] += 1
+        if antispam[ip] == antispam_max:
+          del antispam[ip]
+      antispam_lock.release()
+
+def main():
+  from poll import WeatherPoller
+  weather_reports = WeatherPoller(subscriptions, gdbm_lock)
+  weather_reports.start()                 # this starts another thread
+
+  relaxant = AntispamRelaxer()
+  relaxant.start()
+
+  web.run(urls, globals())
+
+
+if __name__ == "__main__": 
+  main()
+  
+