[or-cvs] r22600: {arm} Revisions to project 113 to track new relays (over the durat (arm/trunk/init)

[Damian Johnson <atagar1@xxxxxxxxx>]

Author: atagar
Date: 2010-07-05 05:20:11 +0000 (Mon, 05 Jul 2010)
New Revision: 22600

Modified:
   arm/trunk/init/project113.py
Log:
Revisions to project 113 to track new relays (over the duration of the runtime) rather than just since the last sampling.



Modified: arm/trunk/init/project113.py
===================================================================
--- arm/trunk/init/project113.py	2010-07-05 02:21:48 UTC (rev 22599)
+++ arm/trunk/init/project113.py	2010-07-05 05:20:11 UTC (rev 22600)
@@ -6,6 +6,9 @@
 if it changes dramatically throughout the week.
 """
 
+# TODO: this whole script is experimental and should be rewritten once we
+# figure out what works best...
+
 import sys
 import time
 import getpass
@@ -21,12 +24,15 @@
 USERNAME = ""
 PASSWORD = ""
 RECEIVER = ""
+ALERT_HOURLY_DROP = False # sends alert for hourly network shrinking if true
 
 # size of change (+/-) at which an alert is sent
 BIHOURLY_THRESHOLD = 15
 DAILY_THRESHOLD = 50
 WEEKLY_THRESHOLD = 100
 
+SEEN_FINGERPRINTS = set()
+
 def sendAlert(msg):
   mimeMsg = MIMEText(msg)
   mimeMsg['Subject'] = "Tor Relay Threshold Alert"
@@ -68,6 +74,15 @@
   
   return exitEntries
 
+def getNewExits(newEntries):
+  # provides relays that have never been seen before
+  diffMapping = dict([(entry.idhex, entry) for entry in newEntries])
+  
+  for fingerprint in SEEN_FINGERPRINTS:
+    if fingerprint in diffMapping.keys(): del diffMapping[fingerprint]
+  
+  return diffMapping.values()
+
 def getExitsDiff(newEntries, oldEntries):
   # provides relays in newEntries but not oldEntries
   diffMapping = dict([(entry.idhex, entry) for entry in newEntries])
@@ -81,31 +96,40 @@
   if not PASSWORD: PASSWORD = getpass.getpass("GMail Password: ")
   conn = util.torTools.connect()
   counts = [] # has entries for up to the past week
+  newCounts = [] # parallel listing for new entries added on each time period
   nsEntries = [] # parallel listing for exiting ns entries
   lastQuery = 0
+  tick = 0
   
   while True:
+    tick += 1
+    
     # sleep for a couple hours
     while time.time() < (lastQuery + SAMPLING_INTERVAL):
       sleepTime = max(1, SAMPLING_INTERVAL - (time.time() - lastQuery))
       time.sleep(sleepTime)
     
     # adds new count to the beginning
-    #newCount = getCount(conn)
     exitEntries = getExits(conn)
-    newCount = len(exitEntries)
+    newExitEntries = getNewExits(exitEntries)
+    count = len(exitEntries)
+    newCount = len(newExitEntries)
     
-    counts.insert(0, newCount)
+    counts.insert(0, count)
+    newCounts.insert(0, newCount)
     nsEntries.insert(0, exitEntries)
     if len(counts) > 84:
       counts.pop()
+      newCounts.pop()
       nsEntries.pop()
     
     # check if we broke any thresholds (alert at the lowest increment)
     alarmHourly, alarmDaily, alarmWeekly = False, False, False
     
     if len(counts) >= 2:
-      alarmHourly = abs(newCount - counts[1]) >= BIHOURLY_THRESHOLD
+      #if ALERT_HOURLY_DROP: alarmHourly = abs(count - counts[1]) >= BIHOURLY_THRESHOLD
+      #else: alarmHourly = count - counts[1] >= BIHOURLY_THRESHOLD
+      alarmHourly = newCounts >= BIHOURLY_THRESHOLD
     
     if len(counts) >= 3:
       dayMin, dayMax = min(counts[:12]), max(counts[:12])
@@ -118,10 +142,10 @@
     # notes entry on terminal
     lastQuery = time.time()
     timeLabel = time.strftime("%H:%M %m/%d/%Y", time.localtime(lastQuery))
-    print "%s - %s exits" % (timeLabel, newCount)
+    print "%s - %s exits (%s new)" % (timeLabel, count, newCount)
     
     # sends a notice with counts for the last week
-    if alarmHourly or alarmDaily or alarmWeekly:
+    if tick > 5 and (alarmHourly or alarmDaily or alarmWeekly):
       if alarmHourly: threshold = "hourly"
       elif alarmDaily: threshold = "daily"
       elif alarmWeekly: threshold = "weekly"
@@ -130,14 +154,14 @@
       
       msg += "\nexit counts:\n"
       entryTime = lastQuery
-      for countEntry in counts:
+      for i in range(len(counts)):
+        countEntry, newCountEntry = counts[i], newCounts[i]
         timeLabel = time.strftime("%H:%M %m/%d/%Y", time.localtime(entryTime))
-        msg += "%s - %i\n" % (timeLabel, countEntry)
+        msg += "%s - %i (%i new)\n" % (timeLabel, countEntry, newCountEntry)
         entryTime -= SAMPLING_INTERVAL
       
       msg += "\nnew exits (hourly):\n"
-      entriesDiff = getExitsDiff(nsEntries[0], nsEntries[1])
-      for entry in entriesDiff:
+      for entry in newExitEntries:
         msg += "%s (%s:%s)\n" % (entry.idhex, entry.ip, entry.orport)
         msg += "    nickname: %s\n    flags: %s\n\n" % (entry.nickname, ", ".join(entry.flags))
       
@@ -158,6 +182,10 @@
       
       sendAlert(msg)
       
+      # add all new fingerprints to seen set
+      for entry in nsEntries[0]:
+        SEEN_FINGERPRINTS.add(entry.idhex)
+      
       # clears entries so we don't repeatidly send alarms for the same event
       if alarmDaily: del counts[2:]
       elif alarmWeekly: del counts[12:]