[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[or-cvs] r17478: {projects} Patch from kaner - added whitelisting, cleaned up blacklisti (projects/gettor)
Author: ioerror
Date: 2008-12-04 04:54:09 -0500 (Thu, 04 Dec 2008)
New Revision: 17478
Modified:
projects/gettor/gettor.py
projects/gettor/gettor_blacklist.py
projects/gettor/gettor_config.py
projects/gettor/gettor_log.py
projects/gettor/gettor_opt.py
projects/gettor/gettor_packages.py
projects/gettor/gettor_responses.py
Log:
Patch from kaner - added whitelisting, cleaned up blacklisting, some fixes / improvements and is lightly broken :)
Modified: projects/gettor/gettor.py
===================================================================
--- projects/gettor/gettor.py 2008-12-03 20:54:49 UTC (rev 17477)
+++ projects/gettor/gettor.py 2008-12-04 09:54:09 UTC (rev 17478)
@@ -69,9 +69,11 @@
# Switch language to 'newlocale'. Return default if language is not supported.
-# XXX: There should be a more elegant way to switch languages during runtime.
def switchLocale(newlocale):
- trans = gettext.translation("gettor", "/usr/share/locale", [newlocale], fallback=True)
+ trans = gettext.translation("gettor",
+ "/usr/share/locale",
+ [newlocale],
+ fallback=True)
trans.install()
def runTests():
@@ -87,11 +89,13 @@
cronProc.communicate()[0]
return cronProc.returncode
-def processMail(conf, log, logLang, packageList):
- if len(packageList) < 1:
+def processMail(conf, log, logLang, packageList, blackList, whiteList):
+ if packageList is None or len(packageList) < 1:
log.error(_("Sorry, your package list is unusable."))
+ log.error(_("Try running with --fetch-packages --prep-packages."))
return False
- # Receive mail
+
+ # Receive mail from stdin
rmail = gettor_requests.requestMail(packageList)
rawMessage = rmail.getRawMessage()
if not rawMessage:
@@ -101,8 +105,8 @@
if not parsedMessage:
log.error(_("No parsed message. Dropping message."))
return False
- # XXX: We should add a blacklist check here so that for exmaple ReplyTo can't be our own
- # address (DoS) (in case we have DKIM)
+ # XXX: We should add a blacklist check here so that for exmaple ReplyTo
+ # can't be our own address (DoS) (in case we have DKIM)
replyTo = rmail.getReplyTo()
if not replyTo:
log.error(_("No help dispatched. Invalid reply address for user."))
@@ -110,26 +114,27 @@
replyLang = rmail.getLocale()
if not replyLang:
replyLang = logLang
+
# Initialize response
srcEmail = conf.getSrcEmail()
resp = gettor_responses.gettorResponse(replyLang, logLang)
signature = rmail.hasVerifiedSignature()
log.info(_("Signature is: %s") % str(signature))
- if not signature:
+ if not signature and not whiteList.lookupListEntry(replyTo):
# Check to see if we've helped them to understand that they need DKIM
# in the past
- previouslyHelped = gettor_blacklist.blackList(replyTo)
+ previouslyHelped = blackList.lookupListEntry(replyTo)
if previouslyHelped:
log.info(_("Unsigned messaged to gettor by blacklisted user dropped."))
return False
else:
# Reply with some help and bail out
- gettor_blacklist.blackList(replyTo, True)
+ blackList.createListEntry(replyTo)
resp.sendHelp(srcEmail, replyTo)
log.info(_("Unsigned messaged to gettor. We issued some help."))
return True
else:
- log.info(_("Signed messaged to gettor."))
+ log.info(_("Good message to gettor."))
package = rmail.getPackage()
if package != None:
log.info(_("Package: %s selected.") % str(package))
@@ -140,53 +145,95 @@
return True
-if __name__ == "__main__":
+def main():
+ success = None
# Parse command line, setup config, logging and language
options, arguments = gettor_opt.parseOpts()
conf = gettor_config.gettorConf(options.configfile)
- logger = gettor_log.gettorLogger()
- log = logger.getLogger()
+ gettor_log.initialize()
+ log = gettor_log.getLogger()
logLang = conf.getLocale()
switchLocale(logLang)
distDir = conf.getDistDir()
if not os.path.isdir(distDir):
- log.info(_("Sorry, %s is not a directory.") % distDir)
- exit(1)
+ log.error(_("Sorry, %s is not a directory.") % distDir)
+ return False
+ packs = gettor_packages.gettorPackages(options.mirror, conf)
+ whiteList = gettor_blacklist.BWList("/tmp/whitelist")
+ blackList = gettor_blacklist.BWList("/tmp/blacklist")
- packs = gettor_packages.gettorPackages(options.mirror, conf)
- # Action
if options.fetchpackages:
if packs.syncWithMirror() != 0:
log.error(_("Syncing Tor packages failed."))
- exit(1)
+ return False
else:
log.info(_("Syncing Tor packages done."))
- exit(0)
+ success = True
if options.preppackages:
if not packs.buildPackages():
log.error(_("Building packages failed."))
- exit(1)
+ return False
else:
log.info(_("Building packages done."))
- exit(0)
+ success = True
if options.runtests:
if not runTests():
log.error(_("Tests failed."))
- exit(1)
+ return False
else:
log.info(_("Tests passed."))
- exit(0)
+ success = True
if options.installcron:
if installCron(packs.getCommandToStr()) != 0:
log.error(_("Installing cron failed"))
- exit(1)
+ return False
else:
log.info(_("Installing cron done."))
- exit(0)
+ success = True
+ if options.whitelist:
+ if not whiteList.createListEntry(options.whitelist):
+ log.error(_("Creating whitelist entry failed."))
+ return False
+ else:
+ log.info(_("Creating whitelist entry ok."))
+ success = True
+ if options.blacklist:
+ if not blackList.createListEntry(options.blacklist):
+ log.error(_("Creating blacklist entry failed."))
+ return False
+ else:
+ log.info(_("Creating blacklist entry ok."))
+ success = True
+ if options.clearwl:
+ if not whiteList.clearAll():
+ log.error(_("Deleting whitelist failed."))
+ return false
+ else:
+ log.info(_("Deleting whitelist done."))
+ success = True
+ if options.clearbl:
+ if not blackList.clearAll():
+ log.error(_("Deleting blacklist failed."))
+ return false
+ else:
+ log.info(_("Deleting blacklist done."))
+ success = True
+
+ # Break here if preparation work has been done
+ if success is not None:
+ return success
# Main loop
- if not processMail(conf, log, logLang, packs.getPackageList()):
+ if not processMail(conf, log, logLang, packs.getPackageList(), blackList,
+ whiteList):
log.error(_("Processing mail failed."))
+ return False
+
+ return True
+
+if __name__ == "__main__":
+ if not main():
+ print >> sys.stderr, _("Main loop exited with errors.")
exit(1)
-
- exit(0)
+ else:
+ exit(0)
Modified: projects/gettor/gettor_blacklist.py
===================================================================
--- projects/gettor/gettor_blacklist.py 2008-12-03 20:54:49 UTC (rev 17477)
+++ projects/gettor/gettor_blacklist.py 2008-12-04 09:54:09 UTC (rev 17478)
@@ -6,110 +6,90 @@
import hashlib
import os
import gettor_config
+import gettor_log
+log = gettor_log.getLogger()
+
conf = gettor_config.gettorConf()
stateDir = conf.getStateDir()
blStateDir = conf.getBlStateDir()
-def blackList(address, createEntry=False):
- """
- Check to see if an address is on our blacklist. If it is - we'll return true.
- If requested, we'll attempt to create a blacklist entry and return true if
- everything works out.
- """
- # XXX TODO: Eventually we may want to sort entries with netcom
- # style /tmp/gettor/2-digits-of-hash/2-more-digits/rest-of-hash files
+class BWList:
+ def __init__(self, listdir):
+ self.listDir = listdir
+ if not os.path.isdir(self.listDir):
+ log.error(_("Bad dir %s.") % self.listDir)
+ # XXX Change this to something more appropriate
+ raise Exception
- prepBLStateDir()
-
- privateAddress = makeAddressPrivate(address)
- blackListed = lookupBlackListEntry(privateAddress)
-
- if blackListed:
+ def lookupListEntry(self, address):
+ """Check to see if we have a list entry for the given address."""
+ entry = self.listDir + "/" + str(hashlib.sha1(address).hexdigest())
+ try:
+ entry = os.stat(entry)
+ except OSError:
+ return False
return True
- elif createEntry:
- return createBlackListEntry(privateAddress)
- return False
-
-def lookupBlackListEntry(privateAddress):
- """ Check to see if we have a blacklist entry for the given address. """
- entry = stateDir + str(privateAddress)
- try:
- entry = os.stat(entry)
- except OSError:
- return False
- return True
-
-def createBlackListEntry(privateAddress):
- """ Create a zero byte file that represents the address in our blacklist. """
- entry = blStateDir + str(privateAddress)
- stat = None
- try:
- stat = os.stat(entry)
- except OSError:
- try:
- fd = open(entry, 'w')
- fd.close()
- return True
- except:
+ def createListEntry(self, address):
+ entry = self.listDir + "/" + str(hashlib.sha1(address).hexdigest())
+ if self.lookupListEntry(address) == False:
+ try:
+ fd = open(entry, 'w')
+ fd.close
+ return True
+ except:
+ log.error(_("Creating list entry %s failed.") % entry)
+ return False
+ else:
+ # List entry already exists
return False
- return False
-def removeBlackListEntry(privateAddress):
- """ Remove the zero byte file that represents an entry in our blacklist."""
- entry = blStateDir + str(privateAddress)
- stat = None
- try:
- entry = os.unlink(entry)
- except OSError:
- return False
- return True
-
-def makeAddressPrivate(address):
- """ Creates a unique identifier for the user. """
- hash = hashlib.sha1(address)
- privateAddress = hash.hexdigest()
- return privateAddress
-
-def prepBLStateDir():
- stat = None
- try:
- stat = os.stat(stateDir)
- return True
- except OSError:
- try:
- os.makedirs(stateDir)
- return True
- except:
+ def removeListEntry(self, address):
+ entry = self.listDir + "/" + str(hashlib.sha1(address).hexdigest())
+ if (self.lookupListEntry(address) == True):
+ try:
+ os.unlink(entry)
+ except OSError:
+ log.error(_("Could not unlink entry %s") % entry)
+ return False
+ else:
+ log.info(_("Requested removal of non-existing entry %s. Abord.")
+ % entry)
return False
+ def removeAll(self):
+ print "Removing all entries from list!"
+
def blackListtests(address):
""" This is a basic evaluation of our blacklist functionality """
- prepBLStateDir()
- privateAddress = makeAddressPrivate(address)
- print "We have a private address of: " + privateAddress
- print "Testing creation of blacklist entry: "
- blackListEntry = createBlackListEntry(privateAddress)
- print blackListEntry
- print "We're testing a lookup of a known positive blacklist entry: "
- blackListEntry = lookupBlackListEntry(privateAddress)
- print blackListEntry
- print "We're testing removal of a known blacklist entry: "
- blackListEntry = removeBlackListEntry(privateAddress)
- print blackListEntry
- print "We're testing a lookup of a known false blacklist entry: "
- blackListEntry = lookupBlackListEntry(privateAddress)
- print blackListEntry
- print "Now we'll test the higher level blacklist functionality..."
- print "This should not find an entry in the blacklist: "
- print blackList(address)
- print "This should add an entry to the blacklist: "
- print blackList(address, True)
- print "This should find the previous addition to the blacklist: "
- print blackList(address)
- print "Please ensure the tests match what you would expect for your environment."
+ bwlist = BWList("/tmp/")
+ print bwlist.createListEntry(address)
+ print bwlist.lookupListEntry(address)
+ #prepBLStateDir()
+ #privateAddress = makeAddressPrivate(address)
+ #print "We have a private address of: " + privateAddress
+ #print "Testing creation of blacklist entry: "
+ #blackListEntry = createBlackListEntry(privateAddress)
+ #print blackListEntry
+ #print "We're testing a lookup of a known positive blacklist entry: "
+ #blackListEntry = lookupBlackListEntry(privateAddress)
+ #print blackListEntry
+ #print "We're testing removal of a known blacklist entry: "
+ #blackListEntry = removeBlackListEntry(privateAddress)
+ #print blackListEntry
+ #print "We're testing a lookup of a known false blacklist entry: "
+ #blackListEntry = lookupBlackListEntry(privateAddress)
+ #print blackListEntry
+ #print "Now we'll test the higher level blacklist functionality..."
+ #print "This should not find an entry in the blacklist: "
+ #print blackList(address)
+ #print "This should add an entry to the blacklist: "
+ #print blackList(address, True)
+ #print "This should find the previous addition to the blacklist: "
+ #print blackList(address)
+ #print "Please ensure the tests match what you would expect for your environment."
if __name__ == "__main__" :
- print "It appears that you're manually calling the blacklisting code. We'll run some tests..."
+ print "Running some tests.."
blackListtests("requestingUser@xxxxxxxxxxx")
Modified: projects/gettor/gettor_config.py
===================================================================
--- projects/gettor/gettor_config.py 2008-12-03 20:54:49 UTC (rev 17477)
+++ projects/gettor/gettor_config.py 2008-12-04 09:54:09 UTC (rev 17478)
@@ -93,23 +93,11 @@
# We're trying to be failsafe here
self.outConf = ConfigParser.ConfigParser()
- # See if config file is accessable
try:
if os.access(self.configFile, os.R_OK):
- readableConfigFile = True
- else:
- readableConfigFile = False
-
- except OSError:
- readableConfigFile = False
-
- if readableConfigFile:
- try:
- # It's likely that a user will make a mistake in their config
- # If they make a mistake for now we'll ignore *everything* :-)
self.config.read(self.configFile)
- except:
- pass
+ except:
+ pass
# Main parser loop:
# * Merge default values with those from the config file, if there are
Modified: projects/gettor/gettor_log.py
===================================================================
--- projects/gettor/gettor_log.py 2008-12-03 20:54:49 UTC (rev 17477)
+++ projects/gettor/gettor_log.py 2008-12-04 09:54:09 UTC (rev 17478)
@@ -25,59 +25,62 @@
import gettor_config
from logging import handlers
-__all__ = ["gettorLogger"]
+__all__ = ["initalize", "getLogger", "getLogSubSystem"]
# Leave this to INFO for now
loglevel = logging.INFO
+format = '%(asctime)-15s (%(process)d) %(message)s'
+logger = None
+logSubSystem = None
-class gettorLogger:
- """A configurable logging system for gettor.
- """
+def initialize():
+ global logger
+ global logSubSystem
+ config = gettor_config.gettorConf()
+ logger = logging.getLogger('gettor')
+ logger.setLevel(loglevel)
+ logSubSystem = config.getLogSubSystem()
- format = '%(asctime)-15s (%(process)d) %(message)s'
+ if logSubSystem == "stdout":
+ handler = logging.StreamHandler()
+ elif logSubSystem == "file":
+ # Silently fail if things are misconfigured
+ logFile = config.getLogFile()
+ try:
+ if os.access(os.path.dirname(logFile), os.W_OK):
+ handler = logging.FileHandler(logFile)
+ else:
+ logSubSystem = "nothing"
+ except:
+ logSubSystem = "nothing"
+ elif logSubSystem == "syslog":
+ handler = logging.handlers.SysLogHandler(address="/dev/log")
+ else:
+ # Failsafe fallback
+ logSubSystem = "nothing"
- def __init__(self):
- self.config = gettor_config.gettorConf()
- self.logger = logging.getLogger('gettor')
- self.logger.setLevel(loglevel)
- self.logSubSystem = self.config.getLogSubSystem()
+ # If anything went wrong or the user doesn't want to log
+ if logSubSystem == "nothing":
+ handler = logging.FileHandler(os.devnull)
- if self.logSubSystem == "stdout":
- handler = logging.StreamHandler()
- elif self.logSubSystem == "file":
- # Silently fail if things are misconfigured
- self.logFile = self.config.getLogFile()
- try:
- if os.access(os.path.dirname(self.logFile), os.W_OK):
- handler = logging.FileHandler(self.logFile)
- else:
- self.logSubSystem = "nothing"
- except:
- self.logSubSystem = "nothing"
- elif self.logSubSystem == "syslog":
- handler = logging.handlers.SysLogHandler(address="/dev/log")
- else:
- # Failsafe fallback
- self.logSubSystem = "nothing"
+ formatter = logging.Formatter(fmt=format)
+ handler.setFormatter(formatter)
+ logger.addHandler(handler)
- # If anything went wrong or the user doesn't want to log
- if self.logSubSystem == "nothing":
- handler = logging.FileHandler(os.devnull)
+def getLogSubSystem():
+ global logSubSystem
+ return logSubSystem
- formatter = logging.Formatter(fmt=self.format)
- handler.setFormatter(formatter)
- self.logger.addHandler(handler)
+def getLogger():
+ global logger
+ if logger is None:
+ initialize()
+ return logger
- def getLogSubSystem(self):
- return self.logSubSystem
-
- def getLogger(self):
- return self.logger
-
if __name__ == "__main__" :
- l = gettorLogger()
+ initialize()
print "This is the logging module. You probably do not want to call it by hand."
print "We'll send a test logging message now with the following subsystem: " + \
- l.getLogSubSystem()
- log = l.getLogger()
+ getLogSubSystem()
+ log = getLogger()
log.info("I'm a logger, logging!")
Modified: projects/gettor/gettor_opt.py
===================================================================
--- projects/gettor/gettor_opt.py 2008-12-03 20:54:49 UTC (rev 17477)
+++ projects/gettor/gettor_opt.py 2008-12-04 09:54:09 UTC (rev 17478)
@@ -23,7 +23,8 @@
help="set config file to FILE", metavar="FILE")
cmdParser.add_option("-m", "--use-mirror", dest="mirror",
default="rsync.torproject.org",
- help="set Tor package mirror to MIRROR", metavar="MIRROR")
+ help="set Tor package mirror to MIRROR",
+ metavar="MIRROR")
cmdParser.add_option("-i", "--install-crontab", dest="installcron",
action="store_true", default=False,
help="install crontab to refresh packagelist")
@@ -36,8 +37,22 @@
cmdParser.add_option("-t", "--run-tests", dest="runtests",
action="store_true", default=False,
help="run some tests")
+ cmdParser.add_option("-w", "--whitelist", dest="whitelist",
+ default="",
+ help="add an email address to the whitelist",
+ metavar="WHITELIST")
+ cmdParser.add_option("-b", "--blacklist", dest="blacklist",
+ default="",
+ help="add an email address to the blacklist",
+ metavar="BLACKLIST")
+ cmdParser.add_option("-x", "--clear-whitelist", dest="clearwl",
+ action="store_true", default=False,
+ help="clear all entrys in the whitelist")
+ cmdParser.add_option("-y", "--clear-blacklist", dest="clearbl",
+ action="store_true", default=False,
+ help="clear all entrys in the blacklist")
return cmdParser.parse_args()
if __name__ == "__main__":
- print >> sys.stderr "You shouldn't run this directly."
+ print >> sys.stderr, "You shouldn't run this directly."
Modified: projects/gettor/gettor_packages.py
===================================================================
--- projects/gettor/gettor_packages.py 2008-12-03 20:54:49 UTC (rev 17477)
+++ projects/gettor/gettor_packages.py 2008-12-04 09:54:09 UTC (rev 17478)
@@ -20,6 +20,8 @@
__all__ = ["gettorPackages"]
+log = gettor_log.getLogger()
+
class gettorPackages:
packageRegex = { "windows-bundle": "vidalia-bundle-.*.exe$",
@@ -44,8 +46,13 @@
def getPackageList(self):
# Build dict like 'name': 'name.z'
- for filename in os.listdir(self.packDir):
- self.packageList[filename[:-2]] = self.packDir + "/" + filename
+ try:
+ for filename in os.listdir(self.packDir):
+ self.packageList[filename[:-2]] = self.packDir + "/" + filename
+ except OSError, (strerror):
+ log.error(_("Failed to build package list: %s") % strerror)
+ return None
+
# Check sanity
for key, val in self.packageList.items():
# Remove invalid packages
@@ -82,10 +89,7 @@
def getCommandToStr(self):
"""This is useful for cronjob installations
"""
- rsyncstr = ""
- for i,v in enumerate(self.rsync):
- rsyncstr += self.rsync[i] + " "
- return rsyncstr
+ return ''.join(self.rsync)
if __name__ == "__main__" :
c = gettor_config.gettorConf()
Modified: projects/gettor/gettor_responses.py
===================================================================
--- projects/gettor/gettor_responses.py 2008-12-03 20:54:49 UTC (rev 17477)
+++ projects/gettor/gettor_responses.py 2008-12-04 09:54:09 UTC (rev 17478)
@@ -19,6 +19,7 @@
__all__ = ["gettorResponse"]
+
class gettorResponse:
def __init__(self, mailLang="en", logLang="en"):
@@ -27,8 +28,10 @@
def setLang(self, language):
# XXX: Sorta hack, have nothing better on my mind right now
- # On every entry to a translation-needing function, call this with lang=maillang
- # On every exit of a translation-needing function, call this with lang=loglang
+ # On every entry to a translation-needing function, call this with
+ # lang=maillang
+ # On every exit of a translation-needing function, call this with
+ # lang=loglang
# :-/
trans = gettext.translation("gettor", "/usr/share/locale", [language])
trans.install()