[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[or-cvs] r21638: {projects} Blacklist handling - Deleting blacklist files after X days i (in projects/gettor: . lib/gettor)



Author: kaner
Date: 2010-02-14 12:55:45 +0000 (Sun, 14 Feb 2010)
New Revision: 21638

Modified:
   projects/gettor/GetTor.py
   projects/gettor/lib/gettor/blacklist.py
   projects/gettor/lib/gettor/constants.py
   projects/gettor/lib/gettor/opt.py
   projects/gettor/lib/gettor/utils.py
Log:
Blacklist handling
- Deleting blacklist files after X days is now part of the GetTor code
- No need to have a 'find -mtime' bash cronjob anymore
- Change commandline option for deleting blacklist entries to accept a DAYS parameter, indicating how old blacklist entries are allowed to be, in days


Modified: projects/gettor/GetTor.py
===================================================================
--- projects/gettor/GetTor.py	2010-02-14 02:22:26 UTC (rev 21637)
+++ projects/gettor/GetTor.py	2010-02-14 12:55:45 UTC (rev 21638)
@@ -97,8 +97,8 @@
         gettor.utils.lookupAddress(conf, options.lookup)
     if options.clearwl:
         gettor.utils.clearWhitelist(conf)
-    if options.clearbl:
-        gettor.utils.clearBlacklist(conf)
+    if options.days:
+        gettor.utils.clearBlacklist(conf, options.days)
     if options.cmdpass:
         gettor.utils.setCmdPassword(conf, options.cmdpass)
 

Modified: projects/gettor/lib/gettor/blacklist.py
===================================================================
--- projects/gettor/lib/gettor/blacklist.py	2010-02-14 02:22:26 UTC (rev 21637)
+++ projects/gettor/lib/gettor/blacklist.py	2010-02-14 12:55:45 UTC (rev 21638)
@@ -78,22 +78,23 @@
             log.info("Requested removal of non-existing entry. Abord.")
             return False
 
-    def removeAll(self):
-        print "Removing all entries from list!"
+    def removeAll(self, olderThanDays=0):
         for root, dirs, files in os.walk(self.blacklistDir):
             for file in files:
                 rmfile = os.path.join(root, file)
-                try:
-                    os.remove(rmfile)
-                except OSError:
+                # Only remove files older than 'olderThanDays'
+                if gettor.utils.fileIsOlderThan(rmfile, olderThanDays):
                     try:
-                        os.rmdir(rmfile)
+                        os.remove(rmfile)
+                    except OSError:
+                        try:
+                            os.rmdir(rmfile)
+                        except:
+                            log.error("Could not remove %s." % rmfile)
+                            return False
                     except:
                         log.error("Could not remove %s." % rmfile)
                         return False
-                except:
-                    log.error("Could not remove %s." % rmfile)
-                    return False
         return True
 
     def stripEmail(self, address):

Modified: projects/gettor/lib/gettor/constants.py
===================================================================
--- projects/gettor/lib/gettor/constants.py	2010-02-14 02:22:26 UTC (rev 21637)
+++ projects/gettor/lib/gettor/constants.py	2010-02-14 12:55:45 UTC (rev 21638)
@@ -502,7 +502,7 @@
     GetTor you can use to make it send you a number of small packages
     instead of one big one.
 
-    """
+    """)
 split_help_2 = _("""
     Simply include the keyword 'split' somewhere in your email like so:
     """)
@@ -511,7 +511,7 @@
         tor-browser-bundle
         split
 
-    """)
+    """
 split_help_4 = _("""
     Sending this text in an email to GetTor will cause it to send you 
     the Tor Browser Bundle in a number of 1,4MB attachments.
@@ -551,7 +551,7 @@
 split_help_11 = """
             http://www.win-rar.com/download.html
 
-    """)
+    """
 split_help_12 = _("""
         To unpack your Tor package, simply doubleclick the ".exe" file.
 
@@ -565,7 +565,7 @@
 split_help_14 = _("""
     7.) That's it. You're done. Thanks for using Tor and have fun!
     
-    """
+    """)
 support = _("""
     SUPPORT
     """)
@@ -666,7 +666,7 @@
                  obtain_localized_1 + obtain_localized_2 + obtain_localized_3 + \
                  list_of_langs_head + list_of_langs_underline + \
                  list_of_langs_1 + list_of_langs_2 + list_of_langs_3 + \
-                 split_help_head + split_help_underline \
+                 split_help_head + split_help_underline + \
                  split_help_1 + split_help_2 + split_help_3 + split_help_4 + \
                  split_help_5 + split_help_6 + split_help_7 + split_help_8 + \
                  split_help_9 + split_help_10 + split_help_11 + split_help_12 + \

Modified: projects/gettor/lib/gettor/opt.py
===================================================================
--- projects/gettor/lib/gettor/opt.py	2010-02-14 02:22:26 UTC (rev 21637)
+++ projects/gettor/lib/gettor/opt.py	2010-02-14 12:55:45 UTC (rev 21638)
@@ -34,9 +34,6 @@
     cmdParser.add_option("-p", "--prep-packages", dest="preppackages",
                         action="store_true", default=False,
                         help="prepare packages (zip them)")
-    cmdParser.add_option("-t", "--run-tests", dest="runtests",
-                        action="store_true", default=False,
-                        help="run some tests")
     cmdParser.add_option("-w", "--whitelist", dest="whitelist",
                          default="",
                          help="add an email address to the whitelist",
@@ -52,19 +49,19 @@
     cmdParser.add_option("-x", "--clear-whitelist", dest="clearwl",
                         action="store_true", default=False,
                         help="clear all entrys in the whitelist")
-    cmdParser.add_option("-y", "--clear-blacklist", dest="clearbl",
-                        action="store_true", default=False,
-                        help="clear all entrys in the blacklist")
+    cmdParser.add_option("-y", "--clear-blacklist", dest="days",
+                        default=0,
+                        help="clear all entrys in the blacklist older than DAYS days")
     cmdParser.add_option("-r", "--install-translations", dest="insttrans",
                         action="store_true", default=False,
-                        help="Compile and install translation files [check -d]")
+                        help="compile and install translation files [check -d]")
     cmdParser.add_option("-s", "--set-cmdpassword", dest="cmdpass",
                         default="",
-                        help="Set the password for mail commands",
+                        help="set the password for mail commands",
                         metavar="CMDPASS")
     cmdParser.add_option("-d", "--i18n-dir", dest="i18ndir",
                         default="./i18n",
-                        help="Set your locale src dir to DIR [default = %default]",
+                        help="set your locale src dir to DIR [default = %default]",
                         metavar="DIR")
 
     return cmdParser.parse_args()

Modified: projects/gettor/lib/gettor/utils.py
===================================================================
--- projects/gettor/lib/gettor/utils.py	2010-02-14 02:22:26 UTC (rev 21637)
+++ projects/gettor/lib/gettor/utils.py	2010-02-14 12:55:45 UTC (rev 21638)
@@ -16,7 +16,8 @@
 import re
 import subprocess
 import hashlib
-import datetime
+from datetime import date, timedelta, datetime
+from time import localtime
 
 import gettor.gtlog
 import gettor.blacklist
@@ -145,7 +146,7 @@
     # XXX: Check if cron is installed and understands our syntax?
     currentCronTab = getCurrentCrontab()
     path = os.getcwd() + "/" + os.path.basename(sys.argv[0])
-    args = " --clear-blacklist --fetch-packages --prep-packages"
+    args = " --clear-blacklist=7 --fetch-packages --prep-packages"
     newCronTab = currentCronTab + '\n' + '3 2 * * * ' + path + args
     echoCmd = ['echo', newCronTab ]
     cronCmd = ['crontab', '-']
@@ -218,14 +219,14 @@
         log.info("Deleting whitelist done.")
         return True
 
-def clearBlacklist(conf):
+def clearBlacklist(conf, olderThanDays):
     log.info("Clearing blacklist..")
     try:
         blackList = gettor.blacklist.BWList(conf.getBlStateDir())
     except IOError, e:
         log.error("Blacklist error: %s" % e)
         return False
-    if not blackList.removeAll():
+    if not blackList.removeAll(olderThanDays):
         log.error("Deleting blacklist failed.")
         return False
     else:
@@ -288,6 +289,18 @@
 
     return newfilename
 
+def fileIsOlderThan(filename, olderThanDays):
+    olderThanDays = int(olderThanDays)
+    if olderThanDays is not 0:
+        daysold = datetime.now() - timedelta(days=olderThanDays)
+        daysold = daysold.timetuple()
+        filetimeUnix = os.path.getmtime(filename)
+        filetime = localtime(filetimeUnix)
+        if daysold < filetime:
+            return False
+        
+    return True
+
 # Helper routines go here ####################################################
 
 def installMo(poFile, targetDir):