[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[or-cvs] r21634: {projects} Remove old files from rsync dirs on sync (in projects/gettor: . lib/gettor)



Author: kaner
Date: 2010-02-14 00:40:06 +0000 (Sun, 14 Feb 2010)
New Revision: 21634

Modified:
   projects/gettor/TODO
   projects/gettor/lib/gettor/packages.py
   projects/gettor/lib/gettor/utils.py
Log:
Remove old files from rsync dirs on sync


Modified: projects/gettor/TODO
===================================================================
--- projects/gettor/TODO	2010-02-13 19:09:00 UTC (rev 21633)
+++ projects/gettor/TODO	2010-02-14 00:40:06 UTC (rev 21634)
@@ -1,6 +1,5 @@
 These are planned changes to the gettor system.
 
-- Clean distdir and packdir on each fetching/packaging run
 - Split (at least) tiger bundle into several smaller archives to avoid ~20MB
 - React in some way if the user requested a split download but no split 
   downloads are available for that specific package (maybe send out the non-

Modified: projects/gettor/lib/gettor/packages.py
===================================================================
--- projects/gettor/lib/gettor/packages.py	2010-02-13 19:09:00 UTC (rev 21633)
+++ projects/gettor/lib/gettor/packages.py	2010-02-14 00:40:06 UTC (rev 21634)
@@ -97,35 +97,91 @@
 
     def buildPackages(self):
         for (pack, (regex_single, regex_split)) in self.packageRegex.items():
-            for filename in os.listdir(self.distDir):
-                # Splitfile hacks. XXX: Refactor
-                if re.compile(regex_split).match(filename):
-                    if not self.buildSplitFiles(pack, filename):
-                        log.error("Could not build split files packages")
-                        return False
-                if re.compile(regex_single).match(filename):
-                    file = os.path.join(self.distDir, filename)
-                    ascfile = file + ".asc"
-                    zpack = pack + ".z"
-                    zipFileName  = os.path.join(self.packDir, zpack)
-                    # If .asc file is there, build Zip file
-                    if os.access(ascfile, os.R_OK):
-                        zip = zipfile.ZipFile(zipFileName, "w")
-                        zip.write(file, os.path.basename(file))
-                        zip.write(ascfile, os.path.basename(ascfile))
-                        zip.close()
-                        self.packageList[pack] = zipFileName
-                        break
+            for dirname in os.listdir(self.distDir):
+                subdir = os.path.join(self.distDir, dirname)
+                for filename in os.listdir(subdir):
+                    # Splitfile hacks. XXX: Refactor
+                    if re.compile(regex_split).match(filename):
+                        if not self.buildSplitFiles(pack, subdir, filename):
+                            log.error("Could not build split files packages")
+                            return False
+                    if re.compile(regex_single).match(filename):
+                        file = os.path.join(subdir, filename)
+                        ascfile = file + ".asc"
+                        zpack = pack + ".z"
+                        zipFileName  = os.path.join(self.packDir, zpack)
+                        # If .asc file is there, build Zip file
+                        if os.access(ascfile, os.R_OK):
+                            zip = zipfile.ZipFile(zipFileName, "w")
+                            zip.write(file, os.path.basename(file))
+                            zip.write(ascfile, os.path.basename(ascfile))
+                            zip.close()
+                            self.packageList[pack] = zipFileName
+                            break
         if len(self.packageList) > 0:
             return True
         else:
             log.error("Failed to build packages")
             return False
 
+    def buildSplitFiles(self, pack, dirname, filename):
+        log.info("Building split files..")
+        packSplitDir = None
+        try:
+            splitpack = pack + ".split"
+            packSplitDir = os.path.join(self.packDir, splitpack)
+            if not os.access(packSplitDir, os.R_OK):
+                os.mkdir(packSplitDir)
+        except OSError, e:
+            log.error("Could not create dir %s: %s" \
+                            % (packSplitDir, e))
+        # Loop through split dir, look if every partXX.ZZZ has a 
+        # matching signature, pack them together in a .z
+        splitdir = os.path.join(dirname, filename)
+        for splitfile in os.listdir(splitdir):
+            # Skip signature files
+            if splitfile.endswith(".asc"):
+                continue
+            if re.compile(".*split.part.*").match(splitfile):
+                ascsplit = splitfile + ".asc"
+                ascfile = os.path.join(splitdir, "signatures", ascsplit)
+                # Rename .exe if needed
+                if gettor.utils.hasExe(ascfile):
+                    try:
+                        ascfile = gettor.utils.renameExe(ascfile)
+                    except:
+                        log.error("Could not rename exe file")
+                file = os.path.join(splitdir, splitfile)
+                if gettor.utils.hasExe(file):
+                    try:
+                        file = gettor.utils.renameExe(file)
+                    except:
+                        log.error("Could not rename exe file")
+                zsplitfile = splitfile + ".z"
+                zipFileName = os.path.join(packSplitDir, zsplitfile)
+                if gettor.utils.hasExe(zipFileName):
+                    try:
+                        zipFileName = gettor.utils.renameExe(zipFileName, False)
+                    except:
+                        log.error("Could not rename zip file exe")
+                        return False
+                if os.access(ascfile, os.R_OK) and os.access(file, os.R_OK):
+                    zip = zipfile.ZipFile(zipFileName, "w")
+                    zip.write(file, os.path.basename(file))
+                    zip.write(ascfile, os.path.basename(ascfile))
+                    zip.close()
+                else:
+                    log.error("Uhm, expected signature file for %s to be: %s" % (file, ascfile))
+
+        log.info("Done.")
+        return True
+
     def initRsync(self, mirror="rsync.torproject.org", silent=False):
         # Rsync command 1
         self.rsync = "rsync -a" 
         self.rsync += " "
+        self.rsync += "--delete"
+        self.rsync += " "
         self.rsync += "--exclude='*current*'"
         self.rsync += " "
         self.rsync += "--exclude='*osx*'"
@@ -143,13 +199,15 @@
             self.rsync += " "
         self.rsync += "rsync://%s/tor/dist/" % mirror
         self.rsync += " "
-        self.rsync += self.distDir
+        self.rsync += self.distDir + "_source"
         self.rsync += " "
         self.rsync += "&&"
         self.rsync += " "
         # Rsync command 2
         self.rsync += "rsync -a"
         self.rsync += " "
+        self.rsync += "--delete"
+        self.rsync += " "
         self.rsync += "--exclude='*current*'"
         self.rsync += " "
         self.rsync += "--exclude='*osx*'"
@@ -167,13 +225,15 @@
             self.rsync += " "
         self.rsync += "rsync://%s/tor/torbrowser/dist/" % mirror
         self.rsync += " "
-        self.rsync += self.distDir
+        self.rsync += self.distDir + "_tbb"
         self.rsync += " "
         self.rsync += "&&"
         self.rsync += " "
         # Rsync command 3
         self.rsync += "rsync -a"
         self.rsync += " "
+        self.rsync += "--delete"
+        self.rsync += " "
         self.rsync += "--exclude='*alpha*'"
         self.rsync += " "
         if not silent:
@@ -181,7 +241,7 @@
             self.rsync += " "
         self.rsync += "rsync://%s/tor/dist/vidalia-bundles/" % mirror
         self.rsync += " "
-        self.rsync += self.distDir
+        self.rsync += self.distDir + "_vidalia"
         self.rsync += " "
 
     def syncWithMirror(self):
@@ -195,54 +255,3 @@
         """
         return self.rsync
 
-    def buildSplitFiles(self, pack, filename):
-        log.info("Building split files..")
-        packSplitDir = None
-        try:
-            splitpack = pack + ".split"
-            packSplitDir = os.path.join(self.packDir, splitpack)
-            if not os.access(packSplitDir, os.R_OK):
-                os.mkdir(packSplitDir)
-        except OSError, e:
-            log.error("Could not create dir %s: %s" \
-                            % (packSplitDir, e))
-        # Loop through split dir, look if every partXX.ZZZ has a 
-        # matching signature, pack them together in a .z
-        splitdir = os.path.join(self.distDir, filename)
-        for splitfile in os.listdir(splitdir):
-            # Skip signature files
-            if splitfile.endswith(".asc"):
-                continue
-            if re.compile(".*split.part.*").match(splitfile):
-                ascsplit = splitfile + ".asc"
-                ascfile = os.path.join(splitdir, "signatures", ascsplit)
-                # Rename .exe if needed
-                if gettor.utils.hasExe(ascfile):
-                    try:
-                        ascfile = gettor.utils.renameExe(ascfile)
-                    except:
-                        log.error("Could not rename exe file")
-                file = os.path.join(splitdir, splitfile)
-                if gettor.utils.hasExe(file):
-                    try:
-                        file = gettor.utils.renameExe(file)
-                    except:
-                        log.error("Could not rename exe file")
-                zsplitfile = splitfile + ".z"
-                zipFileName = os.path.join(packSplitDir, zsplitfile)
-                if gettor.utils.hasExe(zipFileName):
-                    try:
-                        zipFileName = gettor.utils.renameExe(zipFileName)
-                    except:
-                        log.error("Could not rename exe file")
-                if os.access(ascfile, os.R_OK) and os.access(file, os.R_OK):
-                    zip = zipfile.ZipFile(zipFileName, "w")
-                    zip.write(file, os.path.basename(file))
-                    zip.write(ascfile, os.path.basename(ascfile))
-                    zip.close()
-                else:
-                    log.error("Uhm, expected signature file for %s to be: %s" % (file, ascfile))
-                    return False
-
-        log.info("Done.")
-        return True

Modified: projects/gettor/lib/gettor/utils.py
===================================================================
--- projects/gettor/lib/gettor/utils.py	2010-02-13 19:09:00 UTC (rev 21633)
+++ projects/gettor/lib/gettor/utils.py	2010-02-14 00:40:06 UTC (rev 21634)
@@ -276,14 +276,15 @@
     else:
         return False
 
-def renameExe(filename):
+def renameExe(filename, renameFile=True):
     log.info("Renaming exe..")
-    if not os.access(filename, os.R_OK):
+    if renameFile and not os.access(filename, os.R_OK):
         log.error("Could not access file %s" % filename)
         raise OSError
 
     newfilename = filename.replace(".exe", ".ex_RENAME", 1)
-    os.rename(filename, newfilename)
+    if renameFile:
+        os.rename(filename, newfilename)
 
     return newfilename