[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[or-cvs] r21634: {projects} Remove old files from rsync dirs on sync (in projects/gettor: . lib/gettor)
Author: kaner
Date: 2010-02-14 00:40:06 +0000 (Sun, 14 Feb 2010)
New Revision: 21634
Modified:
projects/gettor/TODO
projects/gettor/lib/gettor/packages.py
projects/gettor/lib/gettor/utils.py
Log:
Remove old files from rsync dirs on sync
Modified: projects/gettor/TODO
===================================================================
--- projects/gettor/TODO 2010-02-13 19:09:00 UTC (rev 21633)
+++ projects/gettor/TODO 2010-02-14 00:40:06 UTC (rev 21634)
@@ -1,6 +1,5 @@
These are planned changes to the gettor system.
-- Clean distdir and packdir on each fetching/packaging run
- Split (at least) tiger bundle into several smaller archives to avoid ~20MB
- React in some way if the user requested a split download but no split
downloads are available for that specific package (maybe send out the non-
Modified: projects/gettor/lib/gettor/packages.py
===================================================================
--- projects/gettor/lib/gettor/packages.py 2010-02-13 19:09:00 UTC (rev 21633)
+++ projects/gettor/lib/gettor/packages.py 2010-02-14 00:40:06 UTC (rev 21634)
@@ -97,35 +97,91 @@
def buildPackages(self):
for (pack, (regex_single, regex_split)) in self.packageRegex.items():
- for filename in os.listdir(self.distDir):
- # Splitfile hacks. XXX: Refactor
- if re.compile(regex_split).match(filename):
- if not self.buildSplitFiles(pack, filename):
- log.error("Could not build split files packages")
- return False
- if re.compile(regex_single).match(filename):
- file = os.path.join(self.distDir, filename)
- ascfile = file + ".asc"
- zpack = pack + ".z"
- zipFileName = os.path.join(self.packDir, zpack)
- # If .asc file is there, build Zip file
- if os.access(ascfile, os.R_OK):
- zip = zipfile.ZipFile(zipFileName, "w")
- zip.write(file, os.path.basename(file))
- zip.write(ascfile, os.path.basename(ascfile))
- zip.close()
- self.packageList[pack] = zipFileName
- break
+ for dirname in os.listdir(self.distDir):
+ subdir = os.path.join(self.distDir, dirname)
+ for filename in os.listdir(subdir):
+ # Splitfile hacks. XXX: Refactor
+ if re.compile(regex_split).match(filename):
+ if not self.buildSplitFiles(pack, subdir, filename):
+ log.error("Could not build split files packages")
+ return False
+ if re.compile(regex_single).match(filename):
+ file = os.path.join(subdir, filename)
+ ascfile = file + ".asc"
+ zpack = pack + ".z"
+ zipFileName = os.path.join(self.packDir, zpack)
+ # If .asc file is there, build Zip file
+ if os.access(ascfile, os.R_OK):
+ zip = zipfile.ZipFile(zipFileName, "w")
+ zip.write(file, os.path.basename(file))
+ zip.write(ascfile, os.path.basename(ascfile))
+ zip.close()
+ self.packageList[pack] = zipFileName
+ break
if len(self.packageList) > 0:
return True
else:
log.error("Failed to build packages")
return False
+ def buildSplitFiles(self, pack, dirname, filename):
+ log.info("Building split files..")
+ packSplitDir = None
+ try:
+ splitpack = pack + ".split"
+ packSplitDir = os.path.join(self.packDir, splitpack)
+ if not os.access(packSplitDir, os.R_OK):
+ os.mkdir(packSplitDir)
+ except OSError, e:
+ log.error("Could not create dir %s: %s" \
+ % (packSplitDir, e))
+ # Loop through split dir, look if every partXX.ZZZ has a
+ # matching signature, pack them together in a .z
+ splitdir = os.path.join(dirname, filename)
+ for splitfile in os.listdir(splitdir):
+ # Skip signature files
+ if splitfile.endswith(".asc"):
+ continue
+ if re.compile(".*split.part.*").match(splitfile):
+ ascsplit = splitfile + ".asc"
+ ascfile = os.path.join(splitdir, "signatures", ascsplit)
+ # Rename .exe if needed
+ if gettor.utils.hasExe(ascfile):
+ try:
+ ascfile = gettor.utils.renameExe(ascfile)
+ except:
+ log.error("Could not rename exe file")
+ file = os.path.join(splitdir, splitfile)
+ if gettor.utils.hasExe(file):
+ try:
+ file = gettor.utils.renameExe(file)
+ except:
+ log.error("Could not rename exe file")
+ zsplitfile = splitfile + ".z"
+ zipFileName = os.path.join(packSplitDir, zsplitfile)
+ if gettor.utils.hasExe(zipFileName):
+ try:
+ zipFileName = gettor.utils.renameExe(zipFileName, False)
+ except:
+ log.error("Could not rename zip file exe")
+ return False
+ if os.access(ascfile, os.R_OK) and os.access(file, os.R_OK):
+ zip = zipfile.ZipFile(zipFileName, "w")
+ zip.write(file, os.path.basename(file))
+ zip.write(ascfile, os.path.basename(ascfile))
+ zip.close()
+ else:
+ log.error("Uhm, expected signature file for %s to be: %s" % (file, ascfile))
+
+ log.info("Done.")
+ return True
+
def initRsync(self, mirror="rsync.torproject.org", silent=False):
# Rsync command 1
self.rsync = "rsync -a"
self.rsync += " "
+ self.rsync += "--delete"
+ self.rsync += " "
self.rsync += "--exclude='*current*'"
self.rsync += " "
self.rsync += "--exclude='*osx*'"
@@ -143,13 +199,15 @@
self.rsync += " "
self.rsync += "rsync://%s/tor/dist/" % mirror
self.rsync += " "
- self.rsync += self.distDir
+ self.rsync += self.distDir + "_source"
self.rsync += " "
self.rsync += "&&"
self.rsync += " "
# Rsync command 2
self.rsync += "rsync -a"
self.rsync += " "
+ self.rsync += "--delete"
+ self.rsync += " "
self.rsync += "--exclude='*current*'"
self.rsync += " "
self.rsync += "--exclude='*osx*'"
@@ -167,13 +225,15 @@
self.rsync += " "
self.rsync += "rsync://%s/tor/torbrowser/dist/" % mirror
self.rsync += " "
- self.rsync += self.distDir
+ self.rsync += self.distDir + "_tbb"
self.rsync += " "
self.rsync += "&&"
self.rsync += " "
# Rsync command 3
self.rsync += "rsync -a"
self.rsync += " "
+ self.rsync += "--delete"
+ self.rsync += " "
self.rsync += "--exclude='*alpha*'"
self.rsync += " "
if not silent:
@@ -181,7 +241,7 @@
self.rsync += " "
self.rsync += "rsync://%s/tor/dist/vidalia-bundles/" % mirror
self.rsync += " "
- self.rsync += self.distDir
+ self.rsync += self.distDir + "_vidalia"
self.rsync += " "
def syncWithMirror(self):
@@ -195,54 +255,3 @@
"""
return self.rsync
- def buildSplitFiles(self, pack, filename):
- log.info("Building split files..")
- packSplitDir = None
- try:
- splitpack = pack + ".split"
- packSplitDir = os.path.join(self.packDir, splitpack)
- if not os.access(packSplitDir, os.R_OK):
- os.mkdir(packSplitDir)
- except OSError, e:
- log.error("Could not create dir %s: %s" \
- % (packSplitDir, e))
- # Loop through split dir, look if every partXX.ZZZ has a
- # matching signature, pack them together in a .z
- splitdir = os.path.join(self.distDir, filename)
- for splitfile in os.listdir(splitdir):
- # Skip signature files
- if splitfile.endswith(".asc"):
- continue
- if re.compile(".*split.part.*").match(splitfile):
- ascsplit = splitfile + ".asc"
- ascfile = os.path.join(splitdir, "signatures", ascsplit)
- # Rename .exe if needed
- if gettor.utils.hasExe(ascfile):
- try:
- ascfile = gettor.utils.renameExe(ascfile)
- except:
- log.error("Could not rename exe file")
- file = os.path.join(splitdir, splitfile)
- if gettor.utils.hasExe(file):
- try:
- file = gettor.utils.renameExe(file)
- except:
- log.error("Could not rename exe file")
- zsplitfile = splitfile + ".z"
- zipFileName = os.path.join(packSplitDir, zsplitfile)
- if gettor.utils.hasExe(zipFileName):
- try:
- zipFileName = gettor.utils.renameExe(zipFileName)
- except:
- log.error("Could not rename exe file")
- if os.access(ascfile, os.R_OK) and os.access(file, os.R_OK):
- zip = zipfile.ZipFile(zipFileName, "w")
- zip.write(file, os.path.basename(file))
- zip.write(ascfile, os.path.basename(ascfile))
- zip.close()
- else:
- log.error("Uhm, expected signature file for %s to be: %s" % (file, ascfile))
- return False
-
- log.info("Done.")
- return True
Modified: projects/gettor/lib/gettor/utils.py
===================================================================
--- projects/gettor/lib/gettor/utils.py 2010-02-13 19:09:00 UTC (rev 21633)
+++ projects/gettor/lib/gettor/utils.py 2010-02-14 00:40:06 UTC (rev 21634)
@@ -276,14 +276,15 @@
else:
return False
-def renameExe(filename):
+def renameExe(filename, renameFile=True):
log.info("Renaming exe..")
- if not os.access(filename, os.R_OK):
+ if renameFile and not os.access(filename, os.R_OK):
log.error("Could not access file %s" % filename)
raise OSError
newfilename = filename.replace(".exe", ".ex_RENAME", 1)
- os.rename(filename, newfilename)
+ if renameFile:
+ os.rename(filename, newfilename)
return newfilename