[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]

[or-cvs] r17294: {updater} attach per-file type-specific metainfo to packages. use meta (in updater/trunk/lib/thandy: . packagesys)



Author: nickm
Date: 2008-11-16 17:27:17 -0500 (Sun, 16 Nov 2008)
New Revision: 17294

Modified:
   updater/trunk/lib/thandy/ClientCLI.py
   updater/trunk/lib/thandy/formats.py
   updater/trunk/lib/thandy/packagesys/ExePackages.py
   updater/trunk/lib/thandy/packagesys/PackageDB.py
   updater/trunk/lib/thandy/packagesys/PackageSystem.py
   updater/trunk/lib/thandy/packagesys/RPMPackages.py
   updater/trunk/lib/thandy/repository.py
Log:
attach per-file type-specific metainfo to packages.  use metainfo to tell what is already installed.  use database to track things we have installed with not attacked version.  notice when we have installable files and optionally install them.

Modified: updater/trunk/lib/thandy/ClientCLI.py
===================================================================
--- updater/trunk/lib/thandy/ClientCLI.py	2008-11-16 21:10:45 UTC (rev 17293)
+++ updater/trunk/lib/thandy/ClientCLI.py	2008-11-16 22:27:17 UTC (rev 17294)
@@ -4,32 +4,78 @@
 import logging
 import os
 import sys
+import time
 
+import thandy.formats
 import thandy.util
 import thandy.repository
 import thandy.download
 import thandy.master_keys
+import thandy.packagesys.PackageSystem
 
 def update(args):
     repoRoot = thandy.util.userFilename("cache")
-    options, args = getopt.getopt(args, "", [ "repo=", "no-download" ])
+    options, args = getopt.getopt(args, "", [ "repo=", "no-download",
+                                              "loop", "no-packagesys",
+                                              "install"])
     download = True
+    keep_looping = False
+    use_packagesys = True
+    install = False
 
     for o, v in options:
         if o == '--repo':
             repoRoot = v
         elif o == "--no-download":
             download = False
+        elif o == '--loop':
+            keep_looping = True
+        elif o == '--no-packagesys':
+            use_packagesys = False
+        elif o == '--install':
+            install = True
 
     repo = thandy.repository.LocalRepository(repoRoot)
+    packagesys = None
+    if use_packagesys:
+        packagesys = thandy.packagesys.PackageSystem.PackageMetasystem.create(repo)
 
     while True:
         hashes = {}
+        installable = {}
         logging.info("Checking for files to update.")
-        files = repo.getFilesToUpdate(trackingBundles=args, hashDict=hashes)
+        files = repo.getFilesToUpdate(trackingBundles=args, hashDict=hashes,
+                                      pkgSystems=packagesys,
+                                      installableDict=installable)
+
+        if installable and not files:
+            logging.notice("Ready to install files: %s",
+                           ", ".join(sorted(installable.keys())))
+            if install:
+                # XXXX handle ordering
+                for h in installable.values():
+                    h.install()
+            return
+
+        elif not files:
+            logging.info("No files to download")
+            if not keep_looping:
+                return
+
+            ts = repo.getTimestampFile().get()
+            age = time.time() - thandy.formats.parseTime(ts['at'])
+            delay = thandy.repository.MAX_TIMESTAMP_AGE - age
+            if delay > 3600:
+                delay = 3600
+            elif delay < 0:
+                delay = 300
+            logging.info("Will check again in %s seconds", delay)
+            time.sleep(delay)
+            continue
+
         logging.info("Files to download are: %s", ", ".join(sorted(files)))
 
-        if not download or not files:
+        if not download:
             return
 
         mirrorlist = repo.getMirrorlistFile().get()
@@ -59,18 +105,13 @@
         logging.info("All downloads finished.")
 
 
-# Check my repository
-
-# Tell me what I need to download
-
-# Download stuff
-
 # Tell me what to install.
 
 
 def usage():
     print "Known commands:"
-    print "  update [--repo=repository] [--no-download]"
+    print "  update [--repo=repository] [--no-download] [--loop]"
+    print "         [--no-packagesys] [--install]"
     sys.exit(1)
 
 def main():
@@ -81,7 +122,7 @@
         usage()
     cmd = sys.argv[1]
     args = sys.argv[2:]
-    if cmd in [ "update", "geturls" ]:
+    if cmd in [ "update" ]:
         globals()[cmd](args)
     else:
         usage()

Modified: updater/trunk/lib/thandy/formats.py
===================================================================
--- updater/trunk/lib/thandy/formats.py	2008-11-16 21:10:45 UTC (rev 17293)
+++ updater/trunk/lib/thandy/formats.py	2008-11-16 22:27:17 UTC (rev 17294)
@@ -451,7 +451,8 @@
             version=VERSION_SCHEMA,
             format=S.Obj(),
             ts=TIME_SCHEMA,
-            files=S.ListOf(S.Struct([RELPATH_SCHEMA, HASH_SCHEMA])),
+            files=S.ListOf(S.Struct([RELPATH_SCHEMA, HASH_SCHEMA],
+                                    allowMore=True)),
             shortdesc=S.DictOf(S.AnyStr(), S.AnyStr()),
             longdesc=S.DictOf(S.AnyStr(), S.AnyStr()))
 
@@ -612,23 +613,29 @@
                         'format',
                         'location',
                         'relpath',
-                        ], (), preload)
+                        ], ['rpm_version', 'exe_args'], preload)
 
     f = open(package_fname, 'rb')
     digest = getFileDigest(f)
 
     # Check fields!
+    extra = {}
     result = { '_type' : "Package",
                'ts' : formatTime(time.time()),
                'name' : r['name'],
                'location' : r['location'], #DOCDOC
                'version' : r['version'],
                'format' : r['format'],
-               'files' : [ [ r['relpath'], formatHash(digest) ] ],
+               'files' : [ [ r['relpath'], formatHash(digest), extra ] ],
                'shortdesc' : shortDescs,
                'longdesc' : longDescs
              }
 
+    if format == 'rpm' and r.get('rpm_version'):
+        extra['rpm_version'] = r['rpm_version']
+    elif format == 'exe' and r.get('exe_args') != None:
+        extra['exe_args'] = r['exe_args']
+
     PACKAGE_SCHEMA.checkMatch(result)
 
     return result

Modified: updater/trunk/lib/thandy/packagesys/ExePackages.py
===================================================================
--- updater/trunk/lib/thandy/packagesys/ExePackages.py	2008-11-16 21:10:45 UTC (rev 17293)
+++ updater/trunk/lib/thandy/packagesys/ExePackages.py	2008-11-16 22:27:17 UTC (rev 17294)
@@ -1,16 +1,39 @@
 # Copyright 2008 The Tor Project, Inc.  See LICENSE for licensing information.
 
+import subprocess
+
+import thandy.util
 import thandy.packagesys.PackageSystem as ps
 import thandy.packagesys.PackageDB as pdb
 
 class ExePackageSystem(pdb.DBBackedPackageSystem):
+    def __init__(self, repo):
+        self._repo = repo
 
     def getName(self):
-        return "executable"
+        return "exe"
 
-    def packageHandleFromJSON(self, json):
-        raise NotImplemented()  #XXXX????
+    def packageHandlesFromJSON(self, pkg):
+        if pkg['format'] != 'exe':
+            raise thandy.FormatException()
 
+        handles = []
+        for entry in pkg['files']:
+            if len(entry) < 3:
+                continue
+            rp, h, extra = entry[:3]
+            version = package['version']
+
+            handles.append(
+                ExePackageHandle(self.getDB(),
+                                 package['name'],
+                                 version,
+                                 [],  # filelist not implemented in this.
+                                 rp,
+                                 self._repo.getFilename(rp),
+                                 extra['exe_args']))
+        return handles
+
     def canBeAutomatic(self):
         return True
 
@@ -18,12 +41,16 @@
         return True
 
 class ExePackageHandle(pdb.DBBackedPackageHandle):
-    def __init__(self, packageDB, name, version, filelist, filename,
+    def __init__(self, packageDB, name, version, filelist, relpath, filename,
                  arguments):
         pdb.DBBackedPackageHandle.__init__(packageDB, name, version, filelist)
+        self._relPath = relpath
         self._filename = filename
         self._arguments = arguments
 
+    def getRelativePath(self):
+        return self._relPath
+
     def _doInstall(self):
         commandline = [ self._filename ] + self._arguments
         logging.info("Installing %s.  Command line: %s", self._filename,

Modified: updater/trunk/lib/thandy/packagesys/PackageDB.py
===================================================================
--- updater/trunk/lib/thandy/packagesys/PackageDB.py	2008-11-16 21:10:45 UTC (rev 17293)
+++ updater/trunk/lib/thandy/packagesys/PackageDB.py	2008-11-16 22:27:17 UTC (rev 17294)
@@ -1,34 +1,48 @@
 # Copyright 2008 The Tor Project, Inc.  See LICENSE for licensing information.
 
 import anydbm
+import atexit
 import shelve
 
 import thandy.util
 import thandy.formats
 
+import thandy.packagesys.PackageSystem
+
 class SimplePackageDB:
-
     def __init__(self, filename):
+        thandy.util.ensureParent(filename)
         self._db = anydbm.open(filename, 'c')
+        atexit.register(self.close)
 
+    def close(self):
+        self._db.close()
+
     def setVersion(self, package, version, filelist):
-        pass
+        self._db['pv_%s'%package] = (version, filelist)
 
     def setInstallParameters(self, package, params):
-        pass
+        self._db['ip_%s'%package] = params
 
     def getCurVersion(self, package):
-        pass
+        v = self._db.get('pv_%s'%package)
+        if v != None:
+            return v[0]
 
     def getInstallParameters(self, package):
-        pass
+        return self._db.get('pi_%s'%package)
 
+class DBBackedPackageSystem(thandy.packagesys.PackageSystem.PackageSystem):
+    def __init__(self):
+        self._packageDB = None
 
-class DBBackedPackageSystem(thandy.packagesys.PackageSystem):
-    def __init__(self, packageDB):
-        self._packageDB = packageDB
+    def getDB(self):
+        if self._packageDB is None:
+            fname = thandy.util.userFilename("db/packages")
+            self._packageDB = pdb.PackageDB(fname)
+        return self._packageDB
 
-class DBBackedPackageHandle(thandy.packagesys.PackageHandle):
+class DBBackedPackageHandle(thandy.packagesys.PackageSystem.PackageHandle):
     def __init__(self, packageDB, name, version, filelist):
         thandy.packagesys.PackageSystem.PackageHandle.__init__(self)
         self._packageDB = packageDB
@@ -68,14 +82,13 @@
             if not os.path.exists(fn):
                 all_ok = False
             else:
-                f = open(fn, 'rb')
                 try:
-                    try:
-                        d = thandy.formats.getFileDigest(f)
-                    except OSError:
+                    d = thandy.formats.getFileDigest(fn)
+                    if d != hash:
                         all_ok = False
-                        break
-                finally:
-                    f.close()
+                except OSError:
+                    all_ok = False
+                    break
 
+
         return all_ok

Modified: updater/trunk/lib/thandy/packagesys/PackageSystem.py
===================================================================
--- updater/trunk/lib/thandy/packagesys/PackageSystem.py	2008-11-16 21:10:45 UTC (rev 17293)
+++ updater/trunk/lib/thandy/packagesys/PackageSystem.py	2008-11-16 22:27:17 UTC (rev 17294)
@@ -1,10 +1,40 @@
 # Copyright 2008 The Tor Project, Inc.  See LICENSE for licensing information.
 
+class PackageMetasystem:
+    def __init__(self, repository):
+        self._repostitory = repository
+        self._systems = {}
+
+    def addPackageSystem(self, system):
+        self._systems[system.getName()] = system
+
+    def getSysForPackage(self, pkg):
+        return self._systems.get(pkg['format'], None)
+
+    @staticmethod
+    def create(repository):
+        r = PackageMetasystem(repository)
+
+        try:
+            import rpm
+        except ImportError:
+            pass
+        else:
+            import thandy.packagesys.RPMPackages
+            r.addPackageSystem(thandy.packagesys.RPMPackages.RPMPackageSystem(
+                    repository))
+
+        import thandy.packagesys.ExePackages
+        r.addPackageSystem(thandy.packagesys.ExePackages.ExePackageSystem(
+                repository))
+
+        return r
+
 class PackageSystem:
     def getName(self):
         raise NotImplemented()
 
-    def packageHandleFromJSON(self, json):
+    def packageHandlesFromJSON(self, json):
         raise NotImplemented()
 
     def canBeAutomatic(self):
@@ -39,6 +69,9 @@
         self._transactions.append(packageHandle.remove)
 
 class PackageHandle:
+    def getRelativePath(self):
+        raise NotImplemented()
+
     def isInstalled(self, transaction=None):
         raise NotImplemented()
 

Modified: updater/trunk/lib/thandy/packagesys/RPMPackages.py
===================================================================
--- updater/trunk/lib/thandy/packagesys/RPMPackages.py	2008-11-16 21:10:45 UTC (rev 17293)
+++ updater/trunk/lib/thandy/packagesys/RPMPackages.py	2008-11-16 22:27:17 UTC (rev 17294)
@@ -6,15 +6,40 @@
 import rpm
 import md5
 
+import thandy.formats
+
 __all__ = [ 'RPMPackageSystem' ]
 
 class RPMPackageSystem(thandy.packagesys.PackageSystem.PackageSystem):
+    def __init__(self, repo):
+        self._repo = repo
+
     def getName(self):
-        return "RPM"
+        return "rpm"
 
-    def packageHandleFromJSON(self, json):
-        raise NotImplemented() # XXXX
+    def packageHandlesFromJSON(self, package):
+        if package['format'] != 'rpm':
+            raise thandy.FormatException()
 
+        handles = []
+        for entry in package['files']:
+            if len(entry) < 3:
+                continue
+            fn, h, extra = entry[:3]
+            name = os.path.split(fn)[1]
+
+            try:
+                version = extra['rpm_version']
+            except KeyError:
+                raise thandy.FormatException()
+
+            handles.append(RPMPackageHandle(name,
+                                            version,
+                                            fn,
+                                            self._repo.getFilename(fn)))
+
+        return handles
+
     def getTransaction(self):
         return RPMPackageTransaction()
 
@@ -115,7 +140,7 @@
                     logging.info("%s is missing or unreadable from %s %s; "
                                  "that's ok.", fname, name, h['version'])
                 else:
-                    logging.warn("%s is missing or unreadable from %s %s."
+                    logging.warn("%s is missing or unreadable from %s %s.",
                                  fname, name, h['version'])
                     all_ok = False
             elif haveMD5 == md5sum:
@@ -131,11 +156,15 @@
     return found and all_ok
 
 class RPMPackageHandle(thandy.packagesys.PackageSystem.PackageHandle):
-    def __init__(self, name, version, filename):
+    def __init__(self, name, version, relativePath, filename):
         self._name = name
         self._version = version
+        self._relPath = relativePath
         self._filename = filename
 
+    def getRelativePath(self):
+        return self._relPath
+
     def anyVersionInstalled(self, transaction=None):
         return len(getInstalledRPMVersions(self.name, transaction)) > 1
 

Modified: updater/trunk/lib/thandy/repository.py
===================================================================
--- updater/trunk/lib/thandy/repository.py	2008-11-16 21:10:45 UTC (rev 17293)
+++ updater/trunk/lib/thandy/repository.py	2008-11-16 22:27:17 UTC (rev 17294)
@@ -260,7 +260,7 @@
                 needRole='bundle')
             return pkg
 
-    def getRequestedFile(self, relPath):
+    def getRequestedFile(self, relPath, pkgSystems=None):
         """DOCDOC"""
         for f in self._metafiles:
             if f.getRelativePath() == relPath:
@@ -279,12 +279,14 @@
 
         return None
 
-    def getFilesToUpdate(self, now=None, trackingBundles=(), hashDict=None):
+    def getFilesToUpdate(self, now=None, trackingBundles=(), hashDict=None,
+                         pkgSystems=None, installableDict=None):
         """Return a set of relative paths for all files that we need
            to fetch.  Assumes that we care about the bundles
            'trackingBundles'.  If hashDict is provided, add mappings to it
            from the relative paths we want to fecth to the hashes that we
            want those items to have, when we know those hashes.
+           DOCDOC pkgSystems, installableDict
         """
 
         if now == None:
@@ -294,6 +296,9 @@
             # Use a dummy hashdict.
             hashDict = {}
 
+        if installableDict == None:
+            installableDict = {}
+
         need = set()
 
         # Fetch missing metafiles.
@@ -444,8 +449,29 @@
         # files?
         for pfile in packages.values():
             package = pfile.get()
+
+            alreadyInstalled = {}
+            allHandles = {}
+            if pkgSystems is not None:
+                psys = pkgSystems.getSysForPackage(package)
+                if psys is None:
+                    logging.info("No way to check whether a %s package is "
+                                 "up-to-date." % package['format'])
+                else:
+                    handles = psys.packageHandlesFromJSon(package)
+
+                    for h in handles:
+                        allHandles[h.getRelativePath()] = h
+                        if h.isInstalled():
+                            alreadyInstalled[h.getRelativePath()] = h
+
             for f in package['files']:
                 rp, h = f[:2]
+                if alreadyInstalled.has_key(rp):
+                    logging.info("%s is already installed; no need to download",
+                                 rp)
+                    continue
+
                 h_expected = thandy.formats.parseHash(h)
                 hashDict[rp] = h_expected
                 fn = self.getFilename(rp)
@@ -459,6 +485,9 @@
                 if h_got != h_expected:
                     logging.info("Hash for %s not as expected; must load.", rp)
                     need.add(rp)
+                else:
+                    if allHandles.has_key(rp):
+                        installableDict[rp] = allHandles[rp]
 
         # Okay; these are the files we need.
         return need