[Author Prev][Author Next][Thread Prev][Thread Next][Author Index][Thread Index]
[or-cvs] r16928: {updater} More repository/signing/marshalling/crypto work for glider. (in updater/trunk: . lib/glider lib/sexp)
Author: nickm
Date: 2008-09-19 15:22:43 -0400 (Fri, 19 Sep 2008)
New Revision: 16928
Added:
updater/trunk/Makefile
updater/trunk/lib/glider/repository.py
Modified:
updater/trunk/TODO
updater/trunk/lib/glider/formats.py
updater/trunk/lib/glider/keys.py
updater/trunk/lib/glider/tests.py
updater/trunk/lib/sexp/access.py
updater/trunk/lib/sexp/tests.py
Log:
More repository/signing/marshalling/crypto work for glider.
Added: updater/trunk/Makefile
===================================================================
--- updater/trunk/Makefile (rev 0)
+++ updater/trunk/Makefile 2008-09-19 19:22:43 UTC (rev 16928)
@@ -0,0 +1,6 @@
+
+export PYTHONPATH=./lib
+
+test:
+ python -m sexp.tests
+ python -m glider.tests
Modified: updater/trunk/TODO
===================================================================
--- updater/trunk/TODO 2008-09-19 13:14:27 UTC (rev 16927)
+++ updater/trunk/TODO 2008-09-19 19:22:43 UTC (rev 16928)
@@ -5,9 +5,11 @@
. Write server-side code (python)
o S-expression lib
. Code to manage data formats
+ . Parse
+ o Validate
- Code to wrangle private keys
- - Generate
- - Store, load (password-protected)
+ o Generate
+ . Store, load (password-protected)
- Print for posterity
- Code to generate timestamp files
Modified: updater/trunk/lib/glider/formats.py
===================================================================
--- updater/trunk/lib/glider/formats.py 2008-09-19 13:14:27 UTC (rev 16927)
+++ updater/trunk/lib/glider/formats.py 2008-09-19 19:22:43 UTC (rev 16928)
@@ -48,6 +48,13 @@
unknownSigs = []
tangentialSigs = []
+ assert signed[0] == "signed"
+ data = signed[1]
+
+ d_obj = Crypto.Hash.SHA256.new()
+ sexp.encode.hash_canonical(data, d_obj)
+ digest = d_obj.digest()
+
for signature in sexp.access.s_children(signed, "signature"):
attrs = signature[1]
sig = attrs[2]
@@ -59,7 +66,7 @@
continue
method = s_child(attrs, "method")[1]
try:
- result = key.checkSignature(method, data, sig)
+ result = key.checkSignature(method, sig, digest=digest)
except UnknownMethod:
continue
if result == True:
@@ -75,6 +82,8 @@
else:
badSigs.append(keyid)
+ return goodSigs, badSigs, unknownSigs, tangentialSigs
+
def sign(signed, key):
assert sexp.access.s_tag(signed) == 'signed'
s = signed[1]
@@ -129,7 +138,7 @@
SIGNED_SCHEMA = _parseSchema(SIGNED_TEMPLATE, SCHEMA_TABLE)
-KEYFILE_TEMPLATE = r"""
+KEYLIST_TEMPLATE = r"""
(=keylist
(=ts .TIME)
(=keys
@@ -139,7 +148,7 @@
*
)"""
-KEYFILE_SCHEMA = _parseSchema(KEYFILE_TEMPLATE, SCHEMA_TABLE)
+KEYLIST_SCHEMA = _parseSchema(KEYLIST_TEMPLATE, SCHEMA_TABLE)
MIRRORLIST_TEMPLATE = r"""
(=mirrorlist
@@ -194,3 +203,74 @@
"""
PACKAGE_SCHEMA = _parseSchema(PACKAGE_TEMPLATE, SCHEMA_TABLE)
+
+ALL_ROLES = ('timestamp', 'mirrors', 'bundle', 'package', 'master')
+
+class Key:
+ def __init__(self, key, roles):
+ self.key = key
+ self.roles = []
+ for r,p in roles:
+ self.addRole(r,p)
+
+ def addRole(self, role, path):
+ assert role in ALL_ROLES
+ self.roles.append(role, path)
+
+ def getRoles(self):
+ return self.rules
+
+ @staticmethod
+ def fromSExpression(sexpr):
+ # must match PUBKEY_SCHEMA
+ typeattr = sexp.access.s_attr(sexpr[1], "type")
+ if typeattr == 'rsa':
+ key = glider.keys.RSAKey.fromSExpression(sexpr)
+ if key is not None:
+ return Key(key)
+ else:
+ return None
+
+ def format(self):
+ return self.key.format()
+
+ def getKeyID(self):
+ return self.key.getKeyID()
+
+ def sign(self, sexpr=None, digest=None):
+ return self.key.sign(sexpr, digest=digest)
+
+ def checkSignature(self, method, sexpr=None, digest=None):
+ if digest == None:
+ _, digest = self.key._digest(sexpr, method)
+ ok = self.key.checkSignature(method, digest=digest)
+ # XXXX CACHE HERE.
+ return ok
+
+class Keystore(KeyDB):
+ def __init__(self):
+ KeyDB.__init__(self)
+
+ @staticmethod
+ def addFromKeylist(sexpr, allowMasterKeys=False):
+ # Don't do this until we have validated the structure.
+ for ks in sexpr.access.s_lookup_all("keys.key"):
+ attrs = ks[1]
+ key_s = ks[2]
+ roles = s_attr(attrs, "roles")
+ #XXXX Use interface of Key, not RSAKey.
+ key = Key.fromSExpression(key_s)
+ if not key:
+ #LOG skipping key.
+ continue
+ for r,p in roles:
+ if r == 'master' and not allowMasterKeys:
+ #LOG
+ continue
+ if r not in ALL_ROLES:
+ continue
+ key.addRole(r,p)
+
+ self.addKey(key)
+
+
Modified: updater/trunk/lib/glider/keys.py
===================================================================
--- updater/trunk/lib/glider/keys.py 2008-09-19 13:14:27 UTC (rev 16927)
+++ updater/trunk/lib/glider/keys.py 2008-09-19 19:22:43 UTC (rev 16928)
@@ -2,13 +2,16 @@
# These require PyCrypto.
import Crypto.PublicKey.RSA
import Crypto.Hash.SHA256
+import Crypto.Cipher.AES
import sexp.access
import sexp.encode
import sexp.parse
+import cPickle as pickle
import binascii
import os
+import struct
class CryptoError(Exception):
pass
@@ -134,26 +137,147 @@
self.keyid = ("rsa", d_obj.digest())
return self.keyid
+ def _digest(self, sexpr, method=None):
+ if method in (None, "sha256-pkcs1"):
+ d_obj = Crypto.Hash.SHA256.new()
+ sexp.encode.hash_canonical(sexpr, d_obj)
+ digest = d_obj.digest()
+ return ("sha256-pkcs1", digest)
+
+ raise UnknownMethod(method)
+
def sign(self, sexpr=None, digest=None):
assert _xor(sexpr == None, digest == None)
if digest == None:
- d_obj = Crypto.Hash.SHA256.new()
- sexp.encode.hash_canonical(sexpr, d_obj)
- digest = d_obj.digest()
+ method, digest = self._digest(sexpr)
m = _pkcs1_padding(digest, (self.key.size()+1) // 8)
sig = intToBinary(self.key.sign(m, "")[0])
- return ("sha256-pkcs1", sig)
+ return (method, sig)
def checkSignature(self, method, sig, sexpr=None, digest=None):
assert _xor(sexpr == None, digest == None)
if method != "sha256-pkcs1":
raise UnknownMethod("method")
if digest == None:
- d_obj = Crypto.Hash.SHA256.new()
- sexp.encode.hash_canonical(sexpr, d_obj)
- digest = d_obj.digest()
+ method, digest = self._digest(sexpr, method)
sig = binaryToInt(sig)
m = _pkcs1_padding(digest, (self.key.size()+1) // 8)
return self.key.verify(m, (sig,))
+SALTLEN=16
+def secretToKey(salt, secret):
+ """Convert 'secret' to a 32-byte key, using a version of the algorithm
+ from RFC2440. The salt must be SALTLEN+1 bytes long, and should
+ be random, except for the last byte, which encodes how time-
+ consuming the computation should be.
+
+ (The goal is to make offline password-guessing attacks harder by
+ increasing the time required to convert a password to a key, and to
+ make precomputed password tables impossible to generate by )
+ """
+ assert len(salt) == SALTLEN+1
+
+ # The algorithm is basically, 'call the last byte of the salt the
+ # "difficulty", and all other bytes of the salt S. Now make
+ # an infinite stream of S|secret|S|secret|..., and hash the
+ # first N bytes of that, where N is determined by the difficulty.
+ #
+ # Obviously, this wants a hash algorithm that's tricky to
+ # parallelize.
+ #
+ # Unlike RFC2440, we use a 16-byte salt. Because CPU times
+ # have improved, we start at 16 times the previous minimum.
+
+ difficulty = ord(salt[-1])
+ count = (16L+(difficulty & 15)) << ((difficulty >> 4) + 10)
+
+ # Make 'data' nice and long, so that we don't need to call update()
+ # a zillion times.
+ data = salt[:-1]+secret
+ if len(data)<1024:
+ data *= (1024 // len(data))+1
+
+ d = Crypto.Hash.SHA256.new()
+ iters, leftover = divmod(count, len(data))
+ for _ in xrange(iters):
+ d.update(data)
+ #count -= len(data)
+ if leftover:
+ d.update(data[:leftover])
+ #count -= leftover
+ #assert count == 0
+
+ return d.digest()
+
+def encryptSecret(secret, password, difficulty=0x80):
+ """Encrypt the secret 'secret' using the password 'password',
+ and return the encrypted result."""
+ # The encrypted format is:
+ # "GKEY1" -- 5 octets, fixed, denotes data format.
+ # SALT -- 17 bytes, used to hash password
+ # IV -- 16 bytes; salt for encryption
+ # ENCRYPTED IN AES256-OFB, using a key=s2k(password, salt) and IV=IV:
+ # SLEN -- 4 bytes; length of secret, big-endian.
+ # SECRET -- len(secret) bytes
+ # D -- 32 bytes; SHA256 hash of (salt|secret|salt).
+ #
+ # This format leaks the secret length, obviously.
+ assert 0 <= difficulty < 256
+ salt = os.urandom(SALTLEN)+chr(difficulty)
+ key = secretToKey(salt, password)
+
+ d_obj = Crypto.Hash.SHA256.new()
+ d_obj.update(salt)
+ d_obj.update(secret)
+ d_obj.update(salt)
+ d = d_obj.digest()
+
+ iv = os.urandom(16)
+ e = Crypto.Cipher.AES.new(key, Crypto.Cipher.AES.MODE_OFB, iv)
+
+ # Stupidly, pycrypto doesn't accept that stream ciphers don't need to
+ # take their input in blocks. So pad it, then ignore the padded output.
+
+ padlen = 16-((len(secret)+len(d)+4) % 16)
+ if padlen == 16: padlen = 0
+ pad = '\x00' * padlen
+
+ slen = struct.pack("!L",len(secret))
+ encrypted = e.encrypt("%s%s%s%s" % (slen, secret, d, pad))[:-padlen]
+ return "GKEY1%s%s%s"%(salt, iv, encrypted)
+
+def decryptSecret(encrypted, password):
+ if encrypted[:5] != "GKEY1":
+ raise UnknownFormat()
+ encrypted = encrypted[5:]
+ if len(encrypted) < SALTLEN+1+16:
+ raise FormatError()
+
+ salt = encrypted[:SALTLEN+1]
+ iv = encrypted[SALTLEN+1:SALTLEN+1+16]
+ encrypted = encrypted[SALTLEN+1+16:]
+
+ key = secretToKey(salt, password)
+
+ e = Crypto.Cipher.AES.new(key, Crypto.Cipher.AES.MODE_OFB, iv)
+ padlen = 16-(len(encrypted) % 16)
+ if padlen == 16: padlen = 0
+ pad = '\x00' * padlen
+
+ decrypted = e.decrypt("%s%s"%(encrypted,pad))
+ slen = struct.unpack("!L", decrypted[:4])[0]
+ secret = decrypted[4:4+slen]
+ hash = decrypted[4+slen:4+slen+Crypto.Hash.SHA256.digest_size]
+
+ d = Crypto.Hash.SHA256.new()
+ d.update(salt)
+ d.update(secret)
+ d.update(salt)
+
+ if d.digest() != hash:
+ print repr(decrypted)
+ raise BadPassword()
+
+ return secret
+
Added: updater/trunk/lib/glider/repository.py
===================================================================
--- updater/trunk/lib/glider/repository.py (rev 0)
+++ updater/trunk/lib/glider/repository.py 2008-09-19 19:22:43 UTC (rev 16928)
@@ -0,0 +1,121 @@
+
+import sexp.parse
+import sexp.access
+import glider.formats
+
+import os
+import threading
+
+class RepositoryFile:
+ def __init__(self, repository, relativePath, schema,
+ needRole=None, signedFormat=True, needSigs=1):
+ self._repository = repository
+ self._relativePath = relativePath
+ self._schema = schema
+ self._needRole = needRole
+ self._signedFormat = signedFormat
+ self._needSigs = needSigs
+
+ self._signed_sexpr = None
+ self._main_sexpr = None
+ self._mtime = None
+
+ def getPath(self):
+ return os.path.join(self._repository._root, self._relativePath)
+
+ def _load(self):
+ fname = self.getPath()
+
+ # Propagate OSError
+ f = None
+ fd = os.open(fname, os.O_RDONLY)
+ try:
+ f = os.fdopen(fd, 'r')
+ except:
+ os.close(fd)
+ raise
+ try:
+ mtime = os.fstat(fd).st_mtime
+ content = f.read()
+ finally:
+ f.close()
+
+ signed_sexpr,main_sexpr = self._checkContent(content)
+
+ self._signed_sexpr = signed_sexpr
+ self._main_sexpr = main_sexpr
+ self._mtime = mtime
+
+ def _save(self, content=None):
+ if content == None:
+ content = sexpr.encode
+
+ signed_sexpr,main_sexpr = self._checkContent(content)
+
+ fname = self.getPath()
+ fname_tmp = fname+"_tmp"
+
+ fd = os.open(fname_tmp, os.WRONLY|os.O_CREAT|os.O_TRUNC, 0644)
+ try:
+ os.write(fd, contents)
+ finally:
+ os.close(fd)
+ if sys.platform in ('cygwin', 'win32'):
+ # Win32 doesn't let rename replace an existing file.
+ try:
+ os.unlink(fname)
+ except OSError:
+ pass
+ os.rename(fname_tmp, fname)
+
+ self._signed_sexpr = signed_sexpr
+ self._main_sexpr = main_sexpr
+ self._mtime = mtime
+
+ def _checkContent(self, content):
+ sexpr = sexp.parse.parse(content)
+ if not sexpr:
+ raise ParseError()
+
+ if self._signedFormat:
+ if not glider.formats.SIGNED_SCHEMA.matches(sexpr):
+ raise FormatError()
+
+ sigs = checkSignatures(sexpr, self._repository._keyDB,
+ self._needRole, self._relativePath)
+ good = sigs[0]
+ # XXXX If good is too low but unknown is high, we may need
+ # a new key file.
+ if len(good) < 1:
+ raise SignatureError()
+
+ main_sexpr = sexpr[1]
+ signed_sexpr = sexpr
+ else:
+ signed_sexpr = None
+ main_sexpr = sexpr
+
+ if self._schema != None and not self._schema.matches(main_sexpr):
+ raise FormatError()
+
+ return signed_sexpr, main_sexpr
+
+ def load(self):
+ if self._main_sexpr == None:
+ self._load()
+
+class LocalRepository:
+ def __init__(self, root):
+ self._root = root
+ self._keyDB = None
+
+ self._keylistFile = RepositoryFile(
+ self, "meta/keys.txt", glider.formats.KEYLIST_SCHEMA,
+ needRole="master")
+ self._timestampFile = RepositoryFile(
+ self, "meta/timestamp.txt", glider.formats.TIMESTAMP_SCHEMA,
+ needRole="timestamp")
+ self._mirrorlistFile = RepositoryFile(
+ self, "meta/mirrors.txt", glider.formats.MIRRORLIST_SCHEMA,
+ needRole="mirrors")
+
Modified: updater/trunk/lib/glider/tests.py
===================================================================
--- updater/trunk/lib/glider/tests.py 2008-09-19 13:14:27 UTC (rev 16927)
+++ updater/trunk/lib/glider/tests.py 2008-09-19 19:22:43 UTC (rev 16928)
@@ -4,14 +4,14 @@
import glider.keys
import glider.formats
+import glider.repository
+
import glider.tests
-class EncodingTest(unittest.TestCase):
- def testQuotedString(self):
- self.assertEquals(1,1)
+class EncryptionTest(unittest.TestCase):
+ pass
def suite():
- import sexp.tests
suite = unittest.TestSuite()
suite.addTest(doctest.DocTestSuite(glider.formats))
Modified: updater/trunk/lib/sexp/access.py
===================================================================
--- updater/trunk/lib/sexp/access.py 2008-09-19 13:14:27 UTC (rev 16927)
+++ updater/trunk/lib/sexp/access.py 2008-09-19 19:22:43 UTC (rev 16928)
@@ -97,7 +97,22 @@
s = s[idx]
idx = 0
+def attrs_to_dict(sexpr):
+ """Return a dictionary mapping keys of the attributes in sexpr to
+ their values. Only the last element in the attribute list counts.
+ >>> s = [ 'given-name',
+ ... ["Tigra", 'Rachel'], ["Bunny", "Elana"] ]
+ >>> attrs_to_dict(s)
+ {'Tigra': ['Rachel'], 'Bunny': ['Elana']}
+ """
+ result = {}
+ for ch in sexpr:
+ tag = s_tag(ch)
+ if tag is not None:
+ result[tag]=ch[1:]
+ return result
+
class SExpr(list):
"""Wraps an s-expresion list to return its tagged children as attributes.
@@ -177,9 +192,9 @@
if s_tag(ch) == p_item[2:]:
_s_lookup_all(ch, path[p_idx+1:], callback)
else:
- s = s_child(s, p_item)
- if s is None:
- return
+ for ch in s_children(s, p_item):
+ _s_lookup_all(ch, path[p_idx+1:], callback)
+ return
callback(s)
@@ -190,7 +205,9 @@
>>> x = ['alice',
... ['father', 'bob', ['mother', 'carol'], ['father', 'dave']],
... ['mother', 'eve', ['mother', 'frances', ['dog', 'spot']],
- ... ['father', 'gill']]]
+ ... ['father', 'gill']],
+ ... ['marmoset', 'tiffany'],
+ ... ['marmoset', 'gilbert'] ]
>>> s_lookup_all(x, "father")
[['father', 'bob', ['mother', 'carol'], ['father', 'dave']]]
>>> s_lookup_all(x, "father.mother")
@@ -203,6 +220,8 @@
[['dog', 'spot']]
>>> s_lookup_all(x, "mother.*.dog")
[['dog', 'spot']]
+ >>> s_lookup_all(x, "marmoset")
+ [['marmoset', 'tiffany'], ['marmoset', 'gilbert']]
"""
result = []
_s_lookup_all(s, path, result.append)
Modified: updater/trunk/lib/sexp/tests.py
===================================================================
--- updater/trunk/lib/sexp/tests.py 2008-09-19 13:14:27 UTC (rev 16927)
+++ updater/trunk/lib/sexp/tests.py 2008-09-19 19:22:43 UTC (rev 16928)
@@ -11,7 +11,6 @@
self.assertEquals(1,1)
-
def suite():
import sexp.tests
suite = unittest.TestSuite()