From: Mark Wooding Date: Mon, 20 Jul 2015 13:40:18 +0000 (+0100) Subject: Merge branches 'mdw/pwsafe' and 'mdw/ec-ptcmpr' X-Git-Tag: 1.1.0~2 X-Git-Url: https://www.chiark.greenend.org.uk/ucgi/~mdw/git/catacomb-python/commitdiff_plain/0e29d9164d5cc4e3cafa509cda19de2e025446c1?hp=e12df5f3a2b03390499cb5f7f904220e59d64bc9 Merge branches 'mdw/pwsafe' and 'mdw/ec-ptcmpr' * mdw/pwsafe: pwsafe: New command `xfer' to transfer data to a new database backend. catacomb/pwsafe.py: Add a backend based on SQLite. catacomb/pwsafe.py, pwsafe: Make GDBM support conditional. catacomb/pwsafe.py: New Git-friendly `DirectoryStorageBackend'. catacomb/pwsafe.py: New FlatFileStorageBackend class. catacomb/pwsafe.py: Add a new ABRUPTP argument to `close' methods. catacomb/pwsafe.py, pwsafe: Dispatching for multiple backends. catacomb/pwsafe.py: Split out the GDBM-specifics from StorageBackend. catacomb/pwsafe.py: Factor database handling out into a StorageBackend. catacomb/pwsafe.py: Commentary fix. pwsafe: Abolish the `chomp' function, and only chomp when reading stdin. catacomb/pwsafe.py: Make `PW' be a context manager, and use it. pwsafe: Get the master passphrase before the new password. pwsafe: Report password mismatch as an error, not an exception. pwsafe: Some simple reformatting. catacomb/pwsafe.py, pwsafe: Replace `PW''s MODE parameter with WRITEP flag. catacomb/pwsafe.py: Abolish the `PWIter' class. pwsafe: Eliminate the `dump' subcommand. pwsafe: Present the list of commands in alphabetical order. pwsafe: Don't produce a backtrace on decryption failure. * mdw/ec-ptcmpr: catacomb.c, ec.c: Bindings for the new EC2OSP/OS2ECP functions. --- diff --git a/catacomb/pwsafe.py b/catacomb/pwsafe.py index 3af69d4..4b4b994 100644 --- a/catacomb/pwsafe.py +++ b/catacomb/pwsafe.py @@ -26,8 +26,74 @@ ###-------------------------------------------------------------------------- ### Imported modules. +from __future__ import with_statement + +import errno as _E +import os as _OS +from cStringIO import StringIO as _StringIO + import catacomb as _C -import gdbm as _G + +###-------------------------------------------------------------------------- +### Text encoding utilities. + +def _literalp(s): + """ + Answer whether S can be represented literally. + + If True, then S can be stored literally, as a metadata item name or + value; if False, then S requires some kind of encoding. + """ + return all(ch.isalnum() or ch in '-_:' for ch in s) + +def _enc_metaname(name): + """Encode NAME as a metadata item name, returning the result.""" + if _literalp(name): + return name + else: + sio = _StringIO() + sio.write('!') + for ch in name: + if _literalp(ch): sio.write(ch) + elif ch == ' ': sio.write('+') + else: sio.write('%%%02x' % ord(ch)) + return sio.getvalue() + +def _dec_metaname(name): + """Decode NAME as a metadata item name, returning the result.""" + if not name.startswith('!'): + return name + else: + sio = _StringIO() + i, n = 1, len(name) + while i < n: + ch = name[i] + i += 1 + if ch == '+': + sio.write(' ') + elif ch == '%': + sio.write(chr(int(name[i:i + 2], 16))) + i += 2 + else: + sio.write(ch) + return sio.getvalue() + +def _b64(s): + """Encode S as base64, without newlines, and trimming `=' padding.""" + return s.encode('base64').translate(None, '\n=') +def _unb64(s): + """Decode S as base64 with trimmed `=' padding.""" + return (s + '='*((4 - len(s))%4)).decode('base64') + +def _enc_metaval(val): + """Encode VAL as a metadata item value, returning the result.""" + if _literalp(val): return val + else: return '?' + _b64(val) + +def _dec_metaval(val): + """Decode VAL as a metadata item value, returning the result.""" + if not val.startswith('?'): return val + else: return _unb64(val[1:]) ###-------------------------------------------------------------------------- ### Underlying cryptography. @@ -124,37 +190,767 @@ class PPK (Crypto): me.salt = salt ###-------------------------------------------------------------------------- -### Password storage. +### Backend storage. + +class StorageBackendRefusal (Exception): + """ + I signify that a StorageBackend subclass has refused to open a file. + + This is used by the StorageBackend.open class method. + """ + pass + +class StorageBackendClass (type): + """ + I am a metaclass for StorageBackend classes. + + My main feature is that I register my concrete instances (with a `NAME' + which is not `None') with the StorageBackend class. + """ + def __init__(me, name, supers, dict): + """ + Register a new concrete StorageBackend subclass. + """ + super(StorageBackendClass, me).__init__(name, supers, dict) + if me.NAME is not None: StorageBackend.register_concrete_subclass(me) + +class StorageBackend (object): + """ + I provide basic protocol for password storage backends. + + I'm an abstract class: you want one of my subclasses if you actually want + to do something useful. But I maintain a list of my subclasses and can + choose an appropriate one to open a database file you've found lying about. + + Backends are responsible for storing and retrieving stuff, but not for the + cryptographic details. Backends need to store two kinds of information: + + * metadata, consisting of a number of property names and their values; + and + + * password mappings, consisting of a number of binary labels and + payloads. + + Backends need to implement the following ordinary methods. See the calling + methods for details of the subclass responsibilities. + + BE._create(FILE) Create a new database in FILE; used by `create'. + + BE._open(FILE, WRITEP) + Open the existing database FILE; used by `open'. + + BE._close(ABRUPTP) Close the database, freeing up any resources. If + ABRUPTP then don't try to commit changes. + + BE._get_meta(NAME, DEFAULT) + Return the value of the metadata item with the given + NAME, or DEFAULT if it doesn't exist; used by + `get_meta'. + + BE._put_meta(NAME, VALUE) + Set the VALUE of the metadata item with the given + NAME, creating one if necessary; used by `put_meta'. + + BE._del_meta(NAME) Forget the metadata item with the given NAME; raise + `KeyError' if there is no such item; used by + `del_meta'. + + BE._iter_meta() Return an iterator over the metadata (NAME, VALUE) + pairs; used by `iter_meta'. + + BE._get_passwd(LABEL) + Return the password payload stored with the (binary) + LABEL; used by `get_passwd'. + + BE._put_passwd(LABEL, PAYLOAD) + Associate the (binary) PAYLOAD with the LABEL, + forgetting any previous payload for that LABEL; used + by `put_passwd'. + + BE._del_passwd(LABEL) Forget the password record with the given LABEL; used + by `_del_passwd'. + + BE._iter_passwds() Return an iterator over the password (LABEL, PAYLOAD) + pairs; used by `iter_passwds'. + + Also, concrete subclasses should define the following class attributes. + + NAME The name of the backend, so that the user can select + it when creating a new database. + + PRIO An integer priority: backends are tried in decreasing + priority order when opening an existing database. + """ + + __metaclass__ = StorageBackendClass + NAME = None + PRIO = 10 + + ## The registry of subclasses. + CLASSES = {} + + FAIL = ['FAIL'] + + @staticmethod + def register_concrete_subclass(sub): + """Register a concrete subclass, so that `open' can try it.""" + StorageBackend.CLASSES[sub.NAME] = sub + + @staticmethod + def byname(name): + """ + Return the concrete subclass with the given NAME. + + Raise `KeyError' if the name isn't found. + """ + return StorageBackend.CLASSES[name] + + @staticmethod + def classes(): + """Return an iterator over the concrete subclasses.""" + return StorageBackend.CLASSES.itervalues() + + @staticmethod + def open(file, writep = False): + """Open a database FILE, using some appropriate backend.""" + _OS.stat(file) + for cls in sorted(StorageBackend.CLASSES.values(), reverse = True, + key = lambda cls: cls.PRIO): + try: return cls(file, writep) + except StorageBackendRefusal: pass + raise StorageBackendRefusal + + @classmethod + def create(cls, file): + """ + Create a new database in the named FILE, using this backend. + + Subclasses must implement the `_create' instance method. + """ + return cls(writep = True, _magic = lambda me: me._create(file)) + + def __init__(me, file = None, writep = False, _magic = None, *args, **kw): + """ + Main constructor. + + Subclasses are not, in general, expected to override this: there's a + somewhat hairy protocol between the constructor and some of the class + methods. Instead, the main hook for customization is the subclass's + `_open' method, which is invoked in the usual case. + """ + super(StorageBackend, me).__init__(*args, **kw) + if me.NAME is None: raise ValueError, 'abstract class' + if _magic is not None: _magic(me) + elif file is None: raise ValueError, 'missing file parameter' + else: me._open(file, writep) + me._writep = writep + me._livep = True + + def close(me, abruptp = False): + """ + Close the database. + + It is harmless to attempt to close a database which has been closed + already. Calls the subclass's `_close' method. + """ + if me._livep: + me._livep = False + me._close(abruptp) + + ## Utilities. + + def _check_live(me): + """Raise an error if the receiver has been closed.""" + if not me._livep: raise ValueError, 'database is closed' + + def _check_write(me): + """Raise an error if the receiver is not open for writing.""" + me._check_live() + if not me._writep: raise ValueError, 'database is read-only' + + def _check_meta_name(me, name): + """ + Raise an error unless NAME is a valid name for a metadata item. + + Metadata names may not start with `$': such names are reserved for + password storage. + """ + if name.startswith('$'): + raise ValueError, "invalid metadata key `%s'" % name + + ## Context protocol. + + def __enter__(me): + """Context protocol: make sure the database is closed on exit.""" + return me + def __exit__(me, exctype, excvalue, exctb): + """Context protocol: see `__enter__'.""" + me.close(excvalue is not None) + + ## Metadata. + + def get_meta(me, name, default = FAIL): + """ + Fetch the value for the metadata item NAME. + + If no such item exists, then return DEFAULT if that was set; otherwise + raise a `KeyError'. + + This calls the subclass's `_get_meta' method, which should return the + requested item or return the given DEFAULT value. It may assume that the + name is valid and the database is open. + """ + me._check_meta_name(name) + me._check_live() + value = me._get_meta(name, default) + if value is StorageBackend.FAIL: raise KeyError, name + return value + + def put_meta(me, name, value): + """ + Store VALUE in the metadata item called NAME. + + This calls the subclass's `_put_meta' method, which may assume that the + name is valid and the database is open for writing. + """ + me._check_meta_name(name) + me._check_write() + me._put_meta(name, value) + + def del_meta(me, name): + """ + Forget about the metadata item with the given NAME. + + This calls the subclass's `_del_meta' method, which may assume that the + name is valid and the database is open for writing. + """ + me._check_meta_name(name) + me._check_write() + me._del_meta(name) + + def iter_meta(me): + """ + Return an iterator over the name/value metadata items. + + This calls the subclass's `_iter_meta' method, which may assume that the + database is open. + """ + me._check_live() + return me._iter_meta() + + def get_passwd(me, label): + """ + Fetch and return the payload stored with the (opaque, binary) LABEL. + + If there is no such payload then raise `KeyError'. + + This calls the subclass's `_get_passwd' method, which may assume that the + database is open. + """ + me._check_live() + return me._get_passwd(label) + + def put_passwd(me, label, payload): + """ + Associate the (opaque, binary) PAYLOAD with the (opaque, binary) LABEL. + + Any previous payload for LABEL is forgotten. + + This calls the subclass's `_put_passwd' method, which may assume that the + database is open for writing. + """ + me._check_write() + me._put_passwd(label, payload) + + def del_passwd(me, label): + """ + Forget any PAYLOAD associated with the (opaque, binary) LABEL. + + If there is no such payload then raise `KeyError'. + + This calls the subclass's `_del_passwd' method, which may assume that the + database is open for writing. + """ + me._check_write() + me._del_passwd(label, payload) + + def iter_passwds(me): + """ + Return an iterator over the stored password label/payload pairs. + + This calls the subclass's `_iter_passwds' method, which may assume that + the database is open. + """ + me._check_live() + return me._iter_passwds() + +try: import gdbm as _G +except ImportError: pass +else: + class GDBMStorageBackend (StorageBackend): + """ + My instances store password data in a GDBM database. -class PWIter (object): + Metadata and password entries are mixed into the same database. The key + for a metadata item is simply its name; the key for a password entry is + the entry's label prefixed by `$', since we're guaranteed that no + metadata item name begins with `$'. + """ + + NAME = 'gdbm' + + def _open(me, file, writep): + try: me._db = _G.open(file, writep and 'w' or 'r') + except _G.error, e: raise StorageBackendRefusal, e + + def _create(me, file): + me._db = _G.open(file, 'n', 0600) + + def _close(me, abruptp): + me._db.close() + me._db = None + + def _get_meta(me, name, default): + try: return me._db[name] + except KeyError: return default + + def _put_meta(me, name, value): + me._db[name] = value + + def _del_meta(me, name): + del me._db[name] + + def _iter_meta(me): + k = me._db.firstkey() + while k is not None: + if not k.startswith('$'): yield k, me._db[k] + k = me._db.nextkey(k) + + def _get_passwd(me, label): + return me._db['$' + label] + + def _put_passwd(me, label, payload): + me._db['$' + label] = payload + + def _del_passwd(me, label): + del me._db['$' + label] + + def _iter_passwds(me): + k = me._db.firstkey() + while k is not None: + if k.startswith('$'): yield k[1:], me._db[k] + k = me._db.nextkey(k) + +try: import sqlite3 as _Q +except ImportError: pass +else: + class SQLiteStorageBackend (StorageBackend): + """ + I represent a password database stored in SQLite. + + Metadata and password items are stored in separate tables, so there's no + conflict. Some additional metadata is stored in the `meta' table, with + names beginning with `$' so as not to conflict with clients: + + $version The schema version of the table. + """ + + NAME = 'sqlite' + VERSION = 0 + + def _open(me, file, writep): + try: + me._db = _Q.connect(file) + ver = me._query_scalar( + "SELECT value FROM meta WHERE name = '$version'", + "version check") + except (_Q.DatabaseError, _Q.OperationalError), e: + raise StorageBackendRefusal, e + if ver is None: raise ValueError, 'database broken (missing $version)' + elif ver < me.VERSION: me._upgrade(ver) + elif ver > me.VERSION: + raise ValueError, 'unknown database schema version (%d > %d)' % \ + (ver, me.VERSION) + + def _create(me, file): + fd = _OS.open(file, _OS.O_WRONLY | _OS.O_CREAT | _OS.O_EXCL, 0600) + _OS.close(fd) + try: + me._db = _Q.connect(file) + c = me._db.cursor() + c.execute(""" + CREATE TABLE meta ( + name TEXT PRIMARY KEY NOT NULL, + value BLOB NOT NULL); + """) + c.execute(""" + CREATE TABLE passwd ( + label BLOB PRIMARY KEY NOT NULL, + payload BLOB NOT NULL); + """) + c.execute(""" + INSERT INTO meta (name, value) VALUES ('$version', ?); + """, [me.VERSION]) + except: + try: _OS.unlink(file) + except OSError: pass + raise + + def _upgrade(me, ver): + """Upgrade the database from schema version VER.""" + assert False, 'how embarrassing' + + def _close(me, abruptp): + if not abruptp: me._db.commit() + me._db.close() + me._db = None + + def _fetch_scalar(me, c, what, default = None): + try: row = next(c) + except StopIteration: val = default + else: val, = row + try: row = next(c) + except StopIteration: pass + else: raise ValueError, 'multiple matching records for %s' % what + return val + + def _query_scalar(me, query, what, default = None, args = []): + c = me._db.cursor() + c.execute(query, args) + return me._fetch_scalar(c, what, default) + + def _get_meta(me, name, default): + v = me._query_scalar("SELECT value FROM meta WHERE name = ?", + "metadata item `%s'" % name, + default = default, args = [name]) + if v is default: return v + else: return str(v) + + def _put_meta(me, name, value): + c = me._db.cursor() + c.execute("INSERT OR REPLACE INTO meta (name, value) VALUES (?, ?)", + [name, buffer(value)]) + + def _del_meta(me, name): + c = me._db.cursor() + c.execute("DELETE FROM meta WHERE name = ?", [name]) + if not c.rowcount: raise KeyError, name + + def _iter_meta(me): + c = me._db.cursor() + c.execute("SELECT name, value FROM meta WHERE name NOT LIKE '$%'") + for k, v in c: yield k, str(v) + + def _get_passwd(me, label): + pld = me._query_scalar("SELECT payload FROM passwd WHERE label = ?", + "password", default = None, + args = [buffer(label)]) + if pld is None: raise KeyError, label + return str(pld) + + def _put_passwd(me, label, payload): + c = me._db.cursor() + c.execute("INSERT OR REPLACE INTO passwd (label, payload) " + "VALUES (?, ?)", + [buffer(label), buffer(payload)]) + + def _del_passwd(me, label): + c = me._db.cursor() + c.execute("DELETE FROM passwd WHERE label = ?", [label]) + if not c.rowcount: raise KeyError, label + + def _iter_passwds(me): + c = me._db.cursor() + c.execute("SELECT label, payload FROM passwd") + for k, v in c: yield str(k), str(v) + +class PlainTextBackend (StorageBackend): """ - I am an iterator over items in a password database. + I'm a utility base class for storage backends which use plain text files. + + I provide subclasses with the following capabilities. + + * Creating files, with given modes, optionally ensuring that the file + doesn't exist already. + + * Parsing flat text files, checking leading magic, skipping comments, and + providing standard encodings of troublesome characters and binary + strings in metadata and password records. See below. + + * Maintenance of metadata and password records in in-memory dictionaries, + with ready implementations of the necessary StorageBackend subclass + responsibility methods. (Subclasses can override these if they want to + make different arrangements.) - I implement the usual Python iteration protocol. + Metadata records are written with an optional prefix string chosen by the + caller, followed by a `NAME=VALUE' pair. The NAME is form-urlencoded and + prefixed with `!' if it contains strange characters; the VALUE is base64- + encoded (without the pointless trailing `=' padding) and prefixed with `?' + if necessary. + + Password records are written with an optional prefix string chosen by the + caller, followed by a LABEL=PAYLOAD pair, both of which are base64-encoded + (without padding). + + The following attributes are available for subclasses: + + _meta Dictionary mapping metadata item names to their values. + Populated by `_parse_meta' and managed by `_get_meta' and + friends. + + _pw Dictionary mapping password labels to encrypted payloads. + Populated by `_parse_passwd' and managed by `_get_passwd' and + friends. + + _dirtyp Boolean: set if either of the dictionaries has been modified. """ - def __init__(me, pw): + def __init__(me, *args, **kw): + """ + Hook for initialization. + + Sets up the published instance attributes. + """ + me._meta = {} + me._pw = {} + me._dirtyp = False + super(PlainTextBackend, me).__init__(*args, **kw) + + def _create_file(me, file, mode = 0600, freshp = False): + """ + Make sure FILE exists, creating it with the given MODE if necessary. + + If FRESHP is true, then make sure the file did not exist previously. + Return a file object for the newly created file. + """ + flags = _OS.O_CREAT | _OS.O_WRONLY + if freshp: flags |= _OS.O_EXCL + else: flags |= _OS.O_TRUNC + fd = _OS.open(file, flags, mode) + return _OS.fdopen(fd, 'w') + + def _mark_dirty(me): + """ + Set the `_dirtyp' flag. + + Subclasses might find it useful to intercept this method. + """ + me._dirtyp = True + + def _eqsplit(me, line): + """ + Extract the KEY, VALUE pair from a LINE of the form `KEY=VALUE'. + + Raise `ValueError' if there is no `=' in the LINE. + """ + eq = line.index('=') + return line[:eq], line[eq + 1:] + + def _parse_file(me, file, magic = None): """ - Initialize a PWIter object, to fetch items from PW. + Parse a FILE. + + Specifically: + + * Raise `StorageBackendRefusal' if that the first line doesn't match + MAGIC (if provided). MAGIC should not contain the terminating + newline. + + * Ignore comments (beginning `#') and blank lines. + + * Call `_parse_line' (provided by the subclass) for other lines. """ - me.pw = pw - me.k = me.pw.db.firstkey() + with open(file, 'r') as f: + if magic is not None: + if f.readline().rstrip('\n') != magic: raise StorageBackendRefusal + for line in f: + line = line.rstrip('\n') + if not line or line.startswith('#'): continue + me._parse_line(line) - def next(me): + def _write_file(me, file, writebody, mode = 0600, magic = None): """ - Return the next tag from the database. + Update FILE atomically. - Raises StopIteration if there are no more tags. + The newly created file will have the given MODE. If MAGIC is given, then + write that as the first line. Calls WRITEBODY(F) to write the main body + of the file where F is a file object for the new file. """ - k = me.k - while True: - if k is None: - raise StopIteration - if k[0] == '$': - break - k = me.pw.db.nextkey(k) - me.k = me.pw.db.nextkey(k) - return me.pw.unpack(me.pw.db[k])[0] + new = file + '.new' + with me._create_file(new, mode) as f: + if magic is not None: f.write(magic + '\n') + writebody(f) + _OS.rename(new, file) + + def _parse_meta(me, line): + """Parse LINE as a metadata NAME=VALUE pair, and updates `_meta'.""" + k, v = me._eqsplit(line) + me._meta[_dec_metaname(k)] = _dec_metaval(v) + + def _write_meta(me, f, prefix = ''): + """Write the metadata records to F, each with the given PREFIX.""" + f.write('\n## Metadata.\n') + for k, v in me._meta.iteritems(): + f.write('%s%s=%s\n' % (prefix, _enc_metaname(k), _enc_metaval(v))) + + def _get_meta(me, name, default): + return me._meta.get(name, default) + def _put_meta(me, name, value): + me._mark_dirty() + me._meta[name] = value + def _del_meta(me, name): + me._mark_dirty() + del me._meta[name] + def _iter_meta(me): + return me._meta.iteritems() + + def _parse_passwd(me, line): + """Parse LINE as a password LABEL=PAYLOAD pair, and updates `_pw'.""" + k, v = me._eqsplit(line) + me._pw[_unb64(k)] = _unb64(v) + + def _write_passwd(me, f, prefix = ''): + """Write the password records to F, each with the given PREFIX.""" + f.write('\n## Password data.\n') + for k, v in me._pw.iteritems(): + f.write('%s%s=%s\n' % (prefix, _b64(k), _b64(v))) + + def _get_passwd(me, label): + return me._pw[str(label)] + def _put_passwd(me, label, payload): + me._mark_dirty() + me._pw[str(label)] = payload + def _del_passwd(me, label): + me._mark_dirty() + del me._pw[str(label)] + def _iter_passwds(me): + return me._pw.iteritems() + +class FlatFileStorageBackend (PlainTextBackend): + """ + I maintain a password database in a plain text file. + + The text file consists of lines, as follows. + + * Empty lines, and lines beginning with `#' (in the leftmost column only) + are ignored. + + * Lines of the form `$LABEL=PAYLOAD' store password data. Both LABEL and + PAYLOAD are base64-encoded, without `=' padding. + + * Lines of the form `NAME=VALUE' store metadata. If the NAME contains + characters other than alphanumerics, hyphens, underscores, and colons, + then it is form-urlencoded, and prefixed wth `!'. If the VALUE + contains such characters, then it is base64-encoded, without `=' + padding, and prefixed with `?'. + + * Other lines are erroneous. + + The file is rewritten from scratch when it's changed: any existing + commentary is lost, and items may be reordered. There is no file locking, + but the file is updated atomically, by renaming. + + It is expected that the FlatFileStorageBackend is used mostly for + diagnostics and transfer, rather than for a live system. + """ + + NAME = 'flat' + PRIO = 0 + MAGIC = '### pwsafe password database' + + def _open(me, file, writep): + if not _OS.path.isfile(file): raise StorageBackendRefusal + me._parse_file(file, magic = me.MAGIC) + def _parse_line(me, line): + if line.startswith('$'): me._parse_passwd(line[1:]) + else: me._parse_meta(line) + + def _create(me, file): + with me._create_file(file, freshp = True) as f: pass + me._file = file + me._mark_dirty() + + def _close(me, abruptp): + if not abruptp and me._dirtyp: + me._write_file(me._file, me._write_body, magic = me.MAGIC) + + def _write_body(me, f): + me._write_meta(f) + me._write_passwd(f, '$') + +class DirectoryStorageBackend (PlainTextBackend): + """ + I maintain a password database in a directory, with one file per password. + + This makes password databases easy to maintain in a revision-control system + such as Git. + + The directory is structured as follows. + + dir/meta Contains metadata, similar to the `FlatFileBackend'. + + dir/pw/LABEL Contains the (raw binary) payload for the given password + LABEL (base64-encoded, without the useless `=' padding, and + with `/' replaced by `.'). + + dir/tmp/ Contains temporary files used by the implementation. + """ + + NAME = 'dir' + METAMAGIC = '### pwsafe password directory metadata' + + def _open(me, file, writep): + if not _OS.path.isdir(file) or \ + not _OS.path.isdir(_OS.path.join(file, 'pw')) or \ + not _OS.path.isdir(_OS.path.join(file, 'tmp')) or \ + not _OS.path.isfile(_OS.path.join(file, 'meta')): + raise StorageBackendRefusal + me._dir = file + me._parse_file(_OS.path.join(file, 'meta'), magic = me.METAMAGIC) + def _parse_line(me, line): + me._parse_meta(line) + + def _create(me, file): + _OS.mkdir(file, 0700) + _OS.mkdir(_OS.path.join(file, 'pw'), 0700) + _OS.mkdir(_OS.path.join(file, 'tmp'), 0700) + me._mark_dirty() + me._dir = file + + def _close(me, abruptp): + if not abruptp and me._dirtyp: + me._write_file(_OS.path.join(me._dir, 'meta'), + me._write_meta, magic = me.METAMAGIC) + + def _pwfile(me, label, dir = 'pw'): + return _OS.path.join(me._dir, dir, _b64(label).replace('/', '.')) + def _get_passwd(me, label): + try: + f = open(me._pwfile(label), 'rb') + except (OSError, IOError), e: + if e.errno == _E.ENOENT: raise KeyError, label + else: raise + with f: return f.read() + def _put_passwd(me, label, payload): + new = me._pwfile(label, 'tmp') + fd = _OS.open(new, _OS.O_WRONLY | _OS.O_CREAT | _OS.O_TRUNC, 0600) + _OS.close(fd) + with open(new, 'wb') as f: f.write(payload) + _OS.rename(new, me._pwfile(label)) + def _del_passwd(me, label): + try: + _OS.remove(me._pwfile(label)) + except (OSError, IOError), e: + if e == _E.ENOENT: raise KeyError, label + else: raise + def _iter_passwds(me): + pw = _OS.path.join(me._dir, 'pw') + for i in _OS.listdir(pw): + with open(_OS.path.join(pw, i), 'rb') as f: pld = f.read() + yield _unb64(i.replace('.', '/')), pld + +###-------------------------------------------------------------------------- +### Password storage. class PW (object): """ @@ -163,11 +959,11 @@ class PW (object): I can store short secrets, associated with textual names, in a way which doesn't leak too much information about them. - I implement (some of the) Python mapping protocol. + I implement (some of) the Python mapping protocol. - Here's how we use the underlying GDBM key/value storage to keep track of - the necessary things. Password entries have keys whose name begins with - `$'; other keys have specific meanings, as follows. + I keep track of everything using a StorageBackend object. This contains + password entries, identified by cryptographic labels, and a number of + metadata items. cipher Names the Catacomb cipher selected. @@ -185,34 +981,34 @@ class PW (object): tag The master passphrase's tag, for the Pixie's benefit. - Password entries are assigned keys of the form `$' || H(MAGIC || TAG); the - corresponding value consists of a pair (TAG, PASSWD), prefixed with 16-bit - lengths, concatenated, padded to a multiple of 256 octets, and encrypted - using the stored keys. + Password entries are assigned labels of the form `$' || H(MAGIC || TAG); + the corresponding value consists of a pair (TAG, PASSWD), prefixed with + 16-bit lengths, concatenated, padded to a multiple of 256 octets, and + encrypted using the stored keys. """ - def __init__(me, file, mode = 'r'): + def __init__(me, file, writep = False): """ - Initialize a PW object from the GDBM database in FILE. + Initialize a PW object from the database in FILE. - MODE can be `r' for read-only access to the underlying database, or `w' - for read-write access. Requests the database password from the Pixie, - which may cause interaction. + If WRITEP is false (the default) then the database is opened read-only; + if true then it may be written. Requests the database password from the + Pixie, which may cause interaction. """ ## Open the database. - me.db = _G.open(file, mode) + me.db = StorageBackend.open(file, writep) ## Find out what crypto to use. - c = _C.gcciphers[me.db['cipher']] - h = _C.gchashes[me.db['hash']] - m = _C.gcmacs[me.db['mac']] + c = _C.gcciphers[me.db.get_meta('cipher')] + h = _C.gchashes[me.db.get_meta('hash')] + m = _C.gcmacs[me.db.get_meta('mac')] ## Request the passphrase and extract the master keys. - tag = me.db['tag'] - ppk = PPK(_C.ppread(tag), c, h, m, me.db['salt']) + tag = me.db.get_meta('tag') + ppk = PPK(_C.ppread(tag), c, h, m, me.db.get_meta('salt')) try: - b = _C.ReadBuffer(ppk.decrypt(me.db['key'])) + b = _C.ReadBuffer(ppk.decrypt(me.db.get_meta('key'))) except DecryptError: _C.ppcancel(tag) raise @@ -222,12 +1018,12 @@ class PW (object): ## Set the key, and stash it and the tag-hashing secret. me.k = Crypto(c, h, m, me.ck, me.mk) - me.magic = me.k.decrypt(me.db['magic']) + me.magic = me.k.decrypt(me.db.get_meta('magic')) @classmethod - def create(cls, file, c, h, m, tag): + def create(cls, dbcls, file, tag, c, h, m): """ - Create and initialize a new, empty, database FILE. + Create and initialize a new database FILE using StorageBackend DBCLS. We want a GCipher subclass C, a GHash subclass H, and a GMAC subclass M; and a Pixie passphrase TAG. @@ -244,20 +1040,19 @@ class PW (object): k = Crypto(c, h, m, ck, mk) ## Set up and initialize the database. - db = _G.open(file, 'n', 0600) - db['tag'] = tag - db['salt'] = ppk.salt - db['cipher'] = c.name - db['hash'] = h.name - db['mac'] = m.name - db['key'] = ppk.encrypt(_C.WriteBuffer().putblk16(ck).putblk16(mk)) - db['magic'] = k.encrypt(_C.rand.block(h.hashsz)) + kct = ppk.encrypt(_C.WriteBuffer().putblk16(ck).putblk16(mk)) + with dbcls.create(file) as db: + db.put_meta('tag', tag) + db.put_meta('salt', ppk.salt) + db.put_meta('cipher', c.name) + db.put_meta('hash', h.name) + db.put_meta('mac', m.name) + db.put_meta('key', kct) + db.put_meta('magic', k.encrypt(_C.rand.block(h.hashsz))) def keyxform(me, key): - """ - Transform the KEY (actually a password tag) into a GDBM record key. - """ - return '$' + me.k.h().hash(me.magic).hash(key).done() + """Transform the KEY (actually a password tag) into a password label.""" + return me.k.h().hash(me.magic).hash(key).done() def changepp(me): """ @@ -266,18 +1061,16 @@ class PW (object): Requests the new password from the Pixie, which will probably cause interaction. """ - tag = me.db['tag'] + tag = me.db.get_meta('tag') _C.ppcancel(tag) ppk = PPK(_C.ppread(tag, _C.PMODE_VERIFY), me.k.c.__class__, me.k.h, me.k.m.__class__) - me.db['key'] = \ - ppk.encrypt(_C.WriteBuffer().putblk16(me.ck).putblk16(me.mk)) - me.db['salt'] = ppk.salt + kct = ppk.encrypt(_C.WriteBuffer().putblk16(me.ck).putblk16(me.mk)) + me.db.put_meta('key', kct) + me.db.put_meta('salt', ppk.salt) def pack(me, key, value): - """ - Pack the KEY and VALUE into a ciphertext, and return it. - """ + """Pack the KEY and VALUE into a ciphertext, and return it.""" b = _C.WriteBuffer() b.putblk16(key).putblk16(value) b.zero(((b.size + 255) & ~255) - b.size) @@ -297,33 +1090,29 @@ class PW (object): ## Mapping protocol. def __getitem__(me, key): - """ - Return the password for the given KEY. - """ - try: - return me.unpack(me.db[me.keyxform(key)])[1] - except KeyError: - raise KeyError, key + """Return the password for the given KEY.""" + try: return me.unpack(me.db.get_passwd(me.keyxform(key)))[1] + except KeyError: raise KeyError, key def __setitem__(me, key, value): - """ - Associate the password VALUE with the KEY. - """ - me.db[me.keyxform(key)] = me.pack(key, value) + """Associate the password VALUE with the KEY.""" + me.db.put_passwd(me.keyxform(key), me.pack(key, value)) def __delitem__(me, key): - """ - Forget all about the KEY. - """ - try: - del me.db[me.keyxform(key)] - except KeyError: - raise KeyError, key + """Forget all about the KEY.""" + try: me.db.del_passwd(me.keyxform(key)) + except KeyError: raise KeyError, key def __iter__(me): - """ - Iterate over the known password tags. - """ - return PWIter(me) + """Iterate over the known password tags.""" + for _, pld in me.db.iter_passwds(): + yield me.unpack(pld)[0] + + ## Context protocol. + + def __enter__(me): + return me + def __exit__(me, excty, excval, exctb): + me.db.close(excval is not None) ###----- That's all, folks -------------------------------------------------- diff --git a/pwsafe b/pwsafe index 91684c7..c12f856 100644 --- a/pwsafe +++ b/pwsafe @@ -27,7 +27,8 @@ ###--------------------------------------------------------------------------- ### Imported modules. -import gdbm as G +from __future__ import with_statement + from os import environ from sys import argv, exit, stdin, stdout, stderr from getopt import getopt, GetoptError @@ -52,28 +53,6 @@ def die(msg): moan(msg) exit(1) -def chomp(pp): - """Return the string PP, without its trailing newline if it has one.""" - if len(pp) > 0 and pp[-1] == '\n': - pp = pp[:-1] - return pp - -def asciip(s): - """Answer whether all of the characters of S are plain ASCII.""" - for ch in s: - if ch < ' ' or ch > '~': return False - return True - -def present(s): - """ - Return a presentation form of the string S. - - If S is plain ASCII, then return S unchanged; otherwise return it as one of - Catacomb's ByteString objects. - """ - if asciip(s): return s - return C.ByteString(s) - ###-------------------------------------------------------------------------- ### Subcommand implementations. @@ -86,121 +65,109 @@ def cmd_create(av): ## Parse the options. try: - opts, args = getopt(av, 'c:h:m:', ['cipher=', 'mac=', 'hash=']) + opts, args = getopt(av, 'c:d:h:m:', + ['cipher=', 'database=', 'mac=', 'hash=']) except GetoptError: return 1 + dbty = 'flat' for o, a in opts: - if o in ('-c', '--cipher'): - cipher = a - elif o in ('-m', '--mac'): - mac = a - elif o in ('-h', '--hash'): - hash = a - else: - raise 'Barf!' - if len(args) > 2: - return 1 - if len(args) >= 1: - tag = args[0] - else: - tag = 'pwsafe' + if o in ('-c', '--cipher'): cipher = a + elif o in ('-m', '--mac'): mac = a + elif o in ('-h', '--hash'): hash = a + elif o in ('-d', '--database'): dbty = a + else: raise 'Barf!' + if len(args) > 2: return 1 + if len(args) >= 1: tag = args[0] + else: tag = 'pwsafe' ## Set up the database. if mac is None: mac = hash + '-hmac' - PW.create(file, C.gcciphers[cipher], C.gchashes[hash], C.gcmacs[mac], tag) + try: dbcls = StorageBackend.byname(dbty) + except KeyError: die("Unknown database backend `%s'" % dbty) + PW.create(dbcls, file, tag, + C.gcciphers[cipher], C.gchashes[hash], C.gcmacs[mac]) def cmd_changepp(av): - if len(av) != 0: - return 1 - pw = PW(file, 'w') - pw.changepp() + if len(av) != 0: return 1 + with PW(file, writep = True) as pw: pw.changepp() def cmd_find(av): - if len(av) != 1: - return 1 - pw = PW(file) - try: - print pw[av[0]] - except KeyError, exc: - die('Password `%s\' not found.' % exc.args[0]) + if len(av) != 1: return 1 + with PW(file) as pw: + try: print pw[av[0]] + except KeyError, exc: die("Password `%s' not found" % exc.args[0]) def cmd_store(av): - if len(av) < 1 or len(av) > 2: - return 1 + if len(av) < 1 or len(av) > 2: return 1 tag = av[0] - if len(av) < 2: - pp = C.getpass("Enter passphrase `%s': " % tag) - vpp = C.getpass("Confirm passphrase `%s': " % tag) - if pp != vpp: - raise ValueError, "passphrases don't match" - elif av[1] == '-': - pp = stdin.readline() - else: - pp = av[1] - pw = PW(file, 'w') - pw[av[0]] = chomp(pp) + with PW(file, writep = True) as pw: + if len(av) < 2: + pp = C.getpass("Enter passphrase `%s': " % tag) + vpp = C.getpass("Confirm passphrase `%s': " % tag) + if pp != vpp: die("passphrases don't match") + elif av[1] == '-': + pp = stdin.readline().rstrip('\n') + else: + pp = av[1] + pw[av[0]] = pp def cmd_copy(av): - if len(av) < 1 or len(av) > 2: - return 1 - pw_in = PW(file) - pw_out = PW(av[0], 'w') - if len(av) >= 3: - pat = av[1] - else: - pat = None - for k in pw_in: - if pat is None or fnmatch(k, pat): - pw_out[k] = pw_in[k] + if len(av) < 1 or len(av) > 2: return 1 + with PW(file) as pw_in: + with PW(av[0], writep = True) as pw_out: + if len(av) >= 3: pat = av[1] + else: pat = None + for k in pw_in: + if pat is None or fnmatch(k, pat): pw_out[k] = pw_in[k] def cmd_list(av): - if len(av) > 1: - return 1 - pw = PW(file) - if len(av) >= 1: - pat = av[0] - else: - pat = None - for k in pw: - if pat is None or fnmatch(k, pat): - print k + if len(av) > 1: return 1 + with PW(file) as pw: + if len(av) >= 1: pat = av[0] + else: pat = None + for k in pw: + if pat is None or fnmatch(k, pat): print k def cmd_topixie(av): - if len(av) > 2: - return 1 - pw = PW(file) - pix = C.Pixie() - if len(av) == 0: - for tag in pw: - pix.set(tag, pw[tag]) - else: - tag = av[0] - if len(av) >= 2: - pptag = av[1] + if len(av) > 2: return 1 + with PW(file) as pw: + pix = C.Pixie() + if len(av) == 0: + for tag in pw: pix.set(tag, pw[tag]) else: - pptag = av[0] - pix.set(pptag, pw[tag]) + tag = av[0] + if len(av) >= 2: pptag = av[1] + else: pptag = av[0] + pix.set(pptag, pw[tag]) def cmd_del(av): - if len(av) != 1: - return 1 - pw = PW(file, 'w') - tag = av[0] - try: - del pw[tag] - except KeyError, exc: - die('Password `%s\' not found.' % exc.args[0]) - -def cmd_dump(av): - db = gdbm.open(file, 'r') - k = db.firstkey() - while True: - if k is None: break - print '%r: %r' % (present(k), present(db[k])) - k = db.nextkey(k) + if len(av) != 1: return 1 + with PW(file, writep = True) as pw: + tag = av[0] + try: del pw[tag] + except KeyError, exc: die("Password `%s' not found" % exc.args[0]) + +def cmd_xfer(av): + + ## Parse the command line. + try: opts, args = getopt(av, 'd:', ['database=']) + except GetoptError: return 1 + dbty = 'flat' + for o, a in opts: + if o in ('-d', '--database'): dbty = a + else: raise 'Barf!' + if len(args) != 1: return 1 + try: dbcls = StorageBackend.byname(dbty) + except KeyError: die("Unknown database backend `%s'" % dbty) + + ## Create the target database. + with StorageBackend.open(file) as db_in: + with dbcls.create(args[0]) as db_out: + for k, v in db_in.iter_meta(): db_out.put_meta(k, v) + for k, v in db_in.iter_passwds(): db_out.put_passwd(k, v) commands = { 'create': [cmd_create, - '[-c CIPHER] [-h HASH] [-m MAC] [PP-TAG]'], + '[-c CIPHER] [-d DBTYPE] [-h HASH] [-m MAC] [PP-TAG]'], 'find' : [cmd_find, 'LABEL'], 'store' : [cmd_store, 'LABEL [VALUE]'], 'list' : [cmd_list, '[GLOB-PATTERN]'], @@ -208,13 +175,15 @@ commands = { 'create': [cmd_create, 'copy' : [cmd_copy, 'DEST-FILE [GLOB-PATTERN]'], 'to-pixie' : [cmd_topixie, '[TAG [PIXIE-TAG]]'], 'delete' : [cmd_del, 'TAG'], - 'dump' : [cmd_dump, '']} + 'xfer': [cmd_xfer, '[-d DBTYPE] DEST-FILE'] } ###-------------------------------------------------------------------------- ### Command-line handling and dispatch. def version(): print '%s 1.0.0' % prog + print 'Backend types: %s' % \ + ' '.join([c.NAME for c in StorageBackend.classes()]) def usage(fp): print >>fp, 'Usage: %s COMMAND [ARGS...]' % prog @@ -236,7 +205,7 @@ Options: Commands provided: ''' - for c in commands: + for c in sorted(commands): print '%s %s' % (c, commands[c][1]) ## Choose a default database file. @@ -276,8 +245,11 @@ if argv[0] in commands: argv = argv[1:] else: c = 'find' -if commands[c][0](argv): - print >>stderr, 'Usage: %s %s %s' % (prog, c, commands[c][1]) - exit(1) +try: + if commands[c][0](argv): + print >>stderr, 'Usage: %s %s %s' % (prog, c, commands[c][1]) + exit(1) +except DecryptError: + die("decryption failure") ###----- That's all, folks --------------------------------------------------