3 ### Miscellaneous utilities
5 ### (c) 2013 Mark Wooding
8 ###----- Licensing notice ---------------------------------------------------
10 ### This file is part of Chopwood: a password-changing service.
12 ### Chopwood is free software; you can redistribute it and/or modify
13 ### it under the terms of the GNU Affero General Public License as
14 ### published by the Free Software Foundation; either version 3 of the
15 ### License, or (at your option) any later version.
17 ### Chopwood is distributed in the hope that it will be useful,
18 ### but WITHOUT ANY WARRANTY; without even the implied warranty of
19 ### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 ### GNU Affero General Public License for more details.
22 ### You should have received a copy of the GNU Affero General Public
23 ### License along with Chopwood; if not, see
24 ### <http://www.gnu.org/licenses/>.
26 from __future__ import with_statement
29 import contextlib as CTX
37 try: import threading as TH
38 except ImportError: import dummy_threading as TH
40 ###--------------------------------------------------------------------------
44 """The identity function: returns its argument."""
48 """The function which always returns X."""
51 class struct (object):
52 """A simple object for storing data in attributes."""
54 def __init__(me, *args, **kw):
56 for k, v in kw.iteritems(): setattr(me, k, v)
59 except AttributeError:
60 if args: raise ValueError, 'no slots defined'
62 if len(args) > len(slots): raise ValueError, 'too many arguments'
63 for k, v in zip(slots, args): setattr(me, k, v)
65 if hasattr(me, k): continue
66 try: setattr(me, k, cls.DEFAULTS[k])
67 except KeyError: raise ValueError, "no value for `%s'" % k
70 """An object whose only purpose is to be distinct from other objects."""
71 def __init__(me, name):
74 return '#<%s %r>' % (type(me).__name__, me._name)
76 class DictExpanderClass (type):
78 Metaclass for classes with autogenerated members.
80 If the class body defines a dictionary `__extra__' then the key/value pairs
81 in this dictionary are promoted into attributes of the class. This is much
82 easier -- and safer -- than fiddling about with `locals'.
84 def __new__(cls, name, supers, dict):
86 ex = dict['__extra__']
90 for k, v in ex.iteritems():
93 return super(DictExpanderClass, cls).__new__(cls, name, supers, dict)
95 class ExpectedError (Exception):
97 A (concrete) base class for various errors we expect to encounter.
99 The `msg' attribute carries a human-readable message explaining what the
100 problem actually is. The really important bit, though, is the `code'
101 attribute, which carries an HTTP status code to be reported to the user
102 agent, if we're running through CGI.
104 def __init__(me, code, msg):
108 return '%s (%d)' % (me.msg, me.code)
110 def register(dict, name):
111 """A decorator: add the decorated function to DICT, under the key NAME."""
117 class StringSubst (object):
119 A string substitution. Initialize with a dictionary mapping source strings
120 to target strings. The object is callable, and maps strings in the obvious
123 def __init__(me, map):
125 me._rx = RX.compile('|'.join(RX.escape(s) for s in map))
127 return me._rx.sub(lambda m: me._map[m.group(0)], s)
129 def readline(what, file = SYS.stdin):
130 """Read a single line from FILE (default stdin) and return it."""
131 try: line = SYS.stdin.readline()
132 except IOError, e: raise ExpectedError, (500, str(e))
133 if not line.endswith('\n'):
134 raise ExpectedError, (500, "Failed to read %s" % what)
137 class EscapeHatch (BaseException):
138 """Exception used by the `Escape' context manager"""
139 def __init__(me): pass
141 class Escape (object):
143 A context manager. Executes its body until completion or the `Escape'
144 object itself is invoked as a function. Other exceptions propagate
148 me.exc = EscapeHatch()
153 def __exit__(me, exty, exval, extb):
154 return exval is me.exc
156 class Fluid (object):
158 Stores `fluid' variables which can be temporarily bound to new values, and
161 A caller may use the object's attributes for storing arbitrary values
162 (though storing a `bind' value would be silly). The `bind' method provides
163 a context manager which binds attributes to other values during its
164 execution. This works even with multiple threads.
167 ## We maintain two stores for variables. One is a global store, `_g'; the
168 ## other is a thread-local store `_t'. We look for a variable first in the
169 ## thread-local store, and then if necessary in the global store. Binding
170 ## works by remembering the old state of the variable on entry, setting it
171 ## in the thread-local store (always), and then restoring the old state on
174 ## A special marker for unbound variables. If a variable is bound to a
175 ## value, rebound temporarily with `bind', and then deleted, we must
176 ## pretend that it's not there, and then restore it again afterwards. We
177 ## use this tag to mark variables which have been deleted while they're
179 UNBOUND = Tag('unbound-variable')
181 def __init__(me, **kw):
182 """Create a new set of fluid variables, initialized from the keywords."""
183 me.__dict__.update(_g = struct(),
185 for k, v in kw.iteritems():
188 def __getattr__(me, k):
189 """Return the current value stored with K, or raise AttributeError."""
190 try: v = getattr(me._t, k)
191 except AttributeError: v = getattr(me._g, k)
192 if v is Fluid.UNBOUND: raise AttributeError, k
195 def __setattr__(me, k, v):
196 """Associate the value V with the variable K."""
197 if hasattr(me._t, k): setattr(me._t, k, v)
198 else: setattr(me._g, k, v)
200 def __delattr__(me, k):
202 Forget about the variable K, so that attempts to read it result in an
205 if hasattr(me._t, k): setattr(me._t, k, Fluid.UNBOUND)
206 else: delattr(me._g, k)
209 """Return a list of the currently known variables."""
212 for s in [me._t, me._g]:
214 if k in seen: continue
216 if getattr(s, k) is not Fluid.UNBOUND: keys.append(k)
222 A context manager: bind values to variables according to the keywords KW,
223 and execute the body; when the body exits, restore the rebound variables
224 to their previous values.
227 ## A list of things to do when we finish.
231 ## Remove K from the thread-local store. Only it might already have
232 ## been deleted, so be careful.
233 try: delattr(me._t, k)
234 except AttributeError: pass
237 ## Stash a function for restoring the old state of K. We do this here
238 ## rather than inline only because Python's scoping rules are crazy and
239 ## we need to ensure that all of the necessary variables are
241 try: ov = getattr(me._t, k)
242 except AttributeError: unwind.append(lambda: _delattr(k))
243 else: unwind.append(lambda: setattr(me._t, k, ov))
245 ## Rebind the variables.
246 for k, v in kw.iteritems():
250 ## Run the body, and restore.
255 class Cleanup (object):
257 A context manager for stacking other context managers.
259 By itself, it does nothing. Attach other context managers with `enter' or
260 loose cleanup functions with `add'. On exit, contexts are left and
261 cleanups performed in reverse order.
267 def __exit__(me, exty, exval, extb):
269 for c in reversed(me._cleanups):
270 if c(exty, exval, extb): trap = True
274 me._cleanups.append(ctx.__exit__)
277 me._cleanups.append(lambda exty, exval, extb: func())
279 ###--------------------------------------------------------------------------
282 class Encoding (object):
284 A pairing of injective encoding on binary strings, with its appropriate
287 The two functions are available in the `encode' and `decode' attributes.
288 See also the `ENCODINGS' dictionary.
290 def __init__(me, encode, decode):
295 'base64': Encoding(lambda s: BN.b64encode(s),
296 lambda s: BN.b64decode(s)),
297 'base32': Encoding(lambda s: BN.b32encode(s).lower(),
298 lambda s: BN.b32decode(s, casefold = True)),
299 'hex': Encoding(lambda s: BN.b16encode(s).lower(),
300 lambda s: BN.b16decode(s, casefold = True)),
301 None: Encoding(identity, identity)
304 ###--------------------------------------------------------------------------
305 ### Time and timeouts.
309 Reset our idea of the current time, as kept in the global variable `NOW'.
315 class Alarm (Exception):
317 Exception used internally by the `timeout' context manager.
319 If you're very unlucky, you might get one of these at top level.
323 class Timeout (ExpectedError):
325 Report a timeout, from the `timeout' context manager.
327 def __init__(me, what):
328 ExpectedError.__init__(me, 500, "Timeout %s" % what)
330 ## Set `DEADLINE' to be the absolute time of the next alarm. We'll keep this
331 ## up to date in `timeout'.
332 delta, _ = SIG.getitimer(SIG.ITIMER_REAL)
333 if delta == 0: DEADLINE = None
334 else: DEADLINE = NOW + delta
337 """If we receive `SIGALRM', raise the alarm."""
339 SIG.signal(SIG.SIGALRM, _alarm)
342 def timeout(delta, what):
344 A context manager which interrupts execution of its body after DELTA
345 seconds, if it doesn't finish before then.
347 If execution is interrupted, a `Timeout' exception is raised, carrying WHY
348 (a gerund phrase) as part of its message.
353 if DEADLINE is not None and when >= DEADLINE:
360 SIG.setitimer(SIG.ITIMER_REAL, delta)
367 if od is None: SIG.setitimer(SIG.ITIMER_REAL, 0)
368 else: SIG.setitimer(SIG.ITIMER_REAL, DEADLINE - NOW)
370 ###--------------------------------------------------------------------------
374 def lockfile(lock, t = None):
376 Acquire an exclusive lock on a named file LOCK while executing the body.
378 If T is zero, fail immediately if the lock can't be acquired; if T is none,
379 then wait forever if necessary; otherwise give up after T seconds.
383 fd = OS.open(lock, OS.O_WRONLY | OS.O_CREAT, 0600)
385 F.lockf(fd, F.LOCK_EX)
387 F.lockf(fd, F.LOCK_EX | F.LOCK_NB)
389 with timeout(t, "waiting for lock file `%s'" % lock):
390 F.lockf(fd, F.LOCK_EX)
393 if fd != -1: OS.close(fd)
395 ###--------------------------------------------------------------------------
396 ### Database utilities.
398 ### Python's database API is dreadful: it exposes far too many
399 ### implementation-specific details to the programmer, who may well want to
400 ### write code which works against many different databases.
402 ### One particularly frustrating problem is the variability of placeholder
403 ### syntax in SQL statements: there's no universal convention, just a number
404 ### of possible syntaxes, at least one of which will be implemented (and some
405 ### of which are mutually incompatible). Because not doing this invites all
406 ### sorts of misery such as SQL injection vulnerabilties, we introduce a
407 ### simple abstraction. A database parameter-type object keeps track of one
408 ### particular convention, providing the correct placeholders to be inserted
409 ### into the SQL command string, and the corresponding arguments, in whatever
410 ### way is necessary.
412 ### The protocol is fairly simple. An object of the appropriate class is
413 ### instantiated for each SQL statement, providing it with a dictionary
414 ### mapping placeholder names to their values. The object's `sub' method is
415 ### called for each placeholder found in the statement, with a match object
416 ### as an argument; the match object picks out the name of the placeholder in
417 ### question in group 1, and the method returns a piece of syntax appropriate
418 ### to the database backend. Finally, the collected arguments are made
419 ### available, in whatever format is required, in the object's `args'
422 ## Turn simple Unix not-quite-glob patterns into SQL `LIKE' patterns.
423 ## Match using: x LIKE y ESCAPE '\\'
424 globtolike = StringSubst({
425 '\\*': '*', '%': '\\%', '*': '%',
426 '\\?': '?', '_': '\\_', '?': '_'
429 class LinearParam (object):
431 Abstract parent class for `linear' parameter conventions.
433 A linear convention is one where the arguments are supplied as a list, and
434 placeholders are either all identical (with semantics `insert the next
435 argument'), or identify their argument by its position within the list.
437 def __init__(me, kw):
442 name = match.group(1)
443 me.args.append(me._kw[name])
444 marker = me._format()
447 class QmarkParam (LinearParam):
448 def _format(me): return '?'
449 class NumericParam (LinearParam):
450 def _format(me): return ':%d' % me._i
451 class FormatParam (LinearParam):
452 def _format(me): return '%s'
454 class DictParam (object):
456 Abstract parent class for `dictionary' parameter conventions.
458 A dictionary convention is one where the arguments are provided as a
459 dictionary, and placeholders contain a key name identifying the
460 corresponding value in that dictionary.
462 def __init__(me, kw):
465 name = match.group(1)
466 return me._format(name)
467 def NamedParam (object):
468 def _format(me, name): return ':%s' % name
469 def PyFormatParam (object):
470 def _format(me, name): return '%%(%s)s' % name
472 ### Since we're doing a bunch of work to paper over idiosyncratic placeholder
473 ### syntax, we might as well also sort out other problems. The `DB_FIXUPS'
474 ### dictionary maps database module names to functions which might need to do
475 ### clever stuff at connection setup time.
479 @register(DB_FIXUPS, 'sqlite3')
480 def fixup_sqlite3(db):
482 Unfortunately, SQLite learnt about FOREIGN KEY constraints late, and so
483 doesn't enforce them unless explicitly told to.
486 c.execute("PRAGMA foreign_keys = ON")
488 class SimpleDBConnection (object):
490 Represents a database connection, while trying to hide the differences
491 between various kinds of database backends.
494 __metaclass__ = DictExpanderClass
496 ## A map from placeholder convention names to classes implementing them.
499 'numeric': NumericParam,
501 'format': FormatParam,
502 'pyformat': PyFormatParam
505 ## A pattern for our own placeholder syntax.
506 R_PLACE = RX.compile(r'\$(\w+)')
508 def __init__(me, modname, modargs):
510 Make a new database connection, using the module MODNAME, and passing its
511 `connect' function the MODARGS -- which may be either a list or a
515 ## Get the module, and create a connection.
516 mod = __import__(modname)
517 if isinstance(modargs, dict): me._db = mod.connect(**modargs)
518 else: me._db = mod.connect(*modargs)
520 ## Apply any necessary fixups.
521 try: fixup = DB_FIXUPS[modname]
522 except KeyError: pass
525 ## Grab hold of other interesting things.
527 me.Warning = mod.Warning
528 me._placecls = me.PLACECLS[mod.paramstyle]
530 def execute(me, command, **kw):
532 Execute the SQL COMMAND. The keyword arguments are used to provide
533 values corresponding to `$NAME' placeholders in the COMMAND.
535 Return the receiver, so that iterator protocol is convenient.
537 me._cur = me._db.cursor()
538 plc = me._placecls(kw)
539 subst = me.R_PLACE.sub(plc.sub, command)
540 ##PRINT('*** %s : %r' % (subst, plc.args))
541 me._cur.execute(subst, plc.args)
545 """Iterator protocol: simply return the receiver."""
548 """Iterator protocol: return the next row from the current query."""
550 if row is None: raise StopIteration
555 Context protocol: begin a transaction.
559 def __exit__(me, exty, exval, tb):
560 """Context protocol: commit or roll back a transaction."""
562 ##PRINT('>*> ROLLBACK')
565 ##PRINT('>>> COMMIT')
568 ## Import a number of methods from the underlying connection.
570 for _name in ['fetchone', 'fetchmany', 'fetchall']:
572 extra[name] = lambda me, *args, **kw: \
573 getattr(me._cur, name)(*args, **kw)
575 for _name in ['commit', 'rollback']:
577 extra[name] = lambda me, *args, **kw: \
578 getattr(me._db, name)(*args, **kw)
582 ###----- That's all, folks --------------------------------------------------