chiark / gitweb /
httpauth.py: Don't crash if Base-64 decoding of the CSRF token fails.
[chopwood] / agpl.py
... / ...
CommitLineData
1### -*-python-*-
2###
3### GNU Affero General Public License compliance
4###
5### (c) 2013 Mark Wooding
6###
7
8###----- Licensing notice ---------------------------------------------------
9###
10### This file is part of Chopwood: a password-changing service.
11###
12### Chopwood is free software; you can redistribute it and/or modify
13### it under the terms of the GNU Affero General Public License as
14### published by the Free Software Foundation; either version 3 of the
15### License, or (at your option) any later version.
16###
17### Chopwood is distributed in the hope that it will be useful,
18### but WITHOUT ANY WARRANTY; without even the implied warranty of
19### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20### GNU Affero General Public License for more details.
21###
22### You should have received a copy of the GNU Affero General Public
23### License along with Chopwood; if not, see
24### <http://www.gnu.org/licenses/>.
25
26import contextlib as CTX
27import grp as GR
28import os as OS
29import pwd as PW
30import shlex as SL
31import shutil as SH
32import subprocess as SUB
33import sys as SYS
34import tarfile as TAR
35import tempfile as TF
36import time as T
37
38from cStringIO import StringIO
39
40from auto import PACKAGE, VERSION
41import util as U
42
43###--------------------------------------------------------------------------
44### Initial utilities.
45
46@CTX.contextmanager
47def tempdir():
48 """
49 Context manager: create and return the name of a temporary directory.
50
51 The directory will be deleted automatically on exit from the body.
52 """
53 d = TF.mkdtemp()
54 try: yield d
55 finally: SH.rmtree(d, ignore_errors = True)
56
57###--------------------------------------------------------------------------
58### Determining which files to include.
59
60def dirs_to_dump():
61 """
62 Return a list of directories containing used Python modules.
63
64 Directories under `/usr/' but outside `/usr/local/' are excluded, since
65 they are assumed to be covered by the AGPL exception for parts of the
66 operating system.
67 """
68
69 ## Collect a set of directories.
70 dirs = set()
71
72 ## Work through the list of known modules, adding them to the list.
73 for m in SYS.modules.itervalues():
74 try: f = m.__file__
75 except AttributeError: continue
76 d = OS.path.realpath(OS.path.dirname(f))
77 if d.startswith('/usr/') and not d.startswith('/usr/local/'): continue
78 dirs.add(d)
79
80 ## Now go through the directories again, and remove any which are wholly
81 ## included within other entries.
82 dirs = sorted(dirs)
83 last = '!'
84 dump = []
85 for d in dirs:
86 if d.startswith(last): continue
87 dump.append(d)
88 last = d
89
90 ## We're done: return the filtered list.
91 return dump
92
93### The `DUMPERS' list consists of (PREDICATE, LISTERS) pairs. The PREDICATE
94### is a function of one argument -- a directory name -- which returns true
95### if the LISTERS should be used to enumerate that directory. The LISTERS
96### are a list of functions of one argument -- again, the directory name --
97### which should return an iterable of files within that directory, relative
98### to its top-level. Lister functions should not return the root directory,
99### because it should obviously only be included once. Instead, the root is
100### handled separately by `dump_dir'.
101
102def exists_subdir(subdir):
103 """
104 Predicate for `DUMPERS': match if the directory has a subdirectory SUBDIR.
105
106 This is mainly useful for detecting working trees subject to version
107 control.
108 """
109 return lambda dir: OS.path.isdir(OS.path.join(dir, subdir))
110
111def filez(cmd):
112 """
113 Lister for `DUMPERS': generate the null-terminated items output by CMD.
114
115 Run CMD, a string containing words with shell-like quoting (expected to be
116 a literal in the code, so security concerns don't arise) in the directory
117 of interest, yielding the invidual null-terminated strings which the
118 command writes to its standard output.
119 """
120 def _(dir):
121
122 ## Start the command,
123 kid = SUB.Popen(SL.split(cmd), stdout = SUB.PIPE, cwd = dir)
124
125 ## Collect and return the null-terminated items. Strip off any leading
126 ## `./' and exclude the root directory because that gets handled
127 ## separately.
128 left = ''
129 while True:
130
131 ## Read a new bufferload of stuff. If there's nothing left then we're
132 ## done.
133 buf = kid.stdout.read(16384)
134 if not buf: break
135
136 ## Tack whatever was left over from last time on the front, and carve
137 ## into null-terminated pieces.
138 buf = left + buf
139 i = 0
140 while True:
141 z = buf.find('\0', i)
142 if z < 0: break
143 f = buf[i:z]
144 i = z + 1
145 if f.rstrip('/') == '.': continue
146 if f.startswith('./'): f = f[2:]
147 yield f
148
149 ## Whatever's left over will be dealt with next time through.
150 left = buf[i:]
151
152 ## Make sure the command actually completed successfully.
153 if kid.wait():
154 rc = kid.returncode
155 raise U.ExpectedError, \
156 (500, "lister command `%s' failed (%s) in `%s'" % (
157 cmd,
158 (rc & 0xff00) and 'rc = %d' % (rc >> 8) or 'signal %d' % rc,
159 dir))
160
161 ## If there's trailing junk left over then we should complain.
162 if left:
163 raise U.ExpectedError, \
164 (500, "trailing junk from `%s' in `%s'" % (cmd, dir))
165
166 ## Return the listing function.
167 return _
168
169## The list of predicates and listers.
170DUMPERS = [
171 (exists_subdir('.git'), [filez('git ls-files -coz --exclude-standard'),
172 filez('find .git -print0')]),
173 (lambda d: True, [filez('find . ( ! -perm +004 -prune ) -o -print0')])]
174
175###--------------------------------------------------------------------------
176### Actually dumping files.
177
178def dump_dir(name, dir, dirmap, tf, root):
179 """
180 Add the contents of directory DIR to the tarfile TF, under the given NAME.
181
182 The ROOT names the toplevel of the tarball (we're not in the business of
183 making tarbombs here). The DIRMAP is a list of all of the (DIR, NAME)
184 pairs being dumped, used for fixing up symbolic links between directories.
185 """
186
187 ## Find an appropriate `DUMPERS' list entry.
188 for test, listers in DUMPERS:
189 if test(dir): break
190 else:
191 raise U.ExpectedError, (500, "no dumper for `%s'" % dir)
192
193 ## Write a tarfile entry for the toplevel.
194 tf.add(dir, OS.path.join(root, name), recursive = False)
195
196 ## Work through all of the listers.
197 for lister in listers:
198
199 ## Work through each file.
200 for file in lister(dir):
201 with U.Escape() as skip:
202 full = OS.path.join(dir, file)
203 tarname = OS.path.join(root, name, file)
204
205 ## Check for symbolic links. If we find one that points to another
206 ## directory we're going to dump separately then fiddle it so that it
207 ## works in the directory tree we're going to make.
208 if OS.path.islink(full):
209 dest = OS.path.realpath(full)
210 for d, local in dirmap:
211 if dest.startswith(d):
212 fix = OS.path.relpath(OS.path.join('/', local, dest[len(d):]),
213 OS.path.join('/', name,
214 OS.path.dirname(file)))
215 st = OS.stat(full)
216 ti = tf.gettarinfo(full, tarname)
217 ti.linkname = fix
218 tf.addfile(ti)
219 skip()
220
221 ## Nothing special, so just dump the file. Or whatever it is.
222 tf.add(full, tarname, recursive = False)
223
224def source(out):
225 """
226 Write a tarball for the program's source code to OUT.
227
228 This function automatically dumps all of the program's dependencies except
229 for those covered by the operating-system exemption.
230 """
231
232 ## Make a tarfile writer. There's an annoying incompatibility to bodge
233 ## around.
234 if SYS.version_info >= (2, 6):
235 tf = TAR.open(fileobj = out, mode = 'w|gz', format = TAR.USTAR_FORMAT)
236 else:
237 tf = TAR.open(fileobj = out, mode = 'w|gz')
238 tf.posix = True
239
240 ## First of all, find out what needs to be dumped, and assign names to all
241 ## of the various directories.
242 root = '%s-%s' % (PACKAGE, VERSION)
243 seen = set()
244 dirmap = []
245 festout = StringIO()
246 for dir in dirs_to_dump():
247 dir = dir.rstrip('/')
248 base = OS.path.basename(dir)
249 if base not in seen:
250 name = base
251 else:
252 for i in I.count():
253 name = '%s.%d' % (base, i)
254 if name not in seen: break
255 dirmap.append((dir + '/', name))
256 festout.write('%s = %s\n' % (name, dir))
257
258 ## Write a map of where things were in the filesystem. This may help a
259 ## user figure out how to deploy the thing.
260 fest = festout.getvalue()
261 ti = TAR.TarInfo(OS.path.join(root, 'MANIFEST'))
262 ti.size = len(fest)
263 ti.mtime = T.time()
264 ti.mode = 0664
265 ti.type = TAR.REGTYPE
266 uid = OS.getuid(); ti.uid, ti.uname = uid, PW.getpwuid(uid).pw_name
267 gid = OS.getgid(); ti.gid, ti.gname = gid, GR.getgrgid(gid).gr_name
268 tf.addfile(ti, fileobj = StringIO(fest))
269
270 ## Now actually dump all of the individual directories.
271 for dir, name in dirmap:
272 dump_dir(name, dir, dirmap, tf, root)
273
274 ## We're done.
275 tf.close()
276
277###----- That's all, folks --------------------------------------------------