chiark / gitweb /
httpauth.py: Don't crash if Base-64 decoding of the CSRF token fails.
[chopwood] / agpl.py
CommitLineData
a2916c06
MW
1### -*-python-*-
2###
3### GNU Affero General Public License compliance
4###
5### (c) 2013 Mark Wooding
6###
7
8###----- Licensing notice ---------------------------------------------------
9###
10### This file is part of Chopwood: a password-changing service.
11###
12### Chopwood is free software; you can redistribute it and/or modify
13### it under the terms of the GNU Affero General Public License as
14### published by the Free Software Foundation; either version 3 of the
15### License, or (at your option) any later version.
16###
17### Chopwood is distributed in the hope that it will be useful,
18### but WITHOUT ANY WARRANTY; without even the implied warranty of
19### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20### GNU Affero General Public License for more details.
21###
22### You should have received a copy of the GNU Affero General Public
23### License along with Chopwood; if not, see
24### <http://www.gnu.org/licenses/>.
25
26import contextlib as CTX
09ba568f 27import grp as GR
a2916c06 28import os as OS
09ba568f 29import pwd as PW
a2916c06
MW
30import shlex as SL
31import shutil as SH
32import subprocess as SUB
33import sys as SYS
34import tarfile as TAR
35import tempfile as TF
09ba568f
MW
36import time as T
37
38from cStringIO import StringIO
a2916c06
MW
39
40from auto import PACKAGE, VERSION
41import util as U
42
9424c482
MW
43###--------------------------------------------------------------------------
44### Initial utilities.
45
a2916c06
MW
46@CTX.contextmanager
47def tempdir():
9424c482
MW
48 """
49 Context manager: create and return the name of a temporary directory.
50
51 The directory will be deleted automatically on exit from the body.
52 """
a2916c06
MW
53 d = TF.mkdtemp()
54 try: yield d
55 finally: SH.rmtree(d, ignore_errors = True)
56
9424c482
MW
57###--------------------------------------------------------------------------
58### Determining which files to include.
59
a2916c06 60def dirs_to_dump():
9424c482
MW
61 """
62 Return a list of directories containing used Python modules.
63
64 Directories under `/usr/' but outside `/usr/local/' are excluded, since
65 they are assumed to be covered by the AGPL exception for parts of the
66 operating system.
67 """
68
69 ## Collect a set of directories.
a2916c06 70 dirs = set()
9424c482
MW
71
72 ## Work through the list of known modules, adding them to the list.
a2916c06
MW
73 for m in SYS.modules.itervalues():
74 try: f = m.__file__
75 except AttributeError: continue
76 d = OS.path.realpath(OS.path.dirname(f))
77 if d.startswith('/usr/') and not d.startswith('/usr/local/'): continue
78 dirs.add(d)
9424c482
MW
79
80 ## Now go through the directories again, and remove any which are wholly
81 ## included within other entries.
a2916c06
MW
82 dirs = sorted(dirs)
83 last = '!'
84 dump = []
85 for d in dirs:
86 if d.startswith(last): continue
87 dump.append(d)
88 last = d
9424c482
MW
89
90 ## We're done: return the filtered list.
a2916c06
MW
91 return dump
92
9424c482
MW
93### The `DUMPERS' list consists of (PREDICATE, LISTERS) pairs. The PREDICATE
94### is a function of one argument -- a directory name -- which returns true
95### if the LISTERS should be used to enumerate that directory. The LISTERS
96### are a list of functions of one argument -- again, the directory name --
97### which should return an iterable of files within that directory, relative
98### to its top-level. Lister functions should not return the root directory,
99### because it should obviously only be included once. Instead, the root is
100### handled separately by `dump_dir'.
101
a2916c06 102def exists_subdir(subdir):
9424c482
MW
103 """
104 Predicate for `DUMPERS': match if the directory has a subdirectory SUBDIR.
105
106 This is mainly useful for detecting working trees subject to version
107 control.
108 """
a2916c06
MW
109 return lambda dir: OS.path.isdir(OS.path.join(dir, subdir))
110
111def filez(cmd):
9424c482
MW
112 """
113 Lister for `DUMPERS': generate the null-terminated items output by CMD.
114
115 Run CMD, a string containing words with shell-like quoting (expected to be
116 a literal in the code, so security concerns don't arise) in the directory
117 of interest, yielding the invidual null-terminated strings which the
118 command writes to its standard output.
119 """
a2916c06 120 def _(dir):
9424c482
MW
121
122 ## Start the command,
a2916c06 123 kid = SUB.Popen(SL.split(cmd), stdout = SUB.PIPE, cwd = dir)
9424c482
MW
124
125 ## Collect and return the null-terminated items. Strip off any leading
126 ## `./' and exclude the root directory because that gets handled
127 ## separately.
a2916c06
MW
128 left = ''
129 while True:
9424c482
MW
130
131 ## Read a new bufferload of stuff. If there's nothing left then we're
132 ## done.
a2916c06
MW
133 buf = kid.stdout.read(16384)
134 if not buf: break
9424c482
MW
135
136 ## Tack whatever was left over from last time on the front, and carve
137 ## into null-terminated pieces.
a2916c06
MW
138 buf = left + buf
139 i = 0
140 while True:
141 z = buf.find('\0', i)
142 if z < 0: break
143 f = buf[i:z]
4338f962 144 i = z + 1
fde8f8a7 145 if f.rstrip('/') == '.': continue
a2916c06
MW
146 if f.startswith('./'): f = f[2:]
147 yield f
9424c482
MW
148
149 ## Whatever's left over will be dealt with next time through.
a2916c06 150 left = buf[i:]
9424c482 151
16e57747
MW
152 ## Make sure the command actually completed successfully.
153 if kid.wait():
154 rc = kid.returncode
155 raise U.ExpectedError, \
156 (500, "lister command `%s' failed (%s) in `%s'" % (
157 cmd,
158 (rc & 0xff00) and 'rc = %d' % (rc >> 8) or 'signal %d' % rc,
159 dir))
160
9424c482 161 ## If there's trailing junk left over then we should complain.
a2916c06
MW
162 if left:
163 raise U.ExpectedError, \
164 (500, "trailing junk from `%s' in `%s'" % (cmd, dir))
9424c482
MW
165
166 ## Return the listing function.
a2916c06
MW
167 return _
168
9424c482 169## The list of predicates and listers.
a2916c06
MW
170DUMPERS = [
171 (exists_subdir('.git'), [filez('git ls-files -coz --exclude-standard'),
172 filez('find .git -print0')]),
173 (lambda d: True, [filez('find . ( ! -perm +004 -prune ) -o -print0')])]
174
9424c482
MW
175###--------------------------------------------------------------------------
176### Actually dumping files.
177
4338f962 178def dump_dir(name, dir, dirmap, tf, root):
9424c482
MW
179 """
180 Add the contents of directory DIR to the tarfile TF, under the given NAME.
181
182 The ROOT names the toplevel of the tarball (we're not in the business of
183 making tarbombs here). The DIRMAP is a list of all of the (DIR, NAME)
184 pairs being dumped, used for fixing up symbolic links between directories.
185 """
186
187 ## Find an appropriate `DUMPERS' list entry.
a2916c06
MW
188 for test, listers in DUMPERS:
189 if test(dir): break
190 else:
191 raise U.ExpectedError, (500, "no dumper for `%s'" % dir)
9424c482
MW
192
193 ## Write a tarfile entry for the toplevel.
4338f962 194 tf.add(dir, OS.path.join(root, name), recursive = False)
9424c482
MW
195
196 ## Work through all of the listers.
a2916c06 197 for lister in listers:
9424c482
MW
198
199 ## Work through each file.
a2916c06 200 for file in lister(dir):
47752597
MW
201 with U.Escape() as skip:
202 full = OS.path.join(dir, file)
203 tarname = OS.path.join(root, name, file)
204
205 ## Check for symbolic links. If we find one that points to another
206 ## directory we're going to dump separately then fiddle it so that it
207 ## works in the directory tree we're going to make.
208 if OS.path.islink(full):
209 dest = OS.path.realpath(full)
210 for d, local in dirmap:
211 if dest.startswith(d):
212 fix = OS.path.relpath(OS.path.join('/', local, dest[len(d):]),
213 OS.path.join('/', name,
214 OS.path.dirname(file)))
215 st = OS.stat(full)
216 ti = tf.gettarinfo(full, tarname)
217 ti.linkname = fix
218 tf.addfile(ti)
219 skip()
220
221 ## Nothing special, so just dump the file. Or whatever it is.
5dea0052 222 tf.add(full, tarname, recursive = False)
a2916c06
MW
223
224def source(out):
9424c482
MW
225 """
226 Write a tarball for the program's source code to OUT.
227
228 This function automatically dumps all of the program's dependencies except
229 for those covered by the operating-system exemption.
230 """
231
232 ## Make a tarfile writer. There's an annoying incompatibility to bodge
233 ## around.
a2916c06
MW
234 if SYS.version_info >= (2, 6):
235 tf = TAR.open(fileobj = out, mode = 'w|gz', format = TAR.USTAR_FORMAT)
236 else:
237 tf = TAR.open(fileobj = out, mode = 'w|gz')
238 tf.posix = True
9424c482
MW
239
240 ## First of all, find out what needs to be dumped, and assign names to all
241 ## of the various directories.
09ba568f
MW
242 root = '%s-%s' % (PACKAGE, VERSION)
243 seen = set()
244 dirmap = []
245 festout = StringIO()
246 for dir in dirs_to_dump():
247 dir = dir.rstrip('/')
248 base = OS.path.basename(dir)
249 if base not in seen:
250 name = base
251 else:
252 for i in I.count():
253 name = '%s.%d' % (base, i)
254 if name not in seen: break
255 dirmap.append((dir + '/', name))
256 festout.write('%s = %s\n' % (name, dir))
9424c482
MW
257
258 ## Write a map of where things were in the filesystem. This may help a
259 ## user figure out how to deploy the thing.
09ba568f
MW
260 fest = festout.getvalue()
261 ti = TAR.TarInfo(OS.path.join(root, 'MANIFEST'))
262 ti.size = len(fest)
263 ti.mtime = T.time()
264 ti.mode = 0664
265 ti.type = TAR.REGTYPE
266 uid = OS.getuid(); ti.uid, ti.uname = uid, PW.getpwuid(uid).pw_name
267 gid = OS.getgid(); ti.gid, ti.gname = gid, GR.getgrgid(gid).gr_name
268 tf.addfile(ti, fileobj = StringIO(fest))
9424c482
MW
269
270 ## Now actually dump all of the individual directories.
09ba568f 271 for dir, name in dirmap:
4338f962 272 dump_dir(name, dir, dirmap, tf, root)
9424c482
MW
273
274 ## We're done.
a2916c06
MW
275 tf.close()
276
277###----- That's all, folks --------------------------------------------------