Commit | Line | Data |
---|---|---|
a2916c06 MW |
1 | ### -*-python-*- |
2 | ### | |
3 | ### GNU Affero General Public License compliance | |
4 | ### | |
5 | ### (c) 2013 Mark Wooding | |
6 | ### | |
7 | ||
8 | ###----- Licensing notice --------------------------------------------------- | |
9 | ### | |
10 | ### This file is part of Chopwood: a password-changing service. | |
11 | ### | |
12 | ### Chopwood is free software; you can redistribute it and/or modify | |
13 | ### it under the terms of the GNU Affero General Public License as | |
14 | ### published by the Free Software Foundation; either version 3 of the | |
15 | ### License, or (at your option) any later version. | |
16 | ### | |
17 | ### Chopwood is distributed in the hope that it will be useful, | |
18 | ### but WITHOUT ANY WARRANTY; without even the implied warranty of | |
19 | ### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
20 | ### GNU Affero General Public License for more details. | |
21 | ### | |
22 | ### You should have received a copy of the GNU Affero General Public | |
23 | ### License along with Chopwood; if not, see | |
24 | ### <http://www.gnu.org/licenses/>. | |
25 | ||
0c58273e MW |
26 | from __future__ import with_statement |
27 | ||
a2916c06 | 28 | import contextlib as CTX |
09ba568f | 29 | import grp as GR |
a2916c06 | 30 | import os as OS |
09ba568f | 31 | import pwd as PW |
a2916c06 MW |
32 | import shlex as SL |
33 | import shutil as SH | |
34 | import subprocess as SUB | |
35 | import sys as SYS | |
36 | import tarfile as TAR | |
37 | import tempfile as TF | |
09ba568f MW |
38 | import time as T |
39 | ||
40 | from cStringIO import StringIO | |
a2916c06 MW |
41 | |
42 | from auto import PACKAGE, VERSION | |
43 | import util as U | |
44 | ||
9424c482 MW |
45 | ###-------------------------------------------------------------------------- |
46 | ### Initial utilities. | |
47 | ||
a2916c06 MW |
48 | @CTX.contextmanager |
49 | def tempdir(): | |
9424c482 MW |
50 | """ |
51 | Context manager: create and return the name of a temporary directory. | |
52 | ||
53 | The directory will be deleted automatically on exit from the body. | |
54 | """ | |
a2916c06 MW |
55 | d = TF.mkdtemp() |
56 | try: yield d | |
57 | finally: SH.rmtree(d, ignore_errors = True) | |
58 | ||
9424c482 MW |
59 | ###-------------------------------------------------------------------------- |
60 | ### Determining which files to include. | |
61 | ||
a2916c06 | 62 | def dirs_to_dump(): |
9424c482 MW |
63 | """ |
64 | Return a list of directories containing used Python modules. | |
65 | ||
66 | Directories under `/usr/' but outside `/usr/local/' are excluded, since | |
67 | they are assumed to be covered by the AGPL exception for parts of the | |
68 | operating system. | |
69 | """ | |
70 | ||
71 | ## Collect a set of directories. | |
a2916c06 | 72 | dirs = set() |
9424c482 MW |
73 | |
74 | ## Work through the list of known modules, adding them to the list. | |
a2916c06 MW |
75 | for m in SYS.modules.itervalues(): |
76 | try: f = m.__file__ | |
77 | except AttributeError: continue | |
78 | d = OS.path.realpath(OS.path.dirname(f)) | |
79 | if d.startswith('/usr/') and not d.startswith('/usr/local/'): continue | |
80 | dirs.add(d) | |
9424c482 MW |
81 | |
82 | ## Now go through the directories again, and remove any which are wholly | |
83 | ## included within other entries. | |
a2916c06 MW |
84 | dirs = sorted(dirs) |
85 | last = '!' | |
86 | dump = [] | |
87 | for d in dirs: | |
88 | if d.startswith(last): continue | |
89 | dump.append(d) | |
90 | last = d | |
9424c482 MW |
91 | |
92 | ## We're done: return the filtered list. | |
a2916c06 MW |
93 | return dump |
94 | ||
9424c482 MW |
95 | ### The `DUMPERS' list consists of (PREDICATE, LISTERS) pairs. The PREDICATE |
96 | ### is a function of one argument -- a directory name -- which returns true | |
97 | ### if the LISTERS should be used to enumerate that directory. The LISTERS | |
98 | ### are a list of functions of one argument -- again, the directory name -- | |
99 | ### which should return an iterable of files within that directory, relative | |
100 | ### to its top-level. Lister functions should not return the root directory, | |
101 | ### because it should obviously only be included once. Instead, the root is | |
102 | ### handled separately by `dump_dir'. | |
103 | ||
a2916c06 | 104 | def exists_subdir(subdir): |
9424c482 MW |
105 | """ |
106 | Predicate for `DUMPERS': match if the directory has a subdirectory SUBDIR. | |
107 | ||
108 | This is mainly useful for detecting working trees subject to version | |
109 | control. | |
110 | """ | |
a2916c06 MW |
111 | return lambda dir: OS.path.isdir(OS.path.join(dir, subdir)) |
112 | ||
113 | def filez(cmd): | |
9424c482 MW |
114 | """ |
115 | Lister for `DUMPERS': generate the null-terminated items output by CMD. | |
116 | ||
117 | Run CMD, a string containing words with shell-like quoting (expected to be | |
118 | a literal in the code, so security concerns don't arise) in the directory | |
119 | of interest, yielding the invidual null-terminated strings which the | |
120 | command writes to its standard output. | |
121 | """ | |
a2916c06 | 122 | def _(dir): |
9424c482 MW |
123 | |
124 | ## Start the command, | |
a2916c06 | 125 | kid = SUB.Popen(SL.split(cmd), stdout = SUB.PIPE, cwd = dir) |
9424c482 MW |
126 | |
127 | ## Collect and return the null-terminated items. Strip off any leading | |
128 | ## `./' and exclude the root directory because that gets handled | |
129 | ## separately. | |
a2916c06 MW |
130 | left = '' |
131 | while True: | |
9424c482 MW |
132 | |
133 | ## Read a new bufferload of stuff. If there's nothing left then we're | |
134 | ## done. | |
a2916c06 MW |
135 | buf = kid.stdout.read(16384) |
136 | if not buf: break | |
9424c482 MW |
137 | |
138 | ## Tack whatever was left over from last time on the front, and carve | |
139 | ## into null-terminated pieces. | |
a2916c06 MW |
140 | buf = left + buf |
141 | i = 0 | |
142 | while True: | |
143 | z = buf.find('\0', i) | |
144 | if z < 0: break | |
145 | f = buf[i:z] | |
4338f962 | 146 | i = z + 1 |
fde8f8a7 | 147 | if f.rstrip('/') == '.': continue |
a2916c06 MW |
148 | if f.startswith('./'): f = f[2:] |
149 | yield f | |
9424c482 MW |
150 | |
151 | ## Whatever's left over will be dealt with next time through. | |
a2916c06 | 152 | left = buf[i:] |
9424c482 | 153 | |
16e57747 MW |
154 | ## Make sure the command actually completed successfully. |
155 | if kid.wait(): | |
156 | rc = kid.returncode | |
157 | raise U.ExpectedError, \ | |
158 | (500, "lister command `%s' failed (%s) in `%s'" % ( | |
159 | cmd, | |
160 | (rc & 0xff00) and 'rc = %d' % (rc >> 8) or 'signal %d' % rc, | |
161 | dir)) | |
162 | ||
9424c482 | 163 | ## If there's trailing junk left over then we should complain. |
a2916c06 MW |
164 | if left: |
165 | raise U.ExpectedError, \ | |
166 | (500, "trailing junk from `%s' in `%s'" % (cmd, dir)) | |
9424c482 MW |
167 | |
168 | ## Return the listing function. | |
a2916c06 MW |
169 | return _ |
170 | ||
9424c482 | 171 | ## The list of predicates and listers. |
a2916c06 MW |
172 | DUMPERS = [ |
173 | (exists_subdir('.git'), [filez('git ls-files -coz --exclude-standard'), | |
174 | filez('find .git -print0')]), | |
175 | (lambda d: True, [filez('find . ( ! -perm +004 -prune ) -o -print0')])] | |
176 | ||
9424c482 MW |
177 | ###-------------------------------------------------------------------------- |
178 | ### Actually dumping files. | |
179 | ||
4338f962 | 180 | def dump_dir(name, dir, dirmap, tf, root): |
9424c482 MW |
181 | """ |
182 | Add the contents of directory DIR to the tarfile TF, under the given NAME. | |
183 | ||
184 | The ROOT names the toplevel of the tarball (we're not in the business of | |
185 | making tarbombs here). The DIRMAP is a list of all of the (DIR, NAME) | |
186 | pairs being dumped, used for fixing up symbolic links between directories. | |
187 | """ | |
188 | ||
189 | ## Find an appropriate `DUMPERS' list entry. | |
a2916c06 MW |
190 | for test, listers in DUMPERS: |
191 | if test(dir): break | |
192 | else: | |
193 | raise U.ExpectedError, (500, "no dumper for `%s'" % dir) | |
9424c482 MW |
194 | |
195 | ## Write a tarfile entry for the toplevel. | |
4338f962 | 196 | tf.add(dir, OS.path.join(root, name), recursive = False) |
9424c482 MW |
197 | |
198 | ## Work through all of the listers. | |
a2916c06 | 199 | for lister in listers: |
9424c482 MW |
200 | |
201 | ## Work through each file. | |
a2916c06 | 202 | for file in lister(dir): |
47752597 MW |
203 | with U.Escape() as skip: |
204 | full = OS.path.join(dir, file) | |
205 | tarname = OS.path.join(root, name, file) | |
206 | ||
207 | ## Check for symbolic links. If we find one that points to another | |
208 | ## directory we're going to dump separately then fiddle it so that it | |
209 | ## works in the directory tree we're going to make. | |
210 | if OS.path.islink(full): | |
211 | dest = OS.path.realpath(full) | |
212 | for d, local in dirmap: | |
213 | if dest.startswith(d): | |
214 | fix = OS.path.relpath(OS.path.join('/', local, dest[len(d):]), | |
215 | OS.path.join('/', name, | |
216 | OS.path.dirname(file))) | |
217 | st = OS.stat(full) | |
218 | ti = tf.gettarinfo(full, tarname) | |
219 | ti.linkname = fix | |
220 | tf.addfile(ti) | |
221 | skip() | |
222 | ||
223 | ## Nothing special, so just dump the file. Or whatever it is. | |
5dea0052 | 224 | tf.add(full, tarname, recursive = False) |
a2916c06 MW |
225 | |
226 | def source(out): | |
9424c482 MW |
227 | """ |
228 | Write a tarball for the program's source code to OUT. | |
229 | ||
230 | This function automatically dumps all of the program's dependencies except | |
231 | for those covered by the operating-system exemption. | |
232 | """ | |
233 | ||
234 | ## Make a tarfile writer. There's an annoying incompatibility to bodge | |
235 | ## around. | |
a2916c06 MW |
236 | if SYS.version_info >= (2, 6): |
237 | tf = TAR.open(fileobj = out, mode = 'w|gz', format = TAR.USTAR_FORMAT) | |
238 | else: | |
239 | tf = TAR.open(fileobj = out, mode = 'w|gz') | |
240 | tf.posix = True | |
9424c482 MW |
241 | |
242 | ## First of all, find out what needs to be dumped, and assign names to all | |
243 | ## of the various directories. | |
09ba568f MW |
244 | root = '%s-%s' % (PACKAGE, VERSION) |
245 | seen = set() | |
246 | dirmap = [] | |
247 | festout = StringIO() | |
248 | for dir in dirs_to_dump(): | |
249 | dir = dir.rstrip('/') | |
250 | base = OS.path.basename(dir) | |
251 | if base not in seen: | |
252 | name = base | |
253 | else: | |
254 | for i in I.count(): | |
255 | name = '%s.%d' % (base, i) | |
256 | if name not in seen: break | |
257 | dirmap.append((dir + '/', name)) | |
258 | festout.write('%s = %s\n' % (name, dir)) | |
9424c482 MW |
259 | |
260 | ## Write a map of where things were in the filesystem. This may help a | |
261 | ## user figure out how to deploy the thing. | |
09ba568f MW |
262 | fest = festout.getvalue() |
263 | ti = TAR.TarInfo(OS.path.join(root, 'MANIFEST')) | |
264 | ti.size = len(fest) | |
265 | ti.mtime = T.time() | |
266 | ti.mode = 0664 | |
267 | ti.type = TAR.REGTYPE | |
268 | uid = OS.getuid(); ti.uid, ti.uname = uid, PW.getpwuid(uid).pw_name | |
269 | gid = OS.getgid(); ti.gid, ti.gname = gid, GR.getgrgid(gid).gr_name | |
270 | tf.addfile(ti, fileobj = StringIO(fest)) | |
9424c482 MW |
271 | |
272 | ## Now actually dump all of the individual directories. | |
09ba568f | 273 | for dir, name in dirmap: |
4338f962 | 274 | dump_dir(name, dir, dirmap, tf, root) |
9424c482 MW |
275 | |
276 | ## We're done. | |
a2916c06 MW |
277 | tf.close() |
278 | ||
279 | ###----- That's all, folks -------------------------------------------------- |