Commit | Line | Data |
---|---|---|
a2916c06 MW |
1 | ### -*-python-*- |
2 | ### | |
3 | ### GNU Affero General Public License compliance | |
4 | ### | |
5 | ### (c) 2013 Mark Wooding | |
6 | ### | |
7 | ||
8 | ###----- Licensing notice --------------------------------------------------- | |
9 | ### | |
10 | ### This file is part of Chopwood: a password-changing service. | |
11 | ### | |
12 | ### Chopwood is free software; you can redistribute it and/or modify | |
13 | ### it under the terms of the GNU Affero General Public License as | |
14 | ### published by the Free Software Foundation; either version 3 of the | |
15 | ### License, or (at your option) any later version. | |
16 | ### | |
17 | ### Chopwood is distributed in the hope that it will be useful, | |
18 | ### but WITHOUT ANY WARRANTY; without even the implied warranty of | |
19 | ### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
20 | ### GNU Affero General Public License for more details. | |
21 | ### | |
22 | ### You should have received a copy of the GNU Affero General Public | |
23 | ### License along with Chopwood; if not, see | |
24 | ### <http://www.gnu.org/licenses/>. | |
25 | ||
26 | import contextlib as CTX | |
09ba568f | 27 | import grp as GR |
a2916c06 | 28 | import os as OS |
09ba568f | 29 | import pwd as PW |
a2916c06 MW |
30 | import shlex as SL |
31 | import shutil as SH | |
32 | import subprocess as SUB | |
33 | import sys as SYS | |
34 | import tarfile as TAR | |
35 | import tempfile as TF | |
09ba568f MW |
36 | import time as T |
37 | ||
38 | from cStringIO import StringIO | |
a2916c06 MW |
39 | |
40 | from auto import PACKAGE, VERSION | |
41 | import util as U | |
42 | ||
9424c482 MW |
43 | ###-------------------------------------------------------------------------- |
44 | ### Initial utilities. | |
45 | ||
a2916c06 MW |
46 | @CTX.contextmanager |
47 | def tempdir(): | |
9424c482 MW |
48 | """ |
49 | Context manager: create and return the name of a temporary directory. | |
50 | ||
51 | The directory will be deleted automatically on exit from the body. | |
52 | """ | |
a2916c06 MW |
53 | d = TF.mkdtemp() |
54 | try: yield d | |
55 | finally: SH.rmtree(d, ignore_errors = True) | |
56 | ||
9424c482 MW |
57 | ###-------------------------------------------------------------------------- |
58 | ### Determining which files to include. | |
59 | ||
a2916c06 | 60 | def dirs_to_dump(): |
9424c482 MW |
61 | """ |
62 | Return a list of directories containing used Python modules. | |
63 | ||
64 | Directories under `/usr/' but outside `/usr/local/' are excluded, since | |
65 | they are assumed to be covered by the AGPL exception for parts of the | |
66 | operating system. | |
67 | """ | |
68 | ||
69 | ## Collect a set of directories. | |
a2916c06 | 70 | dirs = set() |
9424c482 MW |
71 | |
72 | ## Work through the list of known modules, adding them to the list. | |
a2916c06 MW |
73 | for m in SYS.modules.itervalues(): |
74 | try: f = m.__file__ | |
75 | except AttributeError: continue | |
76 | d = OS.path.realpath(OS.path.dirname(f)) | |
77 | if d.startswith('/usr/') and not d.startswith('/usr/local/'): continue | |
78 | dirs.add(d) | |
9424c482 MW |
79 | |
80 | ## Now go through the directories again, and remove any which are wholly | |
81 | ## included within other entries. | |
a2916c06 MW |
82 | dirs = sorted(dirs) |
83 | last = '!' | |
84 | dump = [] | |
85 | for d in dirs: | |
86 | if d.startswith(last): continue | |
87 | dump.append(d) | |
88 | last = d | |
9424c482 MW |
89 | |
90 | ## We're done: return the filtered list. | |
a2916c06 MW |
91 | return dump |
92 | ||
9424c482 MW |
93 | ### The `DUMPERS' list consists of (PREDICATE, LISTERS) pairs. The PREDICATE |
94 | ### is a function of one argument -- a directory name -- which returns true | |
95 | ### if the LISTERS should be used to enumerate that directory. The LISTERS | |
96 | ### are a list of functions of one argument -- again, the directory name -- | |
97 | ### which should return an iterable of files within that directory, relative | |
98 | ### to its top-level. Lister functions should not return the root directory, | |
99 | ### because it should obviously only be included once. Instead, the root is | |
100 | ### handled separately by `dump_dir'. | |
101 | ||
a2916c06 | 102 | def exists_subdir(subdir): |
9424c482 MW |
103 | """ |
104 | Predicate for `DUMPERS': match if the directory has a subdirectory SUBDIR. | |
105 | ||
106 | This is mainly useful for detecting working trees subject to version | |
107 | control. | |
108 | """ | |
a2916c06 MW |
109 | return lambda dir: OS.path.isdir(OS.path.join(dir, subdir)) |
110 | ||
111 | def filez(cmd): | |
9424c482 MW |
112 | """ |
113 | Lister for `DUMPERS': generate the null-terminated items output by CMD. | |
114 | ||
115 | Run CMD, a string containing words with shell-like quoting (expected to be | |
116 | a literal in the code, so security concerns don't arise) in the directory | |
117 | of interest, yielding the invidual null-terminated strings which the | |
118 | command writes to its standard output. | |
119 | """ | |
a2916c06 | 120 | def _(dir): |
9424c482 MW |
121 | |
122 | ## Start the command, | |
a2916c06 | 123 | kid = SUB.Popen(SL.split(cmd), stdout = SUB.PIPE, cwd = dir) |
9424c482 MW |
124 | |
125 | ## Collect and return the null-terminated items. Strip off any leading | |
126 | ## `./' and exclude the root directory because that gets handled | |
127 | ## separately. | |
a2916c06 MW |
128 | left = '' |
129 | while True: | |
9424c482 MW |
130 | |
131 | ## Read a new bufferload of stuff. If there's nothing left then we're | |
132 | ## done. | |
a2916c06 MW |
133 | buf = kid.stdout.read(16384) |
134 | if not buf: break | |
9424c482 MW |
135 | |
136 | ## Tack whatever was left over from last time on the front, and carve | |
137 | ## into null-terminated pieces. | |
a2916c06 MW |
138 | buf = left + buf |
139 | i = 0 | |
140 | while True: | |
141 | z = buf.find('\0', i) | |
142 | if z < 0: break | |
143 | f = buf[i:z] | |
4338f962 | 144 | i = z + 1 |
fde8f8a7 | 145 | if f.rstrip('/') == '.': continue |
a2916c06 MW |
146 | if f.startswith('./'): f = f[2:] |
147 | yield f | |
9424c482 MW |
148 | |
149 | ## Whatever's left over will be dealt with next time through. | |
a2916c06 | 150 | left = buf[i:] |
9424c482 | 151 | |
16e57747 MW |
152 | ## Make sure the command actually completed successfully. |
153 | if kid.wait(): | |
154 | rc = kid.returncode | |
155 | raise U.ExpectedError, \ | |
156 | (500, "lister command `%s' failed (%s) in `%s'" % ( | |
157 | cmd, | |
158 | (rc & 0xff00) and 'rc = %d' % (rc >> 8) or 'signal %d' % rc, | |
159 | dir)) | |
160 | ||
9424c482 | 161 | ## If there's trailing junk left over then we should complain. |
a2916c06 MW |
162 | if left: |
163 | raise U.ExpectedError, \ | |
164 | (500, "trailing junk from `%s' in `%s'" % (cmd, dir)) | |
9424c482 MW |
165 | |
166 | ## Return the listing function. | |
a2916c06 MW |
167 | return _ |
168 | ||
9424c482 | 169 | ## The list of predicates and listers. |
a2916c06 MW |
170 | DUMPERS = [ |
171 | (exists_subdir('.git'), [filez('git ls-files -coz --exclude-standard'), | |
172 | filez('find .git -print0')]), | |
173 | (lambda d: True, [filez('find . ( ! -perm +004 -prune ) -o -print0')])] | |
174 | ||
9424c482 MW |
175 | ###-------------------------------------------------------------------------- |
176 | ### Actually dumping files. | |
177 | ||
4338f962 | 178 | def dump_dir(name, dir, dirmap, tf, root): |
9424c482 MW |
179 | """ |
180 | Add the contents of directory DIR to the tarfile TF, under the given NAME. | |
181 | ||
182 | The ROOT names the toplevel of the tarball (we're not in the business of | |
183 | making tarbombs here). The DIRMAP is a list of all of the (DIR, NAME) | |
184 | pairs being dumped, used for fixing up symbolic links between directories. | |
185 | """ | |
186 | ||
187 | ## Find an appropriate `DUMPERS' list entry. | |
a2916c06 MW |
188 | for test, listers in DUMPERS: |
189 | if test(dir): break | |
190 | else: | |
191 | raise U.ExpectedError, (500, "no dumper for `%s'" % dir) | |
9424c482 MW |
192 | |
193 | ## Write a tarfile entry for the toplevel. | |
4338f962 | 194 | tf.add(dir, OS.path.join(root, name), recursive = False) |
9424c482 MW |
195 | |
196 | ## Work through all of the listers. | |
a2916c06 | 197 | for lister in listers: |
9424c482 MW |
198 | |
199 | ## Work through each file. | |
a2916c06 | 200 | for file in lister(dir): |
47752597 MW |
201 | with U.Escape() as skip: |
202 | full = OS.path.join(dir, file) | |
203 | tarname = OS.path.join(root, name, file) | |
204 | ||
205 | ## Check for symbolic links. If we find one that points to another | |
206 | ## directory we're going to dump separately then fiddle it so that it | |
207 | ## works in the directory tree we're going to make. | |
208 | if OS.path.islink(full): | |
209 | dest = OS.path.realpath(full) | |
210 | for d, local in dirmap: | |
211 | if dest.startswith(d): | |
212 | fix = OS.path.relpath(OS.path.join('/', local, dest[len(d):]), | |
213 | OS.path.join('/', name, | |
214 | OS.path.dirname(file))) | |
215 | st = OS.stat(full) | |
216 | ti = tf.gettarinfo(full, tarname) | |
217 | ti.linkname = fix | |
218 | tf.addfile(ti) | |
219 | skip() | |
220 | ||
221 | ## Nothing special, so just dump the file. Or whatever it is. | |
5dea0052 | 222 | tf.add(full, tarname, recursive = False) |
a2916c06 MW |
223 | |
224 | def source(out): | |
9424c482 MW |
225 | """ |
226 | Write a tarball for the program's source code to OUT. | |
227 | ||
228 | This function automatically dumps all of the program's dependencies except | |
229 | for those covered by the operating-system exemption. | |
230 | """ | |
231 | ||
232 | ## Make a tarfile writer. There's an annoying incompatibility to bodge | |
233 | ## around. | |
a2916c06 MW |
234 | if SYS.version_info >= (2, 6): |
235 | tf = TAR.open(fileobj = out, mode = 'w|gz', format = TAR.USTAR_FORMAT) | |
236 | else: | |
237 | tf = TAR.open(fileobj = out, mode = 'w|gz') | |
238 | tf.posix = True | |
9424c482 MW |
239 | |
240 | ## First of all, find out what needs to be dumped, and assign names to all | |
241 | ## of the various directories. | |
09ba568f MW |
242 | root = '%s-%s' % (PACKAGE, VERSION) |
243 | seen = set() | |
244 | dirmap = [] | |
245 | festout = StringIO() | |
246 | for dir in dirs_to_dump(): | |
247 | dir = dir.rstrip('/') | |
248 | base = OS.path.basename(dir) | |
249 | if base not in seen: | |
250 | name = base | |
251 | else: | |
252 | for i in I.count(): | |
253 | name = '%s.%d' % (base, i) | |
254 | if name not in seen: break | |
255 | dirmap.append((dir + '/', name)) | |
256 | festout.write('%s = %s\n' % (name, dir)) | |
9424c482 MW |
257 | |
258 | ## Write a map of where things were in the filesystem. This may help a | |
259 | ## user figure out how to deploy the thing. | |
09ba568f MW |
260 | fest = festout.getvalue() |
261 | ti = TAR.TarInfo(OS.path.join(root, 'MANIFEST')) | |
262 | ti.size = len(fest) | |
263 | ti.mtime = T.time() | |
264 | ti.mode = 0664 | |
265 | ti.type = TAR.REGTYPE | |
266 | uid = OS.getuid(); ti.uid, ti.uname = uid, PW.getpwuid(uid).pw_name | |
267 | gid = OS.getgid(); ti.gid, ti.gname = gid, GR.getgrgid(gid).gr_name | |
268 | tf.addfile(ti, fileobj = StringIO(fest)) | |
9424c482 MW |
269 | |
270 | ## Now actually dump all of the individual directories. | |
09ba568f | 271 | for dir, name in dirmap: |
4338f962 | 272 | dump_dir(name, dir, dirmap, tf, root) |
9424c482 MW |
273 | |
274 | ## We're done. | |
a2916c06 MW |
275 | tf.close() |
276 | ||
277 | ###----- That's all, folks -------------------------------------------------- |