Commit | Line | Data |
---|---|---|
a2916c06 MW |
1 | ### -*-python-*- |
2 | ### | |
3 | ### GNU Affero General Public License compliance | |
4 | ### | |
5 | ### (c) 2013 Mark Wooding | |
6 | ### | |
7 | ||
8 | ###----- Licensing notice --------------------------------------------------- | |
9 | ### | |
10 | ### This file is part of Chopwood: a password-changing service. | |
11 | ### | |
12 | ### Chopwood is free software; you can redistribute it and/or modify | |
13 | ### it under the terms of the GNU Affero General Public License as | |
14 | ### published by the Free Software Foundation; either version 3 of the | |
15 | ### License, or (at your option) any later version. | |
16 | ### | |
17 | ### Chopwood is distributed in the hope that it will be useful, | |
18 | ### but WITHOUT ANY WARRANTY; without even the implied warranty of | |
19 | ### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
20 | ### GNU Affero General Public License for more details. | |
21 | ### | |
22 | ### You should have received a copy of the GNU Affero General Public | |
23 | ### License along with Chopwood; if not, see | |
24 | ### <http://www.gnu.org/licenses/>. | |
25 | ||
26 | import contextlib as CTX | |
09ba568f | 27 | import grp as GR |
a2916c06 | 28 | import os as OS |
09ba568f | 29 | import pwd as PW |
a2916c06 MW |
30 | import shlex as SL |
31 | import shutil as SH | |
32 | import subprocess as SUB | |
33 | import sys as SYS | |
34 | import tarfile as TAR | |
35 | import tempfile as TF | |
09ba568f MW |
36 | import time as T |
37 | ||
38 | from cStringIO import StringIO | |
a2916c06 MW |
39 | |
40 | from auto import PACKAGE, VERSION | |
41 | import util as U | |
42 | ||
9424c482 MW |
43 | ###-------------------------------------------------------------------------- |
44 | ### Initial utilities. | |
45 | ||
a2916c06 MW |
46 | @CTX.contextmanager |
47 | def tempdir(): | |
9424c482 MW |
48 | """ |
49 | Context manager: create and return the name of a temporary directory. | |
50 | ||
51 | The directory will be deleted automatically on exit from the body. | |
52 | """ | |
a2916c06 MW |
53 | d = TF.mkdtemp() |
54 | try: yield d | |
55 | finally: SH.rmtree(d, ignore_errors = True) | |
56 | ||
9424c482 MW |
57 | ###-------------------------------------------------------------------------- |
58 | ### Determining which files to include. | |
59 | ||
a2916c06 | 60 | def dirs_to_dump(): |
9424c482 MW |
61 | """ |
62 | Return a list of directories containing used Python modules. | |
63 | ||
64 | Directories under `/usr/' but outside `/usr/local/' are excluded, since | |
65 | they are assumed to be covered by the AGPL exception for parts of the | |
66 | operating system. | |
67 | """ | |
68 | ||
69 | ## Collect a set of directories. | |
a2916c06 | 70 | dirs = set() |
9424c482 MW |
71 | |
72 | ## Work through the list of known modules, adding them to the list. | |
a2916c06 MW |
73 | for m in SYS.modules.itervalues(): |
74 | try: f = m.__file__ | |
75 | except AttributeError: continue | |
76 | d = OS.path.realpath(OS.path.dirname(f)) | |
77 | if d.startswith('/usr/') and not d.startswith('/usr/local/'): continue | |
78 | dirs.add(d) | |
9424c482 MW |
79 | |
80 | ## Now go through the directories again, and remove any which are wholly | |
81 | ## included within other entries. | |
a2916c06 MW |
82 | dirs = sorted(dirs) |
83 | last = '!' | |
84 | dump = [] | |
85 | for d in dirs: | |
86 | if d.startswith(last): continue | |
87 | dump.append(d) | |
88 | last = d | |
9424c482 MW |
89 | |
90 | ## We're done: return the filtered list. | |
a2916c06 MW |
91 | return dump |
92 | ||
9424c482 MW |
93 | ### The `DUMPERS' list consists of (PREDICATE, LISTERS) pairs. The PREDICATE |
94 | ### is a function of one argument -- a directory name -- which returns true | |
95 | ### if the LISTERS should be used to enumerate that directory. The LISTERS | |
96 | ### are a list of functions of one argument -- again, the directory name -- | |
97 | ### which should return an iterable of files within that directory, relative | |
98 | ### to its top-level. Lister functions should not return the root directory, | |
99 | ### because it should obviously only be included once. Instead, the root is | |
100 | ### handled separately by `dump_dir'. | |
101 | ||
a2916c06 | 102 | def exists_subdir(subdir): |
9424c482 MW |
103 | """ |
104 | Predicate for `DUMPERS': match if the directory has a subdirectory SUBDIR. | |
105 | ||
106 | This is mainly useful for detecting working trees subject to version | |
107 | control. | |
108 | """ | |
a2916c06 MW |
109 | return lambda dir: OS.path.isdir(OS.path.join(dir, subdir)) |
110 | ||
111 | def filez(cmd): | |
9424c482 MW |
112 | """ |
113 | Lister for `DUMPERS': generate the null-terminated items output by CMD. | |
114 | ||
115 | Run CMD, a string containing words with shell-like quoting (expected to be | |
116 | a literal in the code, so security concerns don't arise) in the directory | |
117 | of interest, yielding the invidual null-terminated strings which the | |
118 | command writes to its standard output. | |
119 | """ | |
a2916c06 | 120 | def _(dir): |
9424c482 MW |
121 | |
122 | ## Start the command, | |
a2916c06 | 123 | kid = SUB.Popen(SL.split(cmd), stdout = SUB.PIPE, cwd = dir) |
9424c482 MW |
124 | |
125 | ## Collect and return the null-terminated items. Strip off any leading | |
126 | ## `./' and exclude the root directory because that gets handled | |
127 | ## separately. | |
a2916c06 MW |
128 | left = '' |
129 | while True: | |
9424c482 MW |
130 | |
131 | ## Read a new bufferload of stuff. If there's nothing left then we're | |
132 | ## done. | |
a2916c06 MW |
133 | buf = kid.stdout.read(16384) |
134 | if not buf: break | |
9424c482 MW |
135 | |
136 | ## Tack whatever was left over from last time on the front, and carve | |
137 | ## into null-terminated pieces. | |
a2916c06 MW |
138 | buf = left + buf |
139 | i = 0 | |
140 | while True: | |
141 | z = buf.find('\0', i) | |
142 | if z < 0: break | |
143 | f = buf[i:z] | |
4338f962 MW |
144 | i = z + 1 |
145 | if f == '.': continue | |
a2916c06 MW |
146 | if f.startswith('./'): f = f[2:] |
147 | yield f | |
9424c482 MW |
148 | |
149 | ## Whatever's left over will be dealt with next time through. | |
a2916c06 | 150 | left = buf[i:] |
9424c482 MW |
151 | |
152 | ## If there's trailing junk left over then we should complain. | |
a2916c06 MW |
153 | if left: |
154 | raise U.ExpectedError, \ | |
155 | (500, "trailing junk from `%s' in `%s'" % (cmd, dir)) | |
9424c482 MW |
156 | |
157 | ## Return the listing function. | |
a2916c06 MW |
158 | return _ |
159 | ||
9424c482 | 160 | ## The list of predicates and listers. |
a2916c06 MW |
161 | DUMPERS = [ |
162 | (exists_subdir('.git'), [filez('git ls-files -coz --exclude-standard'), | |
163 | filez('find .git -print0')]), | |
164 | (lambda d: True, [filez('find . ( ! -perm +004 -prune ) -o -print0')])] | |
165 | ||
9424c482 MW |
166 | ###-------------------------------------------------------------------------- |
167 | ### Actually dumping files. | |
168 | ||
4338f962 | 169 | def dump_dir(name, dir, dirmap, tf, root): |
9424c482 MW |
170 | """ |
171 | Add the contents of directory DIR to the tarfile TF, under the given NAME. | |
172 | ||
173 | The ROOT names the toplevel of the tarball (we're not in the business of | |
174 | making tarbombs here). The DIRMAP is a list of all of the (DIR, NAME) | |
175 | pairs being dumped, used for fixing up symbolic links between directories. | |
176 | """ | |
177 | ||
178 | ## Find an appropriate `DUMPERS' list entry. | |
a2916c06 MW |
179 | for test, listers in DUMPERS: |
180 | if test(dir): break | |
181 | else: | |
182 | raise U.ExpectedError, (500, "no dumper for `%s'" % dir) | |
9424c482 MW |
183 | |
184 | ## Write a tarfile entry for the toplevel. | |
4338f962 | 185 | tf.add(dir, OS.path.join(root, name), recursive = False) |
9424c482 MW |
186 | |
187 | ## Work through all of the listers. | |
a2916c06 | 188 | for lister in listers: |
9424c482 MW |
189 | |
190 | ## Work through each file. | |
a2916c06 | 191 | for file in lister(dir): |
5dea0052 MW |
192 | full = OS.path.join(dir, file) |
193 | tarname = OS.path.join(root, name, file) | |
194 | skip = False | |
9424c482 MW |
195 | |
196 | ## Check for symbolic links. If we find one that points to another | |
197 | ## directory we're going to dump separately then fiddle it so that it | |
198 | ## works in the directory tree we're going to make. | |
5dea0052 MW |
199 | if OS.path.islink(full): |
200 | dest = OS.path.realpath(full) | |
201 | for d, local in dirmap: | |
202 | if dest.startswith(d): | |
203 | fix = OS.path.relpath(OS.path.join('/', local, dest[len(d):]), | |
204 | OS.path.join('/', name, | |
205 | OS.path.dirname(file))) | |
206 | st = OS.stat(full) | |
207 | ti = tf.gettarinfo(full, tarname) | |
208 | ti.linkname = fix | |
209 | tf.addfile(ti) | |
210 | skip = True | |
9424c482 MW |
211 | |
212 | ## Nothing special, so just dump the file. Or whatever it is. | |
5dea0052 MW |
213 | if not skip: |
214 | tf.add(full, tarname, recursive = False) | |
a2916c06 MW |
215 | |
216 | def source(out): | |
9424c482 MW |
217 | """ |
218 | Write a tarball for the program's source code to OUT. | |
219 | ||
220 | This function automatically dumps all of the program's dependencies except | |
221 | for those covered by the operating-system exemption. | |
222 | """ | |
223 | ||
224 | ## Make a tarfile writer. There's an annoying incompatibility to bodge | |
225 | ## around. | |
a2916c06 MW |
226 | if SYS.version_info >= (2, 6): |
227 | tf = TAR.open(fileobj = out, mode = 'w|gz', format = TAR.USTAR_FORMAT) | |
228 | else: | |
229 | tf = TAR.open(fileobj = out, mode = 'w|gz') | |
230 | tf.posix = True | |
9424c482 MW |
231 | |
232 | ## First of all, find out what needs to be dumped, and assign names to all | |
233 | ## of the various directories. | |
09ba568f MW |
234 | root = '%s-%s' % (PACKAGE, VERSION) |
235 | seen = set() | |
236 | dirmap = [] | |
237 | festout = StringIO() | |
238 | for dir in dirs_to_dump(): | |
239 | dir = dir.rstrip('/') | |
240 | base = OS.path.basename(dir) | |
241 | if base not in seen: | |
242 | name = base | |
243 | else: | |
244 | for i in I.count(): | |
245 | name = '%s.%d' % (base, i) | |
246 | if name not in seen: break | |
247 | dirmap.append((dir + '/', name)) | |
248 | festout.write('%s = %s\n' % (name, dir)) | |
9424c482 MW |
249 | |
250 | ## Write a map of where things were in the filesystem. This may help a | |
251 | ## user figure out how to deploy the thing. | |
09ba568f MW |
252 | fest = festout.getvalue() |
253 | ti = TAR.TarInfo(OS.path.join(root, 'MANIFEST')) | |
254 | ti.size = len(fest) | |
255 | ti.mtime = T.time() | |
256 | ti.mode = 0664 | |
257 | ti.type = TAR.REGTYPE | |
258 | uid = OS.getuid(); ti.uid, ti.uname = uid, PW.getpwuid(uid).pw_name | |
259 | gid = OS.getgid(); ti.gid, ti.gname = gid, GR.getgrgid(gid).gr_name | |
260 | tf.addfile(ti, fileobj = StringIO(fest)) | |
9424c482 MW |
261 | |
262 | ## Now actually dump all of the individual directories. | |
09ba568f | 263 | for dir, name in dirmap: |
4338f962 | 264 | dump_dir(name, dir, dirmap, tf, root) |
9424c482 MW |
265 | |
266 | ## We're done. | |
a2916c06 MW |
267 | tf.close() |
268 | ||
269 | ###----- That's all, folks -------------------------------------------------- |