3 ### GNU Affero General Public License compliance
5 ### (c) 2013 Mark Wooding
8 ###----- Licensing notice ---------------------------------------------------
10 ### This file is part of Chopwood: a password-changing service.
12 ### Chopwood is free software; you can redistribute it and/or modify
13 ### it under the terms of the GNU Affero General Public License as
14 ### published by the Free Software Foundation; either version 3 of the
15 ### License, or (at your option) any later version.
17 ### Chopwood is distributed in the hope that it will be useful,
18 ### but WITHOUT ANY WARRANTY; without even the implied warranty of
19 ### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 ### GNU Affero General Public License for more details.
22 ### You should have received a copy of the GNU Affero General Public
23 ### License along with Chopwood; if not, see
24 ### <http://www.gnu.org/licenses/>.
26 import contextlib as CTX
32 import subprocess as SUB
38 from cStringIO import StringIO
40 from auto import PACKAGE, VERSION
43 ###--------------------------------------------------------------------------
44 ### Initial utilities.
49 Context manager: create and return the name of a temporary directory.
51 The directory will be deleted automatically on exit from the body.
55 finally: SH.rmtree(d, ignore_errors = True)
57 ###--------------------------------------------------------------------------
58 ### Determining which files to include.
62 Return a list of directories containing used Python modules.
64 Directories under `/usr/' but outside `/usr/local/' are excluded, since
65 they are assumed to be covered by the AGPL exception for parts of the
69 ## Collect a set of directories.
72 ## Work through the list of known modules, adding them to the list.
73 for m in SYS.modules.itervalues():
75 except AttributeError: continue
76 d = OS.path.realpath(OS.path.dirname(f))
77 if d.startswith('/usr/') and not d.startswith('/usr/local/'): continue
80 ## Now go through the directories again, and remove any which are wholly
81 ## included within other entries.
86 if d.startswith(last): continue
90 ## We're done: return the filtered list.
93 ### The `DUMPERS' list consists of (PREDICATE, LISTERS) pairs. The PREDICATE
94 ### is a function of one argument -- a directory name -- which returns true
95 ### if the LISTERS should be used to enumerate that directory. The LISTERS
96 ### are a list of functions of one argument -- again, the directory name --
97 ### which should return an iterable of files within that directory, relative
98 ### to its top-level. Lister functions should not return the root directory,
99 ### because it should obviously only be included once. Instead, the root is
100 ### handled separately by `dump_dir'.
102 def exists_subdir(subdir):
104 Predicate for `DUMPERS': match if the directory has a subdirectory SUBDIR.
106 This is mainly useful for detecting working trees subject to version
109 return lambda dir: OS.path.isdir(OS.path.join(dir, subdir))
113 Lister for `DUMPERS': generate the null-terminated items output by CMD.
115 Run CMD, a string containing words with shell-like quoting (expected to be
116 a literal in the code, so security concerns don't arise) in the directory
117 of interest, yielding the invidual null-terminated strings which the
118 command writes to its standard output.
122 ## Start the command,
123 kid = SUB.Popen(SL.split(cmd), stdout = SUB.PIPE, cwd = dir)
125 ## Collect and return the null-terminated items. Strip off any leading
126 ## `./' and exclude the root directory because that gets handled
131 ## Read a new bufferload of stuff. If there's nothing left then we're
133 buf = kid.stdout.read(16384)
136 ## Tack whatever was left over from last time on the front, and carve
137 ## into null-terminated pieces.
141 z = buf.find('\0', i)
145 if f.rstrip('/') == '.': continue
146 if f.startswith('./'): f = f[2:]
149 ## Whatever's left over will be dealt with next time through.
152 ## Make sure the command actually completed successfully.
155 raise U.ExpectedError, \
156 (500, "lister command `%s' failed (%s) in `%s'" % (
158 (rc & 0xff00) and 'rc = %d' % (rc >> 8) or 'signal %d' % rc,
161 ## If there's trailing junk left over then we should complain.
163 raise U.ExpectedError, \
164 (500, "trailing junk from `%s' in `%s'" % (cmd, dir))
166 ## Return the listing function.
169 ## The list of predicates and listers.
171 (exists_subdir('.git'), [filez('git ls-files -coz --exclude-standard'),
172 filez('find .git -print0')]),
173 (lambda d: True, [filez('find . ( ! -perm +004 -prune ) -o -print0')])]
175 ###--------------------------------------------------------------------------
176 ### Actually dumping files.
178 def dump_dir(name, dir, dirmap, tf, root):
180 Add the contents of directory DIR to the tarfile TF, under the given NAME.
182 The ROOT names the toplevel of the tarball (we're not in the business of
183 making tarbombs here). The DIRMAP is a list of all of the (DIR, NAME)
184 pairs being dumped, used for fixing up symbolic links between directories.
187 ## Find an appropriate `DUMPERS' list entry.
188 for test, listers in DUMPERS:
191 raise U.ExpectedError, (500, "no dumper for `%s'" % dir)
193 ## Write a tarfile entry for the toplevel.
194 tf.add(dir, OS.path.join(root, name), recursive = False)
196 ## Work through all of the listers.
197 for lister in listers:
199 ## Work through each file.
200 for file in lister(dir):
201 with U.Escape() as skip:
202 full = OS.path.join(dir, file)
203 tarname = OS.path.join(root, name, file)
205 ## Check for symbolic links. If we find one that points to another
206 ## directory we're going to dump separately then fiddle it so that it
207 ## works in the directory tree we're going to make.
208 if OS.path.islink(full):
209 dest = OS.path.realpath(full)
210 for d, local in dirmap:
211 if dest.startswith(d):
212 fix = OS.path.relpath(OS.path.join('/', local, dest[len(d):]),
213 OS.path.join('/', name,
214 OS.path.dirname(file)))
216 ti = tf.gettarinfo(full, tarname)
221 ## Nothing special, so just dump the file. Or whatever it is.
222 tf.add(full, tarname, recursive = False)
226 Write a tarball for the program's source code to OUT.
228 This function automatically dumps all of the program's dependencies except
229 for those covered by the operating-system exemption.
232 ## Make a tarfile writer. There's an annoying incompatibility to bodge
234 if SYS.version_info >= (2, 6):
235 tf = TAR.open(fileobj = out, mode = 'w|gz', format = TAR.USTAR_FORMAT)
237 tf = TAR.open(fileobj = out, mode = 'w|gz')
240 ## First of all, find out what needs to be dumped, and assign names to all
241 ## of the various directories.
242 root = '%s-%s' % (PACKAGE, VERSION)
246 for dir in dirs_to_dump():
247 dir = dir.rstrip('/')
248 base = OS.path.basename(dir)
253 name = '%s.%d' % (base, i)
254 if name not in seen: break
255 dirmap.append((dir + '/', name))
256 festout.write('%s = %s\n' % (name, dir))
258 ## Write a map of where things were in the filesystem. This may help a
259 ## user figure out how to deploy the thing.
260 fest = festout.getvalue()
261 ti = TAR.TarInfo(OS.path.join(root, 'MANIFEST'))
265 ti.type = TAR.REGTYPE
266 uid = OS.getuid(); ti.uid, ti.uname = uid, PW.getpwuid(uid).pw_name
267 gid = OS.getgid(); ti.gid, ti.gname = gid, GR.getgrgid(gid).gr_name
268 tf.addfile(ti, fileobj = StringIO(fest))
270 ## Now actually dump all of the individual directories.
271 for dir, name in dirmap:
272 dump_dir(name, dir, dirmap, tf, root)
277 ###----- That's all, folks --------------------------------------------------