3 ### GNU Affero General Public License compliance
5 ### (c) 2013 Mark Wooding
8 ###----- Licensing notice ---------------------------------------------------
10 ### This file is part of Chopwood: a password-changing service.
12 ### Chopwood is free software; you can redistribute it and/or modify
13 ### it under the terms of the GNU Affero General Public License as
14 ### published by the Free Software Foundation; either version 3 of the
15 ### License, or (at your option) any later version.
17 ### Chopwood is distributed in the hope that it will be useful,
18 ### but WITHOUT ANY WARRANTY; without even the implied warranty of
19 ### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 ### GNU Affero General Public License for more details.
22 ### You should have received a copy of the GNU Affero General Public
23 ### License along with Chopwood; if not, see
24 ### <http://www.gnu.org/licenses/>.
26 from __future__ import with_statement
28 import contextlib as CTX
34 import subprocess as SUB
40 from cStringIO import StringIO
42 from auto import PACKAGE, VERSION
45 ###--------------------------------------------------------------------------
46 ### Initial utilities.
51 Context manager: create and return the name of a temporary directory.
53 The directory will be deleted automatically on exit from the body.
57 finally: SH.rmtree(d, ignore_errors = True)
59 ###--------------------------------------------------------------------------
60 ### Determining which files to include.
64 Return a list of directories containing used Python modules.
66 Directories under `/usr/' but outside `/usr/local/' are excluded, since
67 they are assumed to be covered by the AGPL exception for parts of the
71 ## Collect a set of directories.
74 ## Work through the list of known modules, adding them to the list.
75 for m in SYS.modules.itervalues():
77 except AttributeError: continue
78 d = OS.path.realpath(OS.path.dirname(f))
79 if d.startswith('/usr/') and not d.startswith('/usr/local/'): continue
82 ## Now go through the directories again, and remove any which are wholly
83 ## included within other entries.
88 if d.startswith(last): continue
92 ## We're done: return the filtered list.
95 ### The `DUMPERS' list consists of (PREDICATE, LISTERS) pairs. The PREDICATE
96 ### is a function of one argument -- a directory name -- which returns true
97 ### if the LISTERS should be used to enumerate that directory. The LISTERS
98 ### are a list of functions of one argument -- again, the directory name --
99 ### which should return an iterable of files within that directory, relative
100 ### to its top-level. Lister functions should not return the root directory,
101 ### because it should obviously only be included once. Instead, the root is
102 ### handled separately by `dump_dir'.
104 def exists_subdir(subdir):
106 Predicate for `DUMPERS': match if the directory has a subdirectory SUBDIR.
108 This is mainly useful for detecting working trees subject to version
111 return lambda dir: OS.path.isdir(OS.path.join(dir, subdir))
115 Lister for `DUMPERS': generate the null-terminated items output by CMD.
117 Run CMD, a string containing words with shell-like quoting (expected to be
118 a literal in the code, so security concerns don't arise) in the directory
119 of interest, yielding the invidual null-terminated strings which the
120 command writes to its standard output.
124 ## Start the command,
125 kid = SUB.Popen(SL.split(cmd), stdout = SUB.PIPE, cwd = dir)
127 ## Collect and return the null-terminated items. Strip off any leading
128 ## `./' and exclude the root directory because that gets handled
133 ## Read a new bufferload of stuff. If there's nothing left then we're
135 buf = kid.stdout.read(16384)
138 ## Tack whatever was left over from last time on the front, and carve
139 ## into null-terminated pieces.
143 z = buf.find('\0', i)
147 if f.rstrip('/') == '.': continue
148 if f.startswith('./'): f = f[2:]
151 ## Whatever's left over will be dealt with next time through.
154 ## Make sure the command actually completed successfully.
157 raise U.ExpectedError, \
158 (500, "lister command `%s' failed (%s) in `%s'" % (
160 (rc & 0xff00) and 'rc = %d' % (rc >> 8) or 'signal %d' % rc,
163 ## If there's trailing junk left over then we should complain.
165 raise U.ExpectedError, \
166 (500, "trailing junk from `%s' in `%s'" % (cmd, dir))
168 ## Return the listing function.
171 ## The list of predicates and listers.
173 (exists_subdir('.git'), [filez('git ls-files -coz --exclude-standard'),
174 filez('find .git -print0')]),
175 (lambda d: True, [filez('find . ( ! -perm +004 -prune ) -o -print0')])]
177 ###--------------------------------------------------------------------------
178 ### Actually dumping files.
180 def dump_dir(name, dir, dirmap, tf, root):
182 Add the contents of directory DIR to the tarfile TF, under the given NAME.
184 The ROOT names the toplevel of the tarball (we're not in the business of
185 making tarbombs here). The DIRMAP is a list of all of the (DIR, NAME)
186 pairs being dumped, used for fixing up symbolic links between directories.
189 ## Find an appropriate `DUMPERS' list entry.
190 for test, listers in DUMPERS:
193 raise U.ExpectedError, (500, "no dumper for `%s'" % dir)
195 ## Write a tarfile entry for the toplevel.
196 tf.add(dir, OS.path.join(root, name), recursive = False)
198 ## Work through all of the listers.
199 for lister in listers:
201 ## Work through each file.
202 for file in lister(dir):
203 with U.Escape() as skip:
204 full = OS.path.join(dir, file)
205 tarname = OS.path.join(root, name, file)
207 ## Check for symbolic links. If we find one that points to another
208 ## directory we're going to dump separately then fiddle it so that it
209 ## works in the directory tree we're going to make.
210 if OS.path.islink(full):
211 dest = OS.path.realpath(full)
212 for d, local in dirmap:
213 if dest.startswith(d):
214 fix = OS.path.relpath(OS.path.join('/', local, dest[len(d):]),
215 OS.path.join('/', name,
216 OS.path.dirname(file)))
218 ti = tf.gettarinfo(full, tarname)
223 ## Nothing special, so just dump the file. Or whatever it is.
224 tf.add(full, tarname, recursive = False)
228 Write a tarball for the program's source code to OUT.
230 This function automatically dumps all of the program's dependencies except
231 for those covered by the operating-system exemption.
234 ## Make a tarfile writer. There's an annoying incompatibility to bodge
236 if SYS.version_info >= (2, 6):
237 tf = TAR.open(fileobj = out, mode = 'w|gz', format = TAR.USTAR_FORMAT)
239 tf = TAR.open(fileobj = out, mode = 'w|gz')
242 ## First of all, find out what needs to be dumped, and assign names to all
243 ## of the various directories.
244 root = '%s-%s' % (PACKAGE, VERSION)
248 for dir in dirs_to_dump():
249 dir = dir.rstrip('/')
250 base = OS.path.basename(dir)
255 name = '%s.%d' % (base, i)
256 if name not in seen: break
257 dirmap.append((dir + '/', name))
258 festout.write('%s = %s\n' % (name, dir))
260 ## Write a map of where things were in the filesystem. This may help a
261 ## user figure out how to deploy the thing.
262 fest = festout.getvalue()
263 ti = TAR.TarInfo(OS.path.join(root, 'MANIFEST'))
267 ti.type = TAR.REGTYPE
268 uid = OS.getuid(); ti.uid, ti.uname = uid, PW.getpwuid(uid).pw_name
269 gid = OS.getgid(); ti.gid, ti.gname = gid, GR.getgrgid(gid).gr_name
270 tf.addfile(ti, fileobj = StringIO(fest))
272 ## Now actually dump all of the individual directories.
273 for dir, name in dirmap:
274 dump_dir(name, dir, dirmap, tf, root)
279 ###----- That's all, folks --------------------------------------------------