| 1 | ### -*-python-*- |
| 2 | ### |
| 3 | ### GNU Affero General Public License compliance |
| 4 | ### |
| 5 | ### (c) 2013 Mark Wooding |
| 6 | ### |
| 7 | |
| 8 | ###----- Licensing notice --------------------------------------------------- |
| 9 | ### |
| 10 | ### This file is part of Chopwood: a password-changing service. |
| 11 | ### |
| 12 | ### Chopwood is free software; you can redistribute it and/or modify |
| 13 | ### it under the terms of the GNU Affero General Public License as |
| 14 | ### published by the Free Software Foundation; either version 3 of the |
| 15 | ### License, or (at your option) any later version. |
| 16 | ### |
| 17 | ### Chopwood is distributed in the hope that it will be useful, |
| 18 | ### but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 19 | ### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 20 | ### GNU Affero General Public License for more details. |
| 21 | ### |
| 22 | ### You should have received a copy of the GNU Affero General Public |
| 23 | ### License along with Chopwood; if not, see |
| 24 | ### <http://www.gnu.org/licenses/>. |
| 25 | |
| 26 | from __future__ import with_statement |
| 27 | |
| 28 | import contextlib as CTX |
| 29 | import grp as GR |
| 30 | import os as OS |
| 31 | import pwd as PW |
| 32 | import shlex as SL |
| 33 | import shutil as SH |
| 34 | import subprocess as SUB |
| 35 | import sys as SYS |
| 36 | import tarfile as TAR |
| 37 | import tempfile as TF |
| 38 | import time as T |
| 39 | |
| 40 | from cStringIO import StringIO |
| 41 | |
| 42 | from auto import PACKAGE, VERSION |
| 43 | import util as U |
| 44 | |
| 45 | ###-------------------------------------------------------------------------- |
| 46 | ### Initial utilities. |
| 47 | |
| 48 | @CTX.contextmanager |
| 49 | def tempdir(): |
| 50 | """ |
| 51 | Context manager: create and return the name of a temporary directory. |
| 52 | |
| 53 | The directory will be deleted automatically on exit from the body. |
| 54 | """ |
| 55 | d = TF.mkdtemp() |
| 56 | try: yield d |
| 57 | finally: SH.rmtree(d, ignore_errors = True) |
| 58 | |
| 59 | ###-------------------------------------------------------------------------- |
| 60 | ### Determining which files to include. |
| 61 | |
| 62 | def dirs_to_dump(): |
| 63 | """ |
| 64 | Return a list of directories containing used Python modules. |
| 65 | |
| 66 | Directories under `/usr/' but outside `/usr/local/' are excluded, since |
| 67 | they are assumed to be covered by the AGPL exception for parts of the |
| 68 | operating system. |
| 69 | """ |
| 70 | |
| 71 | ## Collect a set of directories. |
| 72 | dirs = set() |
| 73 | |
| 74 | ## Work through the list of known modules, adding them to the list. |
| 75 | for m in SYS.modules.itervalues(): |
| 76 | try: f = m.__file__ |
| 77 | except AttributeError: continue |
| 78 | d = OS.path.realpath(OS.path.dirname(f)) |
| 79 | if d.startswith('/usr/') and not d.startswith('/usr/local/'): continue |
| 80 | dirs.add(d) |
| 81 | |
| 82 | ## Now go through the directories again, and remove any which are wholly |
| 83 | ## included within other entries. |
| 84 | dirs = sorted(dirs) |
| 85 | last = '!' |
| 86 | dump = [] |
| 87 | for d in dirs: |
| 88 | if d.startswith(last): continue |
| 89 | dump.append(d) |
| 90 | last = d |
| 91 | |
| 92 | ## We're done: return the filtered list. |
| 93 | return dump |
| 94 | |
| 95 | ### The `DUMPERS' list consists of (PREDICATE, LISTERS) pairs. The PREDICATE |
| 96 | ### is a function of one argument -- a directory name -- which returns true |
| 97 | ### if the LISTERS should be used to enumerate that directory. The LISTERS |
| 98 | ### are a list of functions of one argument -- again, the directory name -- |
| 99 | ### which should return an iterable of files within that directory, relative |
| 100 | ### to its top-level. Lister functions should not return the root directory, |
| 101 | ### because it should obviously only be included once. Instead, the root is |
| 102 | ### handled separately by `dump_dir'. |
| 103 | |
| 104 | def exists_subdir(subdir): |
| 105 | """ |
| 106 | Predicate for `DUMPERS': match if the directory has a subdirectory SUBDIR. |
| 107 | |
| 108 | This is mainly useful for detecting working trees subject to version |
| 109 | control. |
| 110 | """ |
| 111 | return lambda dir: OS.path.isdir(OS.path.join(dir, subdir)) |
| 112 | |
| 113 | def filez(cmd): |
| 114 | """ |
| 115 | Lister for `DUMPERS': generate the null-terminated items output by CMD. |
| 116 | |
| 117 | Run CMD, a string containing words with shell-like quoting (expected to be |
| 118 | a literal in the code, so security concerns don't arise) in the directory |
| 119 | of interest, yielding the invidual null-terminated strings which the |
| 120 | command writes to its standard output. |
| 121 | """ |
| 122 | def _(dir): |
| 123 | |
| 124 | ## Start the command, |
| 125 | kid = SUB.Popen(SL.split(cmd), stdout = SUB.PIPE, cwd = dir) |
| 126 | |
| 127 | ## Collect and return the null-terminated items. Strip off any leading |
| 128 | ## `./' and exclude the root directory because that gets handled |
| 129 | ## separately. |
| 130 | left = '' |
| 131 | while True: |
| 132 | |
| 133 | ## Read a new bufferload of stuff. If there's nothing left then we're |
| 134 | ## done. |
| 135 | buf = kid.stdout.read(16384) |
| 136 | if not buf: break |
| 137 | |
| 138 | ## Tack whatever was left over from last time on the front, and carve |
| 139 | ## into null-terminated pieces. |
| 140 | buf = left + buf |
| 141 | i = 0 |
| 142 | while True: |
| 143 | z = buf.find('\0', i) |
| 144 | if z < 0: break |
| 145 | f = buf[i:z] |
| 146 | i = z + 1 |
| 147 | if f.rstrip('/') == '.': continue |
| 148 | if f.startswith('./'): f = f[2:] |
| 149 | yield f |
| 150 | |
| 151 | ## Whatever's left over will be dealt with next time through. |
| 152 | left = buf[i:] |
| 153 | |
| 154 | ## Make sure the command actually completed successfully. |
| 155 | if kid.wait(): |
| 156 | rc = kid.returncode |
| 157 | raise U.ExpectedError, \ |
| 158 | (500, "lister command `%s' failed (%s) in `%s'" % ( |
| 159 | cmd, |
| 160 | (rc & 0xff00) and 'rc = %d' % (rc >> 8) or 'signal %d' % rc, |
| 161 | dir)) |
| 162 | |
| 163 | ## If there's trailing junk left over then we should complain. |
| 164 | if left: |
| 165 | raise U.ExpectedError, \ |
| 166 | (500, "trailing junk from `%s' in `%s'" % (cmd, dir)) |
| 167 | |
| 168 | ## Return the listing function. |
| 169 | return _ |
| 170 | |
| 171 | ## The list of predicates and listers. |
| 172 | DUMPERS = [ |
| 173 | (exists_subdir('.git'), [filez('git ls-files -coz --exclude-standard'), |
| 174 | filez('find .git -print0')]), |
| 175 | (lambda d: True, [filez('find . ( ! -perm +004 -prune ) -o -print0')])] |
| 176 | |
| 177 | ###-------------------------------------------------------------------------- |
| 178 | ### Actually dumping files. |
| 179 | |
| 180 | def dump_dir(name, dir, dirmap, tf, root): |
| 181 | """ |
| 182 | Add the contents of directory DIR to the tarfile TF, under the given NAME. |
| 183 | |
| 184 | The ROOT names the toplevel of the tarball (we're not in the business of |
| 185 | making tarbombs here). The DIRMAP is a list of all of the (DIR, NAME) |
| 186 | pairs being dumped, used for fixing up symbolic links between directories. |
| 187 | """ |
| 188 | |
| 189 | ## Find an appropriate `DUMPERS' list entry. |
| 190 | for test, listers in DUMPERS: |
| 191 | if test(dir): break |
| 192 | else: |
| 193 | raise U.ExpectedError, (500, "no dumper for `%s'" % dir) |
| 194 | |
| 195 | ## Write a tarfile entry for the toplevel. |
| 196 | tf.add(dir, OS.path.join(root, name), recursive = False) |
| 197 | |
| 198 | ## Work through all of the listers. |
| 199 | for lister in listers: |
| 200 | |
| 201 | ## Work through each file. |
| 202 | for file in lister(dir): |
| 203 | with U.Escape() as skip: |
| 204 | full = OS.path.join(dir, file) |
| 205 | tarname = OS.path.join(root, name, file) |
| 206 | |
| 207 | ## Check for symbolic links. If we find one that points to another |
| 208 | ## directory we're going to dump separately then fiddle it so that it |
| 209 | ## works in the directory tree we're going to make. |
| 210 | if OS.path.islink(full): |
| 211 | dest = OS.path.realpath(full) |
| 212 | for d, local in dirmap: |
| 213 | if dest.startswith(d): |
| 214 | fix = OS.path.relpath(OS.path.join('/', local, dest[len(d):]), |
| 215 | OS.path.join('/', name, |
| 216 | OS.path.dirname(file))) |
| 217 | st = OS.stat(full) |
| 218 | ti = tf.gettarinfo(full, tarname) |
| 219 | ti.linkname = fix |
| 220 | tf.addfile(ti) |
| 221 | skip() |
| 222 | |
| 223 | ## Nothing special, so just dump the file. Or whatever it is. |
| 224 | tf.add(full, tarname, recursive = False) |
| 225 | |
| 226 | def source(out): |
| 227 | """ |
| 228 | Write a tarball for the program's source code to OUT. |
| 229 | |
| 230 | This function automatically dumps all of the program's dependencies except |
| 231 | for those covered by the operating-system exemption. |
| 232 | """ |
| 233 | |
| 234 | ## Make a tarfile writer. There's an annoying incompatibility to bodge |
| 235 | ## around. |
| 236 | if SYS.version_info >= (2, 6): |
| 237 | tf = TAR.open(fileobj = out, mode = 'w|gz', format = TAR.USTAR_FORMAT) |
| 238 | else: |
| 239 | tf = TAR.open(fileobj = out, mode = 'w|gz') |
| 240 | tf.posix = True |
| 241 | |
| 242 | ## First of all, find out what needs to be dumped, and assign names to all |
| 243 | ## of the various directories. |
| 244 | root = '%s-%s' % (PACKAGE, VERSION) |
| 245 | seen = set() |
| 246 | dirmap = [] |
| 247 | festout = StringIO() |
| 248 | for dir in dirs_to_dump(): |
| 249 | dir = dir.rstrip('/') |
| 250 | base = OS.path.basename(dir) |
| 251 | if base not in seen: |
| 252 | name = base |
| 253 | else: |
| 254 | for i in I.count(): |
| 255 | name = '%s.%d' % (base, i) |
| 256 | if name not in seen: break |
| 257 | dirmap.append((dir + '/', name)) |
| 258 | festout.write('%s = %s\n' % (name, dir)) |
| 259 | |
| 260 | ## Write a map of where things were in the filesystem. This may help a |
| 261 | ## user figure out how to deploy the thing. |
| 262 | fest = festout.getvalue() |
| 263 | ti = TAR.TarInfo(OS.path.join(root, 'MANIFEST')) |
| 264 | ti.size = len(fest) |
| 265 | ti.mtime = T.time() |
| 266 | ti.mode = 0664 |
| 267 | ti.type = TAR.REGTYPE |
| 268 | uid = OS.getuid(); ti.uid, ti.uname = uid, PW.getpwuid(uid).pw_name |
| 269 | gid = OS.getgid(); ti.gid, ti.gname = gid, GR.getgrgid(gid).gr_name |
| 270 | tf.addfile(ti, fileobj = StringIO(fest)) |
| 271 | |
| 272 | ## Now actually dump all of the individual directories. |
| 273 | for dir, name in dirmap: |
| 274 | dump_dir(name, dir, dirmap, tf, root) |
| 275 | |
| 276 | ## We're done. |
| 277 | tf.close() |
| 278 | |
| 279 | ###----- That's all, folks -------------------------------------------------- |