Found in crybaby's working tree.

[chopwood] / agpl.py

diff --git a/agpl.py b/agpl.py
index b89330cb56de6e64f66303af0743e7dd184fc819..0929e78bfb94b6e0c0523b06fd6eea62bfc43aa7 100644 (file)
--- a/agpl.py
+++ b/agpl.py
@@ -23,32 +23,64 @@
 ### License along with Chopwood; if not, see
 ### <http://www.gnu.org/licenses/>.
 
 ### License along with Chopwood; if not, see
 ### <http://www.gnu.org/licenses/>.
 

+from __future__ import with_statement
+

 import contextlib as CTX
 import contextlib as CTX

+import grp as GR

 import os as OS
 import os as OS

+import pwd as PW

 import shlex as SL
 import shutil as SH
 import subprocess as SUB
 import sys as SYS
 import tarfile as TAR
 import tempfile as TF
 import shlex as SL
 import shutil as SH
 import subprocess as SUB
 import sys as SYS
 import tarfile as TAR
 import tempfile as TF

+import time as T
+
+from cStringIO import StringIO

 
 from auto import PACKAGE, VERSION
 import util as U
 
 
 from auto import PACKAGE, VERSION
 import util as U
 

+###--------------------------------------------------------------------------
+### Initial utilities.
+

 @CTX.contextmanager
 def tempdir():
 @CTX.contextmanager
 def tempdir():

+  """
+  Context manager: create and return the name of a temporary directory.
+
+  The directory will be deleted automatically on exit from the body.
+  """

   d = TF.mkdtemp()
   try: yield d
   finally: SH.rmtree(d, ignore_errors = True)
 
   d = TF.mkdtemp()
   try: yield d
   finally: SH.rmtree(d, ignore_errors = True)
 

+###--------------------------------------------------------------------------
+### Determining which files to include.
+

 def dirs_to_dump():
 def dirs_to_dump():

+  """
+  Return a list of directories containing used Python modules.
+
+  Directories under `/usr/' but outside `/usr/local/' are excluded, since
+  they are assumed to be covered by the AGPL exception for parts of the
+  operating system.
+  """
+
+  ## Collect a set of directories.

   dirs = set()
   dirs = set()

+
+  ## Work through the list of known modules, adding them to the list.

   for m in SYS.modules.itervalues():
     try: f = m.__file__
     except AttributeError: continue
     d = OS.path.realpath(OS.path.dirname(f))
     if d.startswith('/usr/') and not d.startswith('/usr/local/'): continue
     dirs.add(d)
   for m in SYS.modules.itervalues():
     try: f = m.__file__
     except AttributeError: continue
     d = OS.path.realpath(OS.path.dirname(f))
     if d.startswith('/usr/') and not d.startswith('/usr/local/'): continue
     dirs.add(d)

+
+  ## Now go through the directories again, and remove any which are wholly
+  ## included within other entries.

   dirs = sorted(dirs)
   last = '!'
   dump = []
   dirs = sorted(dirs)
   last = '!'
   dump = []


@@ -56,57 +88,192 @@ def dirs_to_dump():
     if d.startswith(last): continue
     dump.append(d)
     last = d
     if d.startswith(last): continue
     dump.append(d)
     last = d

+
+  ## We're done: return the filtered list.

   return dump
 
   return dump
 

+### The `DUMPERS' list consists of (PREDICATE, LISTERS) pairs.  The PREDICATE
+### is a function of one argument -- a directory name -- which returns true
+### if the LISTERS should be used to enumerate that directory.  The LISTERS
+### are a list of functions of one argument -- again, the directory name --
+### which should return an iterable of files within that directory, relative
+### to its top-level.  Lister functions should not return the root directory,
+### because it should obviously only be included once.  Instead, the root is
+### handled separately by `dump_dir'.
+

 def exists_subdir(subdir):
 def exists_subdir(subdir):

+  """
+  Predicate for `DUMPERS': match if the directory has a subdirectory SUBDIR.
+
+  This is mainly useful for detecting working trees subject to version
+  control.
+  """

   return lambda dir: OS.path.isdir(OS.path.join(dir, subdir))
 
 def filez(cmd):
   return lambda dir: OS.path.isdir(OS.path.join(dir, subdir))
 
 def filez(cmd):

+  """
+  Lister for `DUMPERS': generate the null-terminated items output by CMD.
+
+  Run CMD, a string containing words with shell-like quoting (expected to be
+  a literal in the code, so security concerns don't arise) in the directory
+  of interest, yielding the invidual null-terminated strings which the
+  command writes to its standard output.
+  """

   def _(dir):
   def _(dir):

+
+    ## Start the command,

     kid = SUB.Popen(SL.split(cmd), stdout = SUB.PIPE, cwd = dir)
     kid = SUB.Popen(SL.split(cmd), stdout = SUB.PIPE, cwd = dir)

+
+    ## Collect and return the null-terminated items.  Strip off any leading
+    ## `./' and exclude the root directory because that gets handled
+    ## separately.

     left = ''
     while True:
     left = ''
     while True:

+
+      ## Read a new bufferload of stuff.  If there's nothing left then we're
+      ## done.

       buf = kid.stdout.read(16384)
       if not buf: break
       buf = kid.stdout.read(16384)
       if not buf: break

+
+      ## Tack whatever was left over from last time on the front, and carve
+      ## into null-terminated pieces.

       buf = left + buf
       i = 0
       while True:
         z = buf.find('\0', i)
         if z < 0: break
         f = buf[i:z]
       buf = left + buf
       i = 0
       while True:
         z = buf.find('\0', i)
         if z < 0: break
         f = buf[i:z]

+        i = z + 1
+        if f.rstrip('/') == '.': continue

         if f.startswith('./'): f = f[2:]
         yield f
         if f.startswith('./'): f = f[2:]
         yield f

-        i = z + 1
+
+      ## Whatever's left over will be dealt with next time through.

       left = buf[i:]
       left = buf[i:]

+
+    ## Make sure the command actually completed successfully.
+    if kid.wait():
+      rc = kid.returncode
+      raise U.ExpectedError, \
+          (500, "lister command `%s' failed (%s) in `%s'" % (
+            cmd,
+            (rc & 0xff00) and 'rc = %d' % (rc >> 8) or 'signal %d' % rc,
+            dir))
+
+    ## If there's trailing junk left over then we should complain.

     if left:
       raise U.ExpectedError, \
             (500, "trailing junk from `%s' in `%s'" % (cmd, dir))
     if left:
       raise U.ExpectedError, \
             (500, "trailing junk from `%s' in `%s'" % (cmd, dir))

+
+  ## Return the listing function.

   return _
 
   return _
 

+## The list of predicates and listers.

 DUMPERS = [
   (exists_subdir('.git'), [filez('git ls-files -coz --exclude-standard'),
                            filez('find .git -print0')]),
   (lambda d: True, [filez('find . ( ! -perm +004 -prune ) -o -print0')])]
 
 DUMPERS = [
   (exists_subdir('.git'), [filez('git ls-files -coz --exclude-standard'),
                            filez('find .git -print0')]),
   (lambda d: True, [filez('find . ( ! -perm +004 -prune ) -o -print0')])]
 

-def dump_dir(dir, tf, root):
+###--------------------------------------------------------------------------
+### Actually dumping files.
+
+def dump_dir(name, dir, dirmap, tf, root):
+  """
+  Add the contents of directory DIR to the tarfile TF, under the given NAME.
+
+  The ROOT names the toplevel of the tarball (we're not in the business of
+  making tarbombs here).  The DIRMAP is a list of all of the (DIR, NAME)
+  pairs being dumped, used for fixing up symbolic links between directories.
+  """
+
+  ## Find an appropriate `DUMPERS' list entry.

   for test, listers in DUMPERS:
     if test(dir): break
   else:
     raise U.ExpectedError, (500, "no dumper for `%s'" % dir)
   for test, listers in DUMPERS:
     if test(dir): break
   else:
     raise U.ExpectedError, (500, "no dumper for `%s'" % dir)

+
+  ## Write a tarfile entry for the toplevel.
+  tf.add(dir, OS.path.join(root, name), recursive = False)
+
+  ## Work through all of the listers.

   for lister in listers:
   for lister in listers:

-    base = OS.path.basename(dir)
+
+    ## Work through each file.

     for file in lister(dir):
     for file in lister(dir):

-      tf.add(OS.path.join(dir, file), OS.path.join(root, base, file),
-             recursive = False)
+      with U.Escape() as skip:
+        full = OS.path.join(dir, file)
+        tarname = OS.path.join(root, name, file)
+
+        ## Check for symbolic links.  If we find one that points to another
+        ## directory we're going to dump separately then fiddle it so that it
+        ## works in the directory tree we're going to make.
+        if OS.path.islink(full):
+          dest = OS.path.realpath(full)
+          for d, local in dirmap:
+            if dest.startswith(d):
+              fix = OS.path.relpath(OS.path.join('/', local, dest[len(d):]),
+                                    OS.path.join('/', name,
+                                                 OS.path.dirname(file)))
+              st = OS.stat(full)
+              ti = tf.gettarinfo(full, tarname)
+              ti.linkname = fix
+              tf.addfile(ti)
+              skip()
+
+        ## Nothing special, so just dump the file.  Or whatever it is.
+        tf.add(full, tarname, recursive = False)

 
 def source(out):
 
 def source(out):

+  """
+  Write a tarball for the program's source code to OUT.
+
+  This function automatically dumps all of the program's dependencies except
+  for those covered by the operating-system exemption.
+  """
+
+  ## Make a tarfile writer.  There's an annoying incompatibility to bodge
+  ## around.

   if SYS.version_info >= (2, 6):
     tf = TAR.open(fileobj = out, mode = 'w|gz', format = TAR.USTAR_FORMAT)
   else:
     tf = TAR.open(fileobj = out, mode = 'w|gz')
     tf.posix = True
   if SYS.version_info >= (2, 6):
     tf = TAR.open(fileobj = out, mode = 'w|gz', format = TAR.USTAR_FORMAT)
   else:
     tf = TAR.open(fileobj = out, mode = 'w|gz')
     tf.posix = True

-  for d in dirs_to_dump():
-    dump_dir(d, tf, '%s-%s' % (PACKAGE, VERSION))
+
+  ## First of all, find out what needs to be dumped, and assign names to all
+  ## of the various directories.
+  root = '%s-%s' % (PACKAGE, VERSION)
+  seen = set()
+  dirmap = []
+  festout = StringIO()
+  for dir in dirs_to_dump():
+    dir = dir.rstrip('/')
+    base = OS.path.basename(dir)
+    if base not in seen:
+      name = base
+    else:
+      for i in I.count():
+        name = '%s.%d' % (base, i)
+        if name not in seen: break
+    dirmap.append((dir + '/', name))
+    festout.write('%s = %s\n' % (name, dir))
+
+  ## Write a map of where things were in the filesystem.  This may help a
+  ## user figure out how to deploy the thing.
+  fest = festout.getvalue()
+  ti = TAR.TarInfo(OS.path.join(root, 'MANIFEST'))
+  ti.size = len(fest)
+  ti.mtime = T.time()
+  ti.mode = 0664
+  ti.type = TAR.REGTYPE
+  uid = OS.getuid(); ti.uid, ti.uname = uid, PW.getpwuid(uid).pw_name
+  gid = OS.getgid(); ti.gid, ti.gname = gid, GR.getgrgid(gid).gr_name
+  tf.addfile(ti, fileobj = StringIO(fest))
+
+  ## Now actually dump all of the individual directories.
+  for dir, name in dirmap:
+    dump_dir(name, dir, dirmap, tf, root)
+
+  ## We're done.

   tf.close()
 
 ###----- That's all, folks --------------------------------------------------
   tf.close()
 
 ###----- That's all, folks --------------------------------------------------