chiark / gitweb /
ownsource: wip packages
authorIan Jackson <ijackson@chiark.greenend.org.uk>
Tue, 4 Apr 2017 18:29:25 +0000 (19:29 +0100)
committerIan Jackson <ijackson@chiark.greenend.org.uk>
Tue, 4 Apr 2017 23:38:02 +0000 (00:38 +0100)
Signed-off-by: Ian Jackson <ijackson@chiark.greenend.org.uk>
hippotatd
hippotatlib/ownsource.py

index 0f6a318..9dccf98 100755 (executable)
--- a/hippotatd
+++ b/hippotatd
@@ -237,37 +237,36 @@ class IphttpResource(NotStupidResource):
 <html><body>
 hippotat
 <p>
-<a href="source">source</a> available
+<a href="source">source</a>
+(and that of dependency <a href="source">packages</a>)
+available
 </body></html>
 '''
 
-class SourceResource(twisted.web.static.File):
-  def __init__(self):
-    td = tempfile.mkdtemp()
-
-    def cleanup():
-      try: shutil.rmtree(td)
-      except FileNotFoundError: pass
-
-    cleanups.append(cleanup)
-
-    self._ssp = SourceShipmentPreparer(td)
-    self._ssp.logger = self.log
-    self._ssp.generate()
-
-    super().__init__(self._ssp.output_path)
-
-  def log(self, m):
-    log_debug(DBG.OWNSOURCE, m)
-
 def start_http():
   resource = IphttpResource()
-  resource.putChild(b'source',SourceResource())
   site = twisted.web.server.Site(resource)
+
   for sa in c.saddrs:
     ep = sa.make_endpoint()
     crash_on_defer(ep.listen(site))
     log_debug(DBG.INIT, 'listening on %s' % sa)
+
+  td = tempfile.mkdtemp()
+
+  def cleanup():
+    try: shutil.rmtree(td)
+    except FileNotFoundError: pass
+    
+  cleanups.append(cleanup)
+
+  ssp = SourceShipmentPreparer(td)
+  ssp.logger = partial(log_debug, DBG.OWNSOURCE)
+  ssp.generate()
+
+  resource.putChild(b'source', twisted.web.static.File(ssp.output_paths[0]))
+  resource.putChild(b'srcpkgs', twisted.web.static.File(ssp.output_paths[0]))
+
   reactor.callLater(0.1, (lambda: log.info('hippotatd started', dflag=False)))
 
 #---------- config and setup ----------
index 4269055..385c9ea 100644 (file)
@@ -29,12 +29,16 @@ import fnmatch
 import stat
 import subprocess
 import tempfile
+import shutil
+
+try: import debian.deb822
+except ImportError: pass
 
 class SourceShipmentPreparer():
   def __init__(s, destdir):
     # caller may modify, and should read after calling generate()
-    s.output_name = 'srcbomb.tar.gz'
-    # s.output_path   alternatively caller may read this
+    s.output_names = ['srcbomb.tar.gz', 'fullsrcbomb.tar']
+    s.output_paths = [None,None] # alternatively caller may read this
     # defaults, caller can modify after creation
     s.logger = lambda m: print('SourceShipmentPreparer',m)
     s.src_filter = s.src_filter_glob
@@ -49,6 +53,7 @@ class SourceShipmentPreparer():
                   '[0-9][0-9][0-9][0-9]-src.cpio']
     s.rune_shell = ['/bin/bash', '-ec']
     s.show_pathnames = True
+    s.download_packages = True
     s.rune_cpio = r'''
             set -o pipefail
            (
@@ -60,9 +65,10 @@ class SourceShipmentPreparer():
             )
     '''
     s.rune_portmanteau = r'''
-            outfile=$1; shift
-            rm -f "$outfile"
-            GZIP=-1 tar zcf "$outfile" "$@"
+            GZIP=-1 tar zcf - "$@"
+    '''
+    s.rune_portmanteau_uncompressed = r'''
+            tar cf - "$@"
     '''
     s.manifest_name='0000-MANIFEST.txt'
     # private
@@ -71,6 +77,8 @@ class SourceShipmentPreparer():
     s._manifest = []
     s._dirmap = { }
     s._package_files = { } # map filename => infol
+    s._packages_path = os.path.join(s._destdir, 'packages')
+    s._package_sources = []
 
   def thing_matches_globs(s, thing, globs):
     for pat in globs:
@@ -140,7 +148,7 @@ class SourceShipmentPreparer():
 
   def srcdir_find_rune(s, d):
     script = s.find_rune_base
-    ignores = s.ignores + [s.output_name, s.manifest_name]
+    ignores = s.ignores + s.output_names + [s.manifest_name]
     ignores += s.src_direxcludes(d)
     for excl in ignores:
       assert("'" not in excl)
@@ -224,7 +232,7 @@ class SourceShipmentPreparer():
     assert(dpkg_S.wait() == 0)
     dpkg_show_in.seek(0)
     cmdl = ['xargs','-r','dpkg-query',
-            r'-f${binary:Package}\t${Package}\t${Architecture}\t${Version}\t${source:Package}\t${source:Version}\n',
+            r'-f${binary:Package}\t${Package}\t${Architecture}\t${Version}\t${source:Package}\t${source:Version}\t${source:Upstream-Version}\n',
             '--show','--']
     dpkg_show = subprocess.Popen(cmdl,
                                  cwd='/',
@@ -234,12 +242,13 @@ class SourceShipmentPreparer():
                                  close_fds=False)
     for l in dpkg_show.stdout:
       l = l.strip(b'\n').decode('utf-8')
-      (pk,p,a,v,sp,sv) = l.split('\t')
+      (pk,p,a,v,sp,sv,suv) = l.split('\t')
       pkginfos[pk]['binary'] = p
       pkginfos[pk]['arch'] = a
       pkginfos[pk]['version'] = v
       pkginfos[pk]['source'] = sp
       pkginfos[pk]['sourceversion'] = sv
+      pkginfos[pk]['sourceupstreamversion'] = sv
     assert(dpkg_show.wait() == 0)
     for pk in sorted(pkginfos.keys()):
       pi = pkginfos[pk]
@@ -253,6 +262,25 @@ class SourceShipmentPreparer():
         if s.show_pathnames: infol = infol + ['loaded='+fname]
         s.manifest_append_absentfile(' \t' + debfname, infol)
 
+      if s.download_packages:
+        try: os.mkdir(s._packages_path)
+        except FileExistsError: pass
+
+        cmdl = ['apt-get','--download-only','source',
+                '%s=%s' % (pi['source'], pi['sourceversion'])]
+        subprocess.run(cmdl,
+                       cwd=s._packages_path,
+                       stdin=subprocess.DEVNULL,
+                       stdout=sys.stdout,
+                       stderr=sys.stderr,
+                       restore_signals=True,
+                       check=True)
+
+        s._package_sources.append(dscfname)
+        dsc = debian.deb822.Dsc(open(s._packages_path + '/' + dscfname))
+        for indsc in dsc['Files']:
+          s._package_sources.append(indsc['name'])
+
   def thing_ought_packaged(s, fname):
     return s.thing_matches_globs(fname, s.src_package_globs)
 
@@ -300,26 +328,37 @@ class SourceShipmentPreparer():
     s.report_from_packages(s._package_files)
     s.logger('allitems done')
 
-  def mk_portmanteau(s):
-    s.logger('making portmanteau')
-    cmdl = s.rune_shell + [ s.rune_portmanteau, 'x',
-                            s.output_name, s.manifest_name ]
+  def _mk_portmanteau(s, ix, rune, cwd, files):
+    output_name = s.output_names[ix]
+    s.logger('making portmanteau %s' % output_name)
+    output_path = os.path.join(s._destdir, output_name)
+    subprocess.run(s.rune_shell + [ rune, 'x' ] + files,
+                   cwd=cwd,
+                   stdin=subprocess.DEVNULL,
+                   stdout=open(output_path, 'wb'),
+                   restore_signals=True,
+                   check=True)
+    s.output_paths[ix] = output_path
+
+  def mk_inner_portmanteau(s):
+    outputs = [s.manifest_name]
     mfh = s.open_output_fh(s.manifest_name,'w')
     for me in s._manifest:
       try: fname = me['file']
       except KeyError: fname = me.get('file_print','')
-      else: cmdl.append(fname)
+      else: outputs.append(fname)
       print('%s\t%s' % (fname, me['info']), file=mfh)
     mfh.close()
-    subprocess.run(cmdl,
-                   cwd=s._destdir,
-                   stdin=subprocess.DEVNULL,
-                   stdout=sys.stderr,
-                   restore_signals=True,
-                   check=True)
-    s.output_path = os.path.join(s._destdir, s.output_name)
-    s.logger('portmanteau ready in %s' % s.output_path)
+
+    s._mk_portmanteau(0, s.rune_portmanteau,
+                      s._destdir, outputs)
+
+  def mk_packages_portmanteau(s):
+    s._mk_portmanteau(1, s.rune_portmanteau_uncompressed,
+                      s._packages_path, s._package_sources)
 
   def generate(s):
     s.srcs_allitems()
-    s.mk_portmanteau()
+    s.mk_inner_portmanteau()
+    s.mk_packages_portmanteau()
+    s.logger('portmanteau ready in %s %s' % tuple(s.output_paths))