3 # Hippotat - Asinine IP Over HTTP program
4 # hippotatlib/ownsource.py - Automatic source code provision (AGPL compliance)
6 # Copyright 2017 Ian Jackson
10 # This program is free software: you can redistribute it and/or
11 # modify it under the terms of the GNU Affero General Public
12 # License as published by the Free Software Foundation, either
13 # version 3 of the License, or (at your option) any later version,
14 # with the "CAF Login Exception" as published by Ian Jackson
15 # (version 2, or at your option any later version) as an Additional
18 # This program is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
21 # Affero General Public License for more details.
23 # You should have received a copy of the GNU Affero General Public
24 # License and the CAF Login Exception along with this program, in
25 # the file AGPLv3+CAFv2. If not, email Ian Jackson
26 # <ijackson@chiark.greenend.org.uk>.
37 try: import debian.deb822
38 except ImportError: pass
40 class SourceShipmentPreparer():
41 def __init__(s, destdir):
42 # caller may modify, and should read after calling generate()
43 s.output_names = ['srcbomb.tar.gz', 'srcpkgsbomb.tar']
44 s.output_paths = [None,None] # alternatively caller may read this
45 # defaults, caller can modify after creation
46 s.logger = lambda m: print('SourceShipmentPreparer',m)
47 s.src_filter = s.src_filter_glob
48 s.src_package_globs = ['!/usr/local/*', '/usr*']
49 s.src_filter_globs = ['!/etc/*']
50 s.src_likeparent = s.src_likeparent_git
51 s.src_direxcludes = s.src_direxcludes_git
52 s.report_from_packages = s.report_from_packages_debian
54 s.find_rune_base = "find -type f -perm -004 \! -path '*/tmp/*'"
55 s.ignores = ['*~', '*.bak', '*.tmp', '#*#', '__pycache__',
56 '[0-9][0-9][0-9][0-9]-src.tar']
57 s.rune_shell = ['/bin/bash', '-ec']
58 s.show_pathnames = True
59 s.download_packages = True
60 s.stream_stderr = sys.stderr
61 s.stream_debug = open('/dev/null','w')
66 # ^ by default, is find ... -print0
68 cpio -Hustar -o --quiet -0 -R 1000:1000 || \
69 cpio -Hustar -o --quiet -0
72 s.rune_portmanteau = r'''
73 GZIP=-1 tar zcf - "$@"
75 s.rune_portmanteau_uncompressed = r'''
78 s.manifest_name='0000-MANIFEST.txt'
84 s._package_files = { } # map filename => infol
85 s._packages_path = os.path.join(s._destdir, 'packages')
86 s._package_sources = []
88 def thing_matches_globs(s, thing, globs):
90 negate = pat.startswith('!')
91 if negate: pat = pat[1:]
92 if fnmatch.fnmatch(thing, pat):
96 def src_filter_glob(s, src): # default s.src_filter
97 return s.thing_matches_globs(src, s.src_filter_globs)
99 def src_direxcludes_git(s, d):
101 excl = open(os.path.join(d, '.gitignore'))
102 except FileNotFoundError:
107 if l.startswith('#'): next
112 def src_likeparent_git(s, src):
114 os.stat(os.path.join(src, '.git/.'))
115 except FileNotFoundError:
120 def src_parentfinder(s, src, infol): # callers may monkey-patch away
121 for deref in (False,True):
126 search = os.path.realpath(search)
130 xinfo.append(os.path.basename(search))
131 search = os.path.dirname(search)
134 stab = os.lstat(search)
135 except FileNotFoundError:
137 if stat.S_ISREG(stab.st_mode):
140 while not os.path.ismount(search):
141 if s.src_likeparent(search):
143 if len(xinfo): infol.append('want=' + os.path.join(*xinfo))
148 # no .git found anywhere
151 def path_prenormaliser(s, d, infol): # callers may monkey-patch away
152 return os.path.join(s.cwd, os.path.abspath(d))
154 def srcdir_find_rune(s, d):
155 script = s.find_rune_base
156 ignores = s.ignores + s.output_names + [s.manifest_name]
157 ignores += s.src_direxcludes(d)
159 assert("'" not in excl)
160 script += r" \! -name '%s'" % excl
161 script += r" \! -path '*/%s/*'" % excl
165 def manifest_append(s, name, infol):
166 s._manifest.append({ 'file':name, 'info':' '.join(infol) })
168 def manifest_append_absentfile(s, name, infol):
169 s._manifest.append({ 'file_print':name, 'info':' '.join(infol) })
171 def new_output_name(s, nametail, infol):
173 name = '%04d-%s' % (s._outcounter, nametail)
174 s.manifest_append(name, infol)
177 def open_output_fh(s, name, mode):
178 return open(os.path.join(s._destdir, name), mode)
180 def src_dir(s, d, infol):
181 try: name = s._dirmap[d]
182 except KeyError: pass
184 s.manifest_append(name, infol)
187 if s.show_pathnames: infol.append(d)
188 find_rune = s.srcdir_find_rune(d)
189 total_rune = s.rune_cpio % find_rune
191 name = s.new_output_name('src.tar', infol)
193 fh = s.open_output_fh(name, 'wb')
195 s.logger('packing up into %s: %s (because %s)' %
196 (name, d, ' '.join(infol)))
198 subprocess.run(s.rune_shell + [total_rune],
200 stdin=subprocess.DEVNULL,
202 restore_signals=True,
206 def src_indir(s, d, infol):
207 d = s.path_prenormaliser(d, infol)
208 if not s.src_filter(d): return
210 d = s.src_parentfinder(d, infol)
214 def report_from_packages_debian(s, files):
215 dpkg_S_in = tempfile.TemporaryFile(mode='w+')
216 for (file, infols) in files.items():
217 assert('\n' not in file)
218 dpkg_S_in.write(file)
219 dpkg_S_in.write('\0')
221 cmdl = ['xargs','-0r','dpkg','-S','--']
222 dpkg_S = subprocess.Popen(cmdl,
225 stdout=subprocess.PIPE,
228 dpkg_show_in = tempfile.TemporaryFile(mode='w+')
230 for l in dpkg_S.stdout:
231 l = l.strip(b'\n').decode('utf-8')
232 (pkgs, fname) = l.split(': ',1)
233 pks = pkgs.split(', ')
235 pkginfos.setdefault(pk,{'files':[]})['files'].append(fname)
236 print(pk, file=dpkg_show_in)
237 assert(dpkg_S.wait() == 0)
239 cmdl = ['xargs','-r','dpkg-query',
240 r'-f${binary:Package}\t${Package}\t${Architecture}\t${Version}\t${source:Package}\t${source:Version}\t${source:Upstream-Version}\n',
242 dpkg_show = subprocess.Popen(cmdl,
245 stdout=subprocess.PIPE,
248 for l in dpkg_show.stdout:
249 l = l.strip(b'\n').decode('utf-8')
250 (pk,p,a,v,sp,sv,suv) = l.split('\t')
251 pkginfos[pk]['binary'] = p
252 pkginfos[pk]['arch'] = a
253 pkginfos[pk]['version'] = v
254 pkginfos[pk]['source'] = sp
255 pkginfos[pk]['sourceversion'] = sv
256 pkginfos[pk]['sourceupstreamversion'] = sv
257 assert(dpkg_show.wait() == 0)
258 for pk in sorted(pkginfos.keys()):
260 debfname = '%s_%s_%s.deb' % (pi['binary'], pi['version'], pi['arch'])
261 dscfname = '%s_%s.dsc' % (pi['source'], pi['sourceversion'])
262 s.manifest_append_absentfile(dscfname, [debfname])
263 s.logger('mentioning %s and %s because %s' %
264 (dscfname, debfname, pi['files'][0]))
265 for fname in pi['files']:
267 if s.show_pathnames: infol = infol + ['loaded='+fname]
268 s.manifest_append_absentfile(' \t' + debfname, infol)
270 if s.download_packages:
271 try: os.mkdir(s._packages_path)
272 except FileExistsError: pass
274 cmdl = ['apt-get','--download-only','source',
275 '%s=%s' % (pi['source'], pi['sourceversion'])]
277 cwd=s._packages_path,
278 stdin=subprocess.DEVNULL,
279 stdout=s.stream_debug,
280 stderr=s.stream_stderr,
281 restore_signals=True,
284 s._package_sources.append(dscfname)
285 dsc = debian.deb822.Dsc(open(s._packages_path + '/' + dscfname))
286 for indsc in dsc['Files']:
287 s._package_sources.append(indsc['name'])
289 def thing_ought_packaged(s, fname):
290 return s.thing_matches_globs(fname, s.src_package_globs)
292 def src_file_packaged(s, fname, infol):
293 s._package_files.setdefault(fname,[]).extend(infol)
295 def src_file(s, fname, infol):
298 infol_copy = infol.copy()
299 yield (infol_copy, s.path_prenormaliser(fname, infol_copy))
300 yield (infol, os.path.realpath(fname))
302 for (tinfol, tfname) in fngens():
303 if s.thing_ought_packaged(tfname):
304 s.src_file_packaged(tfname, tinfol)
307 s.src_indir(fname, infol)
309 def src_argv0(s, program, infol):
310 s.src_file(program, infol)
312 def src_syspath(s, fname, infol):
313 if s.thing_ought_packaged(fname): return
314 s.src_indir(fname, infol)
316 def src_module(s, m, infol):
317 try: fname = m.__file__
318 except AttributeError: return
319 infol.append('module='+m.__name__)
321 if s.thing_ought_packaged(fname):
322 s.src_file_packaged(fname, infol)
324 s.src_indir(fname, infol)
326 def srcs_allitems(s, dirs=sys.path):
328 s.src_argv0(sys.argv[0], ['argv[0]'])
330 s.src_syspath(d, ['sys.path'])
331 for m in sys.modules.values():
332 s.src_module(m, ['sys.modules'])
333 s.report_from_packages(s._package_files)
334 s.logger('allitems done')
336 def _mk_portmanteau(s, ix, rune, cwd, files):
337 output_name = s.output_names[ix]
338 s.logger('making portmanteau %s' % output_name)
339 output_path = os.path.join(s._destdir, output_name)
340 subprocess.run(s.rune_shell + [ rune, 'x' ] + files,
342 stdin=subprocess.DEVNULL,
343 stdout=open(output_path, 'wb'),
344 restore_signals=True,
346 s.output_paths[ix] = output_path
348 def mk_inner_portmanteau(s):
349 outputs = [s.manifest_name]
351 mfh = s.open_output_fh(s.manifest_name,'w')
352 for me in s._manifest:
353 try: fname = me['file']
354 except KeyError: fname = me.get('file_print','')
356 try: outputs_done[fname]
358 outputs.append(fname)
359 outputs_done[fname] = 1
360 print('%s\t%s' % (fname, me['info']), file=mfh)
363 s._mk_portmanteau(0, s.rune_portmanteau,
366 def mk_packages_portmanteau(s):
367 if not s.download_packages: return
368 s._mk_portmanteau(1, s.rune_portmanteau_uncompressed,
369 s._packages_path, s._package_sources)
373 s.mk_inner_portmanteau()
374 s.mk_packages_portmanteau()
375 s.logger('portmanteau ready in %s %s' % tuple(s.output_paths))