gbp/scripts/export_orig.py

   1 # vim: set fileencoding=utf-8 :
   2 #
   3 # (C) 2017 Guido Günther <agx@sigxcpu.org>
   4 #    This program is free software; you can redistribute it and/or modify
   5 #    it under the terms of the GNU General Public License as published by
   6 #    the Free Software Foundation; either version 2 of the License, or
   7 #    (at your option) any later version.
   8 #
   9 #    This program is distributed in the hope that it will be useful,
  10 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 #    GNU General Public License for more details.
  13 #
  14 #    You should have received a copy of the GNU General Public License
  15 #    along with this program; if not, please see
  16 #    <http://www.gnu.org/licenses/>
  17 #
  18 """Create orig tarballs from git"""
  19
  20 import os
  21 import sys
  22 import gbp.deb as du
  23 from gbp.command_wrappers import CommandExecFailed
  24 from gbp.config import (GbpOptionParserDebian, GbpOptionGroup)
  25 from gbp.deb.git import (GitRepositoryError, DebianGitRepository)
  26 from gbp.deb.source import DebianSource, DebianSourceError
  27 from gbp.errors import GbpError
  28 import gbp.log
  29 import gbp.notifications
  30 from gbp.scripts.common import ExitCodes
  31 from gbp.pkg import Compressor, Archive
  32
  33
  34 def prepare_upstream_tarballs(repo, source, options, tarball_dir, output_dir):
  35     """
  36     Make sure we have the needed upstream tarballs. The default order is:
  37     - look in tarball_dir and if found symlink to it
  38     - create tarball using pristine-tar
  39     - create tarball using git-archive
  40
  41     Afterwards
  42     - create pristine-tar commmits if pristine-tar-commit is in use
  43     - verify tarball checksums if pristine-tar is in use
  44     """
  45     if hasattr(options, 'no_create_orig') and options.no_create_orig:
  46         return
  47
  48     if not source.is_native() and not source.upstream_version:
  49         raise GbpError("Non-native package '%s' "
  50                        "has invalid version '%s'" % (source.name, source.version))
  51
  52     options.comp_type = guess_comp_type(options.comp_type,
  53                                         source,
  54                                         repo,
  55                                         options.tarball_dir)
  56     orig_files = source.upstream_tarball_names(options.comp_type, options.components)
  57
  58     # look in tarball_dir first, if found force a symlink to it
  59     if options.tarball_dir:
  60         gbp.log.debug("Looking for orig tarballs '%s' at '%s'" % (", ".join(orig_files), tarball_dir))
  61         missing = du.DebianPkgPolicy.symlink_origs(orig_files, tarball_dir, output_dir, force=True)
  62         if missing:
  63             msg = "Tarballs '%s' not found at '%s'" % (", ".join(missing), tarball_dir)
  64         else:
  65             msg = "All Orig tarballs '%s' found at '%s'" % (", ".join(orig_files), tarball_dir)
  66         gbp.log.info(msg)
  67
  68     # Create tarball if missing or forced
  69     if not du.DebianPkgPolicy.has_origs(orig_files, output_dir) or options.force_create:
  70         if not pristine_tar_build_origs(repo, source, output_dir, options):
  71             git_archive_build_origs(repo, source, output_dir, options)
  72     maybe_pristine_tar_commit(repo, source, options, output_dir, orig_files)
  73     pristine_tar_verify_origs(repo, source, options, output_dir, orig_files)
  74
  75
  76 def pristine_tar_prepare_orig_tree(repo, source, options):
  77     """
  78     Make sure the upstream tree exists
  79
  80     In case of component tarballs we need to recreate a tree for the
  81     main tarball without the component subdirs.
  82     """
  83     if options.components:
  84         try:
  85             upstream_tag = repo.version_to_tag(options.upstream_tag,
  86                                                source.upstream_version)
  87             tree_name = "%s^{tree}" % upstream_tag
  88             repo.tree_drop_dirs(tree_name, options.components)
  89         except GitRepositoryError:
  90             raise GbpError("Couldn't find upstream tree '%s' to create "
  91                            "orig tarball via pristine-tar" % tree_name)
  92
  93
  94 def pristine_tar_build_origs(repo, source, output_dir, options):
  95     """
  96     Build orig tarball using pristine-tar
  97
  98     @returns: C{True} if tarball was build, C{False} otherwise
  99     """
 100     if not options.pristine_tar:
 101         return False
 102
 103     if not repo.has_branch(repo.pristine_tar_branch):
 104         gbp.log.warn('Pristine-tar branch "%s" not found' %
 105                      repo.pristine_tar.branch)
 106
 107     comp = Compressor(options.comp_type)
 108     pristine_tar_prepare_orig_tree(repo, source, options)
 109     try:
 110         gbp.log.info("Creating %s" %
 111                      os.path.abspath(os.path.join(output_dir,
 112                                                   source.upstream_tarball_name(comp.type))))
 113         repo.create_upstream_tarball_via_pristine_tar(source,
 114                                                       output_dir,
 115                                                       comp)
 116         for component in options.components:
 117             gbp.log.info("Creating %s" %
 118                          os.path.abspath(os.path.join(output_dir,
 119                                                       source.upstream_tarball_name(comp.type, component))))
 120             repo.create_upstream_tarball_via_pristine_tar(source,
 121                                                           output_dir,
 122                                                           comp,
 123                                                           component=component)
 124         return True
 125     except GitRepositoryError:
 126         if hasattr(options, 'pristine_tar_commit') and options.pristine_tar_commit:
 127             gbp.log.debug("pristine-tar checkout failed, will commit tarball "
 128                           "due to '--pristine-tar-commit'")
 129         else:
 130             raise
 131     return False
 132
 133
 134 def pristine_tar_verify_origs(repo, source, options, output_dir, orig_files):
 135     """
 136     Verify orig tarballs using prstine tar
 137
 138     @returns: C{True} if tarball was build, C{False} otherwise
 139     """
 140     if not options.pristine_tar:
 141         return True
 142
 143     if not repo.pristine_tar.has_feature_verify():
 144         gbp.log.warn("pristine-tar does not support verify. "
 145                      "Skipping verification.")
 146         return True
 147
 148     pristine_tar_prepare_orig_tree(repo, source, options)
 149     for f in orig_files:
 150         repo.pristine_tar.verify(os.path.join(output_dir, f))
 151     return True
 152
 153
 154 def maybe_pristine_tar_commit(repo, source, options, output_dir, orig_files):
 155     if not (hasattr(options, 'pristine_tar_commit') and options.pristine_tar_commit):
 156         return
 157
 158     if repo.pristine_tar.has_commit(source.name,
 159                                     source.upstream_version,
 160                                     options.comp_type):
 161         gbp.log.debug("%s already on pristine tar branch" % orig_files[0])
 162     else:
 163         upstream_tree = git_archive_get_upstream_tree(repo, source, options)
 164         archive = os.path.join(output_dir, orig_files[0])
 165         gbp.log.debug("Adding %s to pristine-tar branch" % archive)
 166         repo.pristine_tar.commit(archive, upstream_tree)
 167
 168
 169 def git_archive_get_upstream_tree(repo, source, options):
 170     """
 171     Determine the upstream tree from the given options
 172
 173     for a git archive export
 174     """
 175     if options.upstream_tree.upper() == 'TAG':
 176         if source.upstream_version is None:
 177             raise GitRepositoryError("Can't determine upstream version from changelog")
 178         upstream_tree = repo.version_to_tag(options.upstream_tag,
 179                                             source.upstream_version)
 180     elif options.upstream_tree.upper() == 'BRANCH':
 181         if not repo.has_branch(options.upstream_branch):
 182             raise GbpError("%s is not a valid branch" % options.upstream_branch)
 183         upstream_tree = options.upstream_branch
 184     elif options.upstream_tree.upper() == 'SLOPPY':
 185         tree_name = "%s^{tree}" % options.debian_branch
 186         upstream_tree = repo.tree_drop_dirs(tree_name, ["debian"])
 187     else:
 188         upstream_tree = options.upstream_tree
 189     if not repo.has_treeish(upstream_tree):
 190         raise GbpError("%s is not a valid treeish" % upstream_tree)
 191     return upstream_tree
 192
 193
 194 def git_archive_build_origs(repo, source, output_dir, options):
 195     """
 196     Build orig tarball(s) using git-archive
 197
 198     @param source: the source of the package we're acting on
 199     @type source: L{DebianSource}
 200     @param output_dir: where to put the tarball
 201     @type output_dir: C{Str}
 202     @param options: the parsed options
 203     @type options: C{dict} of options
 204     """
 205     comp = Compressor(options.comp_type, options.comp_level)
 206     upstream_tree = git_archive_get_upstream_tree(repo, source, options)
 207     gbp.log.info("Creating %s from '%s'" % (source.upstream_tarball_name(comp.type),
 208                                             upstream_tree))
 209     gbp.log.debug("Building upstream tarball with compression %s" % comp)
 210     tree = repo.tree_drop_dirs(upstream_tree, options.components) if options.components else upstream_tree
 211     repo.create_upstream_tarball_via_git_archive(source, output_dir, tree, comp, options.with_submodules)
 212     for component in options.components:
 213         subtree = repo.tree_get_dir(upstream_tree, component)
 214         if not subtree:
 215             raise GbpError("No tree for '%s' found in '%s' to create additional tarball from"
 216                            % (component, upstream_tree))
 217         gbp.log.info("Creating additional tarball '%s' from '%s'"
 218                      % (source.upstream_tarball_name(options.comp_type, component=component),
 219                         subtree))
 220         repo.create_upstream_tarball_via_git_archive(source, output_dir, subtree, comp,
 221                                                      options.with_submodules, component=component)
 222
 223
 224 def guess_comp_type(comp_type, source, repo, tarball_dir):
 225     """Guess compression type to use for the to be built upstream tarball
 226
 227     We prefer pristine-tar over everything else since this is what's carried around with
 228     the repo and might be more reliable than what a user has in tarball_dir.
 229     """
 230     if comp_type != 'auto':
 231         comp_type = Compressor.Aliases.get(comp_type, comp_type)
 232         if comp_type not in Compressor.Opts:
 233             gbp.log.warn("Unknown compression type - guessing.")
 234             comp_type = 'auto'
 235
 236     if comp_type == 'auto':
 237         if repo and repo.has_pristine_tar_branch():
 238             regex = r'pristine-tar .* %s_%s\.orig.tar\.' % (source.name, source.upstream_version)
 239             commits = repo.grep_log(regex, repo.pristine_tar_branch, merges=False)
 240             if commits:
 241                 commit = commits[-1]
 242                 gbp.log.debug("Found pristine-tar commit at '%s'" % commit)
 243             else:
 244                 commit = repo.pristine_tar_branch
 245             tarball = repo.get_commit_info(commit)['subject']
 246             (base_name, archive_fmt, comp_type) = Archive.parse_filename(tarball)
 247             gbp.log.debug("Determined compression type '%s'" % comp_type)
 248             if not comp_type:
 249                 comp_type = 'gzip'
 250                 gbp.log.warn("Unknown compression type of %s, assuming %s" % (tarball, comp_type))
 251         else:
 252             if not tarball_dir:
 253                 tarball_dir = '..'
 254             detected = None
 255             for comp in Compressor.Opts.keys():
 256                 if du.DebianPkgPolicy.has_orig(source.upstream_tarball_name(comp), tarball_dir):
 257                     if detected is not None:
 258                         raise GbpError("Multiple orig tarballs found.")
 259                     detected = comp
 260             comp_type = 'gzip' if detected is None else detected
 261     return comp_type
 262
 263
 264 def build_parser(name):
 265     try:
 266         parser = GbpOptionParserDebian(command=os.path.basename(name), prefix='')
 267     except GbpError as err:
 268         gbp.log.err(err)
 269         return None
 270
 271     tag_group = GbpOptionGroup(parser,
 272                                "tag options",
 273                                "options related to git tag creation")
 274     orig_group = GbpOptionGroup(parser,
 275                                 "orig tarball options",
 276                                 "options related to the creation of the orig tarball")
 277     branch_group = GbpOptionGroup(parser,
 278                                   "branch options",
 279                                   "branch layout options")
 280     for group in [tag_group, orig_group, branch_group]:
 281         parser.add_option_group(group)
 282
 283     parser.add_option("--verbose", action="store_true", dest="verbose", default=False,
 284                       help="verbose command execution")
 285     parser.add_config_file_option(option_name="color", dest="color", type='tristate')
 286     parser.add_config_file_option(option_name="color-scheme",
 287                                   dest="color_scheme")
 288     tag_group.add_config_file_option(option_name="upstream-tag", dest="upstream_tag")
 289     orig_group.add_config_file_option(option_name="upstream-tree", dest="upstream_tree")
 290     orig_group.add_boolean_config_file_option(option_name="pristine-tar", dest="pristine_tar")
 291     orig_group.add_config_file_option(option_name="force-create", dest="force_create",
 292                                       help="force creation of orig tarball", action="store_true")
 293     orig_group.add_config_file_option(option_name="tarball-dir", dest="tarball_dir", type="path",
 294                                       help="location to look for external tarballs")
 295     orig_group.add_config_file_option(option_name="compression", dest="comp_type",
 296                                       help="Compression type, default is '%(compression)s'")
 297     orig_group.add_config_file_option(option_name="compression-level", dest="comp_level",
 298                                       help="Compression level, default is '%(compression-level)s'")
 299     orig_group.add_config_file_option("component", action="append", metavar='COMPONENT',
 300                                       dest="components")
 301     branch_group.add_config_file_option(option_name="upstream-branch", dest="upstream_branch")
 302     branch_group.add_boolean_config_file_option(option_name="submodules", dest="with_submodules")
 303     return parser
 304
 305
 306 def parse_args(argv, prefix):
 307     parser = build_parser(argv[0])
 308     if not parser:
 309         return None, None
 310     options, args = parser.parse_args(argv[1:])
 311
 312     gbp.log.setup(options.color, options.verbose, options.color_scheme)
 313     return options, args
 314
 315
 316 def main(argv):
 317     retval = 0
 318     source = None
 319
 320     options, args = parse_args(argv, '')
 321
 322     if args or not options:
 323         return ExitCodes.parse_error
 324
 325     try:
 326         repo = DebianGitRepository(os.path.curdir, toplevel=False)
 327     except GitRepositoryError:
 328         gbp.log.err("%s is not inside a git repository" % (os.path.abspath('.')))
 329         return 1
 330
 331     try:
 332         try:
 333             source = DebianSource(repo.path)
 334             source.is_native()
 335         except Exception as e:
 336             raise GbpError("Can't determine package type: %s" % e)
 337
 338         output_dir = options.tarball_dir or os.path.join(repo.path, '..')
 339
 340         if source.is_native():
 341             gbp.log.info("Nothing to be done for native package")
 342             return 0
 343
 344         prepare_upstream_tarballs(repo, source, options, output_dir,
 345                                   output_dir)
 346     except KeyboardInterrupt:
 347         retval = 1
 348         gbp.log.err("Interrupted. Aborting.")
 349     except CommandExecFailed:
 350         retval = 1
 351     except (GbpError, GitRepositoryError) as err:
 352         if str(err):
 353             gbp.log.err(err)
 354         retval = 1
 355     except DebianSourceError as err:
 356         gbp.log.err(err)
 357         source = None
 358         retval = 1
 359
 360     return retval
 361
 362
 363 if __name__ == '__main__':
 364     sys.exit(main(sys.argv))
 365
 366 # vim:et:ts=4:sw=4:et:sts=4:ai:set list listchars=tab\:»·,trail\:·: