gbp/scripts/pq.py

   1 # vim: set fileencoding=utf-8 :
   2 #
   3 # (C) 2011,2014,2017 Guido Günther <agx@sigxcpu.org>
   4 #    This program is free software; you can redistribute it and/or modify
   5 #    it under the terms of the GNU General Public License as published by
   6 #    the Free Software Foundation; either version 2 of the License, or
   7 #    (at your option) any later version.
   8 #
   9 #    This program is distributed in the hope that it will be useful,
  10 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 #    GNU General Public License for more details.
  13 #
  14 #    You should have received a copy of the GNU General Public License
  15 #    along with this program; if not, please see
  16 #    <http://www.gnu.org/licenses/>
  17 #
  18 """Manage Debian patches on a patch queue branch"""
  19
  20 import errno
  21 import os
  22 import shutil
  23 import sys
  24 import tempfile
  25 import re
  26 from gbp.config import GbpOptionParserDebian
  27 from gbp.deb.source import DebianSource
  28 from gbp.deb.git import DebianGitRepository
  29 from gbp.git import GitRepositoryError
  30 from gbp.command_wrappers import (GitCommand, CommandExecFailed)
  31 from gbp.errors import GbpError
  32 import gbp.log
  33 from gbp.patch_series import (PatchSeries, Patch)
  34 from gbp.scripts.common.pq import (is_pq_branch, pq_branch_name, pq_branch_base,
  35                                    parse_gbp_commands, format_patch,
  36                                    apply_single_patch,
  37                                    apply_and_commit_patch,
  38                                    drop_pq, get_maintainer_from_control,
  39                                    switch_to_pq_branch)
  40 from gbp.scripts.common import ExitCodes
  41 from gbp.dch import extract_bts_cmds
  42
  43 PATCH_DIR = "debian/patches/"
  44 SERIES_FILE = os.path.join(PATCH_DIR, "series")
  45
  46
  47 def parse_old_style_topic(commit_info):
  48     """Parse 'gbp-pq-topic:' line(s) from commit info"""
  49
  50     commit = commit_info['id']
  51     topic_regex = r'gbp-pq-topic:\s*(?P<topic>\S.*)'
  52     mangled_body = ''
  53     topic = ''
  54     # Parse and filter commit message body
  55     for line in commit_info['body'].splitlines():
  56         match = re.match(topic_regex, line, flags=re.I)
  57         if match:
  58             topic = match.group('topic')
  59             gbp.log.debug("Topic %s found for %s" % (topic, commit))
  60             gbp.log.warn("Deprecated 'gbp-pq-topic: <topic>' in %s, please "
  61                          "use 'Gbp[-Pq]: Topic <topic>' instead" % commit)
  62             continue
  63         mangled_body += line + '\n'
  64     commit_info['body'] = mangled_body
  65     return topic
  66
  67
  68 def generate_patches(repo, start, end, outdir, options):
  69     """
  70     Generate patch files from git
  71     """
  72     gbp.log.info("Generating patches from git (%s..%s)" % (start, end))
  73     patches = []
  74     for treeish in [start, end]:
  75         if not repo.has_treeish(treeish):
  76             raise GbpError('%s not a valid tree-ish' % treeish)
  77
  78     # Generate patches
  79     rev_list = reversed(repo.get_commits(start, end))
  80     for commit in rev_list:
  81         info = repo.get_commit_info(commit)
  82         # Parse 'gbp-pq-topic:'
  83         topic = parse_old_style_topic(info)
  84         cmds = {'topic': topic} if topic else {}
  85         # Parse 'Gbp: ' style commands
  86         (cmds_gbp, info['body']) = parse_gbp_commands(info, 'gbp',
  87                                                       ('ignore'),
  88                                                       ('topic', 'name'),
  89                                                       ('topic', 'name'))
  90         cmds.update(cmds)
  91         # Parse 'Gbp-Pq: ' style commands
  92         (cmds_gbp_pq, info['body']) = parse_gbp_commands(info,
  93                                                          'gbp-pq',
  94                                                          ('ignore'),
  95                                                          ('topic', 'name'),
  96                                                          ('topic', 'name'))
  97         cmds.update(cmds_gbp_pq)
  98         if 'ignore' not in cmds:
  99             if 'topic' in cmds:
 100                 topic = cmds['topic']
 101             name = cmds.get('name', None)
 102             format_patch(outdir, repo, info, patches, options.abbrev,
 103                          numbered=options.patch_numbers,
 104                          topic=topic, name=name,
 105                          renumber=options.renumber,
 106                          patch_num_prefix_format=options.patch_num_format)
 107         else:
 108             gbp.log.info('Ignoring commit %s' % info['id'])
 109
 110     return patches
 111
 112
 113 def compare_series(old, new):
 114     """
 115     Compare new pathes to lists of patches already exported
 116
 117     >>> compare_series(['# comment', 'a', 'b'], ['b', 'c'])
 118     (['c'], ['a'])
 119     >>> compare_series([], [])
 120     ([], [])
 121     """
 122     added = set(new).difference(old)
 123     removed = [l for l in set(old).difference(new) if not l.startswith('#')]
 124     return (list(added), removed)
 125
 126
 127 def format_series_diff(added, removed, options):
 128     """
 129     Format the patch differences into a suitable commit message
 130
 131     >>> format_series_diff(['a'], ['b'], None)
 132     'Rediff patches\\n\\nAdd a: <REASON>\\nDrop b: <REASON>\\n'
 133     """
 134     if len(added) == 1 and not removed:
 135         # Single patch added, create a more thorough commit message
 136         patch = Patch(os.path.join('debian', 'patches', added[0]))
 137         msg = patch.subject
 138         bugs, dummy = extract_bts_cmds(patch.long_desc.split('\n'), options)
 139         if bugs:
 140             msg += '\n'
 141             for k, v in bugs.items():
 142                 msg += '\n%s: %s' % (k, ', '.join(v))
 143     else:
 144         msg = "Rediff patches\n\n"
 145         for p in added:
 146             msg += 'Add %s: <REASON>\n' % p
 147         for p in removed:
 148             msg += 'Drop %s: <REASON>\n' % p
 149     return msg
 150
 151
 152 def commit_patches(repo, branch, patches, options, patch_dir):
 153     """
 154     Commit chanages exported from patch queue
 155     """
 156     clean, dummy = repo.is_clean()
 157     if clean:
 158         return ([], [])
 159
 160     vfs = gbp.git.vfs.GitVfs(repo, branch)
 161     try:
 162         with vfs.open('debian/patches/series') as oldseries:
 163             oldpatches = [p.strip() for p in oldseries.readlines()]
 164     except IOError:
 165         # No series file yet
 166         oldpatches = []
 167     newpatches = [p[len(patch_dir):] for p in patches]
 168
 169     # FIXME: handle case were only the contents of the patches changed
 170     added, removed = compare_series(oldpatches, newpatches)
 171     msg = format_series_diff(added, removed, options)
 172
 173     if not repo.is_clean(paths='debian/patches')[0]:
 174         repo.add_files(PATCH_DIR, force=True)
 175         repo.commit_staged(msg=msg)
 176     return added, removed
 177
 178
 179 def find_upstream_commit(repo, branch, upstream_tag):
 180     """
 181     Find commit corresponding to upstream version based on changelog
 182     """
 183     vfs = gbp.git.vfs.GitVfs(repo, pq_branch_base(branch))
 184     cl = DebianSource(vfs).changelog
 185     upstream_commit = repo.find_version(upstream_tag, cl.upstream_version)
 186     if not upstream_commit:
 187         raise GbpError("Couldn't find upstream version %s" %
 188                        cl.upstream_version)
 189     return upstream_commit
 190
 191
 192 def pq_on_upstream_tag(pq_from):
 193     """Return True if the patch queue is based on the uptream tag,
 194     False if its based on the debian packaging branch"""
 195     return True if pq_from.upper() == 'TAG' else False
 196
 197
 198 def export_patches(repo, branch, options):
 199     """Export patches from the pq branch into a patch series"""
 200     patch_dir = os.path.join(repo.path, PATCH_DIR)
 201     series_file = os.path.join(repo.path, SERIES_FILE)
 202     if is_pq_branch(branch):
 203         base = pq_branch_base(branch)
 204         gbp.log.info("On '%s', switching to '%s'" % (branch, base))
 205         branch = base
 206         repo.set_branch(branch)
 207
 208     pq_branch = pq_branch_name(branch)
 209     try:
 210         shutil.rmtree(patch_dir)
 211     except OSError as e:
 212         if e.errno != errno.ENOENT:
 213             raise GbpError("Failed to remove patch dir: %s" % e.strerror)
 214         else:
 215             gbp.log.debug("%s does not exist." % patch_dir)
 216
 217     if pq_on_upstream_tag(options.pq_from):
 218         base = find_upstream_commit(repo, branch, options.upstream_tag)
 219     else:
 220         base = branch
 221
 222     patches = generate_patches(repo, base, pq_branch, patch_dir, options)
 223
 224     if patches:
 225         with open(series_file, 'w') as seriesfd:
 226             for patch in patches:
 227                 seriesfd.write(os.path.relpath(patch, patch_dir) + '\n')
 228     else:
 229         gbp.log.info("No patches on '%s' - nothing to export." % pq_branch)
 230
 231     if options.commit:
 232         added, removed = commit_patches(repo, branch, patches, options, patch_dir)
 233         if added:
 234             what = 'patches' if len(added) > 1 else 'patch'
 235             gbp.log.info("Added %s %s to patch series" % (what, ', '.join(added)))
 236         if removed:
 237             what = 'patches' if len(removed) > 1 else 'patch'
 238             gbp.log.info("Removed %s %s from patch series" % (what, ', '.join(removed)))
 239         else:
 240             gbp.log.info("Updated existing patches.")
 241
 242     if options.drop:
 243         drop_pq(repo, branch)
 244
 245
 246 def safe_patches(series, repo):
 247     """
 248     Safe the current patches in a temporary directory
 249     below .git/
 250
 251     @param series: path to series file
 252     @return: tmpdir and path to safed series file
 253     @rtype: tuple
 254     """
 255
 256     src = os.path.dirname(series)
 257     name = os.path.basename(series)
 258
 259     tmpdir = tempfile.mkdtemp(dir=repo.git_dir, prefix='gbp-pq')
 260     patches = os.path.join(tmpdir, 'patches')
 261     series = os.path.join(patches, name)
 262
 263     gbp.log.debug("Safeing patches '%s' in '%s'" % (src, tmpdir))
 264     shutil.copytree(src, patches)
 265
 266     return (tmpdir, series)
 267
 268
 269 def import_quilt_patches(repo, branch, series, tries, force, pq_from,
 270                          upstream_tag):
 271     """
 272     apply a series of quilt patches in the series file 'series' to branch
 273     the patch-queue branch for 'branch'
 274
 275     @param repo: git repository to work on
 276     @param branch: branch to base patch queue on
 277     @param series: series file to read patches from
 278     @param tries: try that many times to apply the patches going back one
 279                   commit in the branches history after each failure.
 280     @param force: import the patch series even if the branch already exists
 281     @param pq_from: what to use as the starting point for the pq branch.
 282                     DEBIAN indicates the current branch, TAG indicates that
 283                     the corresponding upstream tag should be used.
 284     @param upstream_tag: upstream tag template to use
 285     """
 286     tmpdir = None
 287     series = os.path.join(repo.path, series)
 288
 289     if is_pq_branch(branch):
 290         if force:
 291             branch = pq_branch_base(branch)
 292             pq_branch = pq_branch_name(branch)
 293             repo.checkout(branch)
 294         else:
 295             raise GbpError("Already on a patch-queue branch '%s' - doing nothing." % branch)
 296     else:
 297         pq_branch = pq_branch_name(branch)
 298
 299     if repo.has_branch(pq_branch):
 300         if force:
 301             drop_pq(repo, branch)
 302         else:
 303             raise GbpError("Patch queue branch '%s'. already exists. Try 'rebase' or 'switch' instead."
 304                            % pq_branch)
 305
 306     maintainer = get_maintainer_from_control(repo)
 307     if pq_on_upstream_tag(pq_from):
 308         commits = [find_upstream_commit(repo, branch, upstream_tag)]
 309     else:  # pq_from == 'DEBIAN'
 310         commits = repo.get_commits(num=tries, first_parent=True)
 311     # If we go back in history we have to safe our pq so we always try to apply
 312     # the latest one
 313     # If we are using the upstream_tag, we always need a copy of the patches
 314     if len(commits) > 1 or pq_on_upstream_tag(pq_from):
 315         if os.path.exists(series):
 316             tmpdir, series = safe_patches(series, repo)
 317
 318     queue = PatchSeries.read_series_file(series)
 319
 320     i = len(commits)
 321     for commit in commits:
 322         if len(commits) > 1:
 323             gbp.log.info("%d %s left" % (i, 'tries' if i > 1 else 'try'))
 324         try:
 325             gbp.log.info("Trying to apply patches at '%s'" % commit)
 326             repo.create_branch(pq_branch, commit)
 327         except GitRepositoryError:
 328             raise GbpError("Cannot create patch-queue branch '%s'." % pq_branch)
 329
 330         repo.set_branch(pq_branch)
 331         for patch in queue:
 332             gbp.log.debug("Applying %s" % patch.path)
 333             try:
 334                 name = os.path.basename(patch.path)
 335                 apply_and_commit_patch(repo, patch, maintainer, patch.topic, name)
 336             except (GbpError, GitRepositoryError) as e:
 337                 gbp.log.err("Failed to apply '%s': %s" % (patch.path, e))
 338                 repo.force_head('HEAD', hard=True)
 339                 repo.set_branch(branch)
 340                 repo.delete_branch(pq_branch)
 341                 break
 342         else:
 343             # All patches applied successfully
 344             break
 345         i -= 1
 346     else:
 347         raise GbpError("Couldn't apply patches")
 348
 349     if tmpdir:
 350         gbp.log.debug("Remove temporary patch safe '%s'" % tmpdir)
 351         shutil.rmtree(tmpdir)
 352
 353     return len(queue)
 354
 355
 356 def rebase_pq(repo, branch, options):
 357     maybe_import_pq(repo, branch, options)
 358     # Make sure we're on the pq branch
 359     switch_to_pq_branch(repo, branch)
 360     if pq_on_upstream_tag(options.pq_from):
 361         base = find_upstream_commit(repo, branch, options.upstream_tag)
 362     else:
 363         base = pq_branch_base(repo.branch)
 364
 365     GitCommand("rebase", cwd=repo.path)([base])
 366
 367
 368 def import_pq(repo, branch, options):
 369     """Import quilt patches onto pq branch"""
 370     series = SERIES_FILE
 371     tries = options.time_machine if (options.time_machine > 0) else 1
 372     num = import_quilt_patches(repo, branch, series, tries,
 373                                options.force, options.pq_from,
 374                                options.upstream_tag)
 375     gbp.log.info("%d patches listed in '%s' imported on '%s'" %
 376                  (num, series, repo.get_branch()))
 377
 378
 379 def maybe_import_pq(repo, branch, options):
 380     """Import quilt patches onto pq branch if pq branch does not exist yet"""
 381     if not repo.has_branch(pq_branch_name(branch)):
 382         gbp.log.info("No pq branch found, importing patches")
 383         import_pq(repo, branch, options)
 384         return True
 385     return False
 386
 387
 388 def switch_pq(repo, branch, options):
 389     """Switch to patch-queue branch if on base branch and vice versa"""
 390     if is_pq_branch(branch):
 391         base = pq_branch_base(branch)
 392         gbp.log.info("Switching to %s" % base)
 393         repo.checkout(base)
 394     else:
 395         maybe_import_pq(repo, branch, options)
 396         switch_to_pq_branch(repo, branch)
 397
 398
 399 def usage_msg():
 400     return """%prog [options] action - maintain patches on a patch queue branch
 401 Actions:
 402   export         export the patch queue associated to the current branch
 403                  into a quilt patch series in debian/patches/ and update the
 404                  series file.
 405   import         create a patch queue branch from quilt patches in debian/patches.
 406   rebase         switch to patch queue branch associated to the current
 407                  branch and rebase against current branch.
 408   drop           drop (delete) the patch queue associated to the current branch.
 409   apply          apply a patch
 410   switch         switch to patch-queue branch and vice versa"""
 411
 412
 413 def build_parser(name):
 414     try:
 415         parser = GbpOptionParserDebian(command=os.path.basename(name),
 416                                        usage=usage_msg())
 417     except GbpError as err:
 418         gbp.log.err(err)
 419         return None
 420
 421     parser.add_boolean_config_file_option(option_name="patch-numbers", dest="patch_numbers")
 422     parser.add_config_file_option(option_name="patch-num-format", dest="patch_num_format")
 423     parser.add_boolean_config_file_option(option_name="renumber", dest="renumber")
 424     parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
 425                       help="verbose command execution")
 426     parser.add_option("--topic", dest="topic", help="in case of 'apply' topic (subdir) to put patch into")
 427     parser.add_config_file_option(option_name="time-machine", dest="time_machine", type="int")
 428     parser.add_boolean_config_file_option("drop", dest='drop')
 429     parser.add_boolean_config_file_option(option_name="commit", dest="commit")
 430     parser.add_config_file_option(option_name="abbrev", dest="abbrev", type="int")
 431     parser.add_option("--force", dest="force", action="store_true", default=False,
 432                       help="in case of import even import if the branch already exists")
 433     parser.add_config_file_option(option_name="color", dest="color", type='tristate')
 434     parser.add_config_file_option(option_name="color-scheme",
 435                                   dest="color_scheme")
 436     parser.add_config_file_option(option_name="meta-closes", dest="meta_closes")
 437     parser.add_config_file_option(option_name="meta-closes-bugnum", dest="meta_closes_bugnum")
 438     parser.add_config_file_option(option_name="pq-from", dest="pq_from", choices=['DEBIAN', 'TAG'])
 439     parser.add_config_file_option(option_name="upstream-tag", dest="upstream_tag")
 440     return parser
 441
 442
 443 def parse_args(argv):
 444     parser = build_parser(argv[0])
 445     if not parser:
 446         return None, None
 447     return parser.parse_args(argv)
 448
 449
 450 def main(argv):
 451     retval = 0
 452
 453     (options, args) = parse_args(argv)
 454     if not options:
 455         return ExitCodes.parse_error
 456
 457     gbp.log.setup(options.color, options.verbose, options.color_scheme)
 458
 459     if len(args) < 2:
 460         gbp.log.err("No action given.")
 461         return 1
 462     else:
 463         action = args[1]
 464
 465     if args[1] in ["export", "import", "rebase", "drop", "switch"]:
 466         pass
 467     elif args[1] in ["apply"]:
 468         if len(args) != 3:
 469             gbp.log.err("No patch name given.")
 470             return 1
 471         else:
 472             patchfile = args[2]
 473     else:
 474         gbp.log.err("Unknown action '%s'." % args[1])
 475         return 1
 476
 477     try:
 478         repo = DebianGitRepository(os.path.curdir)
 479     except GitRepositoryError:
 480         gbp.log.err("%s is not a git repository" % (os.path.abspath('.')))
 481         return 1
 482
 483     try:
 484         current = repo.get_branch()
 485         if action == "export":
 486             export_patches(repo, current, options)
 487         elif action == "import":
 488             import_pq(repo, current, options)
 489         elif action == "drop":
 490             drop_pq(repo, current)
 491         elif action == "rebase":
 492             rebase_pq(repo, current, options)
 493         elif action == "apply":
 494             patch = Patch(patchfile)
 495             maintainer = get_maintainer_from_control(repo)
 496             apply_single_patch(repo, current, patch, maintainer, options.topic)
 497         elif action == "switch":
 498             switch_pq(repo, current, options)
 499     except KeyboardInterrupt:
 500         retval = 1
 501         gbp.log.err("Interrupted. Aborting.")
 502     except CommandExecFailed:
 503         retval = 1
 504     except (GbpError, GitRepositoryError) as err:
 505         if str(err):
 506             gbp.log.err(err)
 507         retval = 1
 508
 509     return retval
 510
 511
 512 if __name__ == '__main__':
 513     sys.exit(main(sys.argv))