gbp/scripts/dch.py

   1 # vim: set fileencoding=utf-8 :
   2 #
   3 # (C) 2007,2008,2009,2010,2013,2015,2017 Guido Günther <agx@sigxcpu.org>
   4 #    This program is free software; you can redistribute it and/or modify
   5 #    it under the terms of the GNU General Public License as published by
   6 #    the Free Software Foundation; either version 2 of the License, or
   7 #    (at your option) any later version.
   8 #
   9 #    This program is distributed in the hope that it will be useful,
  10 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 #    GNU General Public License for more details.
  13 #
  14 #    You should have received a copy of the GNU General Public License
  15 #    along with this program; if not, please see
  16 #    <http://www.gnu.org/licenses/>
  17 #
  18 """Generate Debian changelog entries from Git commit messages"""
  19
  20 import os.path
  21 import re
  22 import sys
  23 import shutil
  24 import gbp.command_wrappers as gbpc
  25 import gbp.dch as dch
  26 import gbp.log
  27 from gbp.config import GbpOptionParserDebian, GbpOptionGroup
  28 from gbp.errors import GbpError
  29 from gbp.deb import compare_versions
  30 from gbp.deb.source import DebianSource, DebianSourceError
  31 from gbp.deb.git import GitRepositoryError, DebianGitRepository
  32 from gbp.deb.changelog import ChangeLog, NoChangeLogError
  33 from gbp.scripts.common import ExitCodes, maybe_debug_raise
  34 from gbp.scripts.common.hook import Hook
  35
  36 user_customizations = {}
  37 snapshot_re = re.compile(r"\s*\*\* SNAPSHOT build @(?P<commit>[a-z0-9]+)\s+\*\*")
  38
  39
  40 def guess_version_from_upstream(repo, upstream_tag_format, upstream_branch, cp=None):
  41     """
  42     Guess the version based on the latest version on the upstream branch.
  43     If the version in dch is already higher this function returns None.
  44     """
  45     epoch = cp.epoch if cp else None
  46     cmp_version = cp.version if cp else '0~'
  47     try:
  48         version = repo.debian_version_from_upstream(upstream_tag_format,
  49                                                     upstream_branch,
  50                                                     epoch=epoch,
  51                                                     debian_release=False)
  52         gbp.log.debug("Found upstream version %s." % version)
  53         if compare_versions(version, cmp_version) > 0:
  54             return "%s-1" % version
  55     except GitRepositoryError as e:
  56         gbp.log.debug("No upstream tag found: %s" % e)
  57     return None
  58
  59
  60 def get_author_email(repo, use_git_config):
  61     """Get author and email from git configuration"""
  62     author = email = None
  63
  64     if use_git_config:
  65         try:
  66             author = repo.get_config('user.name')
  67         except KeyError:
  68             pass
  69
  70         try:
  71             email = repo.get_config('user.email')
  72         except KeyError:
  73             pass
  74     return author, email
  75
  76
  77 def fixup_section(repo, use_git_author, options, dch_options):
  78     """
  79     Fixup the changelog header and trailer's committer and email address
  80
  81     It might otherwise point to the last git committer instead of the person
  82     creating the changelog
  83
  84     This also applies --distribution and --urgency options passed to gbp dch
  85     """
  86     author, email = get_author_email(repo, use_git_author)
  87     used_options = ['distribution', 'urgency']
  88     opts = []
  89     mainttrailer_opts = ['--nomainttrailer', '--mainttrailer', '-t']
  90
  91     # This must not be done for snapshots or snapshots changelog entries
  92     # will not be concatenated
  93     if not options.snapshot:
  94         for opt in used_options:
  95             val = getattr(options, opt)
  96             if val:
  97                 gbp.log.debug("Set header option '%s' to '%s'" % (opt, val))
  98                 opts.append("--%s=%s" % (opt, val))
  99     else:
 100         gbp.log.debug("Snapshot enabled: do not fixup options in header")
 101
 102     for opt in mainttrailer_opts:
 103         if opt in dch_options:
 104             break
 105     else:
 106         opts.append(mainttrailer_opts[0])
 107     ChangeLog.spawn_dch(msg='', author=author, email=email, dch_options=dch_options + opts)
 108
 109
 110 def snapshot_version(version):
 111     """
 112     Get the current release and snapshot version.
 113
 114     Format is <debian-version>~<release>.gbp<short-commit-id>
 115
 116     >>> snapshot_version('1.0-1')
 117     ('1.0-1', 0)
 118     >>> snapshot_version('1.0-1~1.test0')
 119     ('1.0-1~1.test0', 0)
 120     >>> snapshot_version('1.0-1~2.gbp1234')
 121     ('1.0-1', 2)
 122     """
 123     try:
 124         (release, suffix) = version.rsplit('~', 1)
 125         (snapshot, commit) = suffix.split('.', 1)
 126         if not commit.startswith('gbp'):
 127             raise ValueError
 128         else:
 129             snapshot = int(snapshot)
 130     except ValueError:  # not a snapshot release
 131         release = version
 132         snapshot = 0
 133     return release, snapshot
 134
 135
 136 def mangle_changelog(changelog, cp, snapshot=''):
 137     """
 138     Mangle changelog to either add or remove snapshot markers
 139
 140     @param snapshot: SHA1 if snapshot header should be added/maintained,
 141         empty if it should be removed
 142     @type  snapshot: C{str}
 143     """
 144     try:
 145         tmpfile = '%s.%s' % (changelog, snapshot)
 146         cw = open(tmpfile, 'w', encoding='utf-8')
 147         cr = open(changelog, 'r', encoding='utf-8')
 148
 149         print("%(Source)s (%(MangledVersion)s) "
 150               "%(Distribution)s; urgency=%(urgency)s\n" % cp, file=cw)
 151
 152         cr.readline()  # skip version and empty line
 153         cr.readline()
 154         line = cr.readline()
 155         if snapshot_re.match(line):
 156             cr.readline()  # consume the empty line after the snapshot header
 157             line = ''
 158
 159         if snapshot:
 160             print("  ** SNAPSHOT build @%s **\n" % snapshot, file=cw)
 161
 162         if line:
 163             print(line.rstrip(), file=cw)
 164         shutil.copyfileobj(cr, cw)
 165         cw.close()
 166         cr.close()
 167         os.unlink(changelog)
 168         os.rename(tmpfile, changelog)
 169     except OSError as e:
 170         raise GbpError("Error mangling changelog %s" % e)
 171
 172
 173 def do_release(changelog, repo, cp, use_git_author, dch_options):
 174     """Remove the snapshot header and set the distribution"""
 175     author, email = get_author_email(repo, use_git_author)
 176     (release, snapshot) = snapshot_version(cp['Version'])
 177     if snapshot:
 178         cp['MangledVersion'] = release
 179         mangle_changelog(changelog, cp)
 180     cp.spawn_dch(release=True, author=author, email=email, dch_options=dch_options)
 181
 182
 183 def do_snapshot(changelog, repo, next_snapshot):
 184     """
 185     Add new snapshot banner to most recent changelog section.
 186     The next snapshot number is calculated by eval()'ing next_snapshot.
 187     """
 188     commit = repo.head
 189
 190     cp = ChangeLog(filename=changelog)
 191     (release, snapshot) = snapshot_version(cp['Version'])
 192     snapshot = int(eval(next_snapshot))
 193
 194     suffix = "%d.gbp%s" % (snapshot, "".join(commit[0:6]))
 195     cp['MangledVersion'] = "%s~%s" % (release, suffix)
 196
 197     mangle_changelog(changelog, cp, commit)
 198     return snapshot, commit, cp['MangledVersion']
 199
 200
 201 def parse_commit(repo, commitid, opts, last_commit=False):
 202     """Parse a commit and return message, author, and author email"""
 203     commit_info = repo.get_commit_info(commitid)
 204     author = commit_info['author'].name
 205     email = commit_info['author'].email
 206     format_entry = user_customizations.get('format_changelog_entry')
 207     if not format_entry:
 208         format_entry = dch.format_changelog_entry
 209     entry = format_entry(commit_info, opts, last_commit=last_commit)
 210     return entry, (author, email)
 211
 212
 213 def guess_documented_commit(cp, repo, tagformat):
 214     """
 215     Guess the last commit documented in the changelog from the snapshot banner,
 216     the last tagged version or the last point the changelog was touched.
 217
 218     @param cp: the changelog
 219     @param repo: the git repository
 220     @param tagformat: the format for Debian tags
 221     @returns: the commit that was last documented in the changelog
 222     @rtype: C{str}
 223     @raises GbpError: In case we fail to find a commit to start at
 224     """
 225     # Check for snapshot banner
 226     sr = re.search(snapshot_re, cp['Changes'])
 227     if sr:
 228         return sr.group('commit')
 229
 230     # Check if the latest version in the changelog is already tagged. If
 231     # so this is the last documented commit.
 232     commit = repo.find_version(tagformat, cp.version)
 233     if commit:
 234         gbp.log.info("Found tag for topmost changelog version '%s'" % commit)
 235         return commit
 236
 237     # Check when the changelog was last touched
 238     last = repo.get_commits(paths="debian/changelog", num=1)
 239     if last:
 240         gbp.log.info("Changelog last touched at '%s'" % last[0])
 241         return last[0]
 242
 243     # Changelog not touched yet
 244     return None
 245
 246
 247 def has_snapshot_banner(cp):
 248     """Whether the changelog has a snapshot banner"""
 249     sr = re.search(snapshot_re, cp['Changes'])
 250     return True if sr else False
 251
 252
 253 def get_customizations(customization_file):
 254     if customization_file:
 255         try:
 256             with open(customization_file) as f:
 257                 exec(f.read(), user_customizations, user_customizations)
 258         except Exception as err:
 259             raise GbpError("Failed to load customization file: %s" % err)
 260
 261
 262 def process_options(options, parser):
 263     if options.snapshot and options.release:
 264         parser.error("'--snapshot' and '--release' are incompatible options")
 265
 266     if options.since and options.auto:
 267         parser.error("'--since' and '--auto' are incompatible options")
 268
 269     if not options.since and not options.auto:
 270         options.auto = True
 271
 272     dch_options = []
 273     if options.multimaint_merge:
 274         dch_options.append("--multimaint-merge")
 275     else:
 276         dch_options.append("--nomultimaint-merge")
 277
 278     if options.multimaint:
 279         dch_options.append("--multimaint")
 280     else:
 281         dch_options.append("--nomultimaint")
 282
 283     if options.force_distribution:
 284         dch_options.append("--force-distribution")
 285
 286     return dch_options + options.dch_opts
 287
 288
 289 def process_editor_option(options):
 290     """Determine text editor and check if we need it"""
 291     states = ['always']
 292
 293     if options.snapshot:
 294         states.append("snapshot")
 295     elif options.release:
 296         states.append("release")
 297
 298     if options.spawn_editor == 'never' or options.spawn_editor not in states:
 299         return None
 300     else:
 301         return "sensible-editor"
 302
 303
 304 def changelog_commit_msg(options, version):
 305     return options.commit_msg % dict(version=version)
 306
 307
 308 def create_changelog(repo, source, options):
 309     try:
 310         name = source.control.name
 311     except DebianSourceError:
 312         raise GbpError("Did not find debian/changelog or debian/source. Is this a Debian package?")
 313     version = guess_version_from_upstream(repo, options.upstream_tag,
 314                                           options.upstream_branch, None)
 315     return ChangeLog.create(name, version)
 316
 317
 318 def maybe_create_changelog(repo, source, options):
 319     """
 320     Get the changelog or create a new one if it does not exist yet
 321     """
 322     try:
 323         return source.changelog
 324     except DebianSourceError:
 325         return create_changelog(repo, source, options)
 326
 327
 328 def build_parser(name):
 329     try:
 330         parser = GbpOptionParserDebian(command=os.path.basename(name),
 331                                        usage='%prog [options] paths')
 332     except GbpError as err:
 333         gbp.log.err(err)
 334         return None
 335
 336     range_group = GbpOptionGroup(parser, "commit range options",
 337                                  "which commits to add to the changelog")
 338     version_group = GbpOptionGroup(parser, "release & version number options",
 339                                    "what version number and release to use")
 340     commit_group = GbpOptionGroup(parser, "commit message formatting",
 341                                   "howto format the changelog entries")
 342     naming_group = GbpOptionGroup(parser, "branch and tag naming",
 343                                   "branch names and tag formats")
 344     custom_group = GbpOptionGroup(parser, "customization",
 345                                   "options for customization")
 346     parser.add_option_group(range_group)
 347     parser.add_option_group(version_group)
 348     parser.add_option_group(commit_group)
 349     parser.add_option_group(naming_group)
 350     parser.add_option_group(custom_group)
 351
 352     parser.add_boolean_config_file_option(option_name="ignore-branch", dest="ignore_branch")
 353     naming_group.add_config_file_option(option_name="upstream-branch", dest="upstream_branch")
 354     naming_group.add_config_file_option(option_name="debian-branch", dest="debian_branch")
 355     naming_group.add_config_file_option(option_name="upstream-tag", dest="upstream_tag")
 356     naming_group.add_config_file_option(option_name="debian-tag", dest="debian_tag")
 357     naming_group.add_config_file_option(option_name="snapshot-number", dest="snapshot_number",
 358                                         help="expression to determine the next snapshot number, "
 359                                         "default is '%(snapshot-number)s'")
 360     parser.add_config_file_option(option_name="git-log", dest="git_log",
 361                                   help="options to pass to git-log, "
 362                                   "default is '%(git-log)s'")
 363     parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
 364                       help="verbose command execution")
 365     parser.add_config_file_option(option_name="color", dest="color", type='tristate')
 366     parser.add_config_file_option(option_name="color-scheme",
 367                                   dest="color_scheme")
 368     range_group.add_option("-s", "--since", dest="since", help="commit to start from (e.g. HEAD^^^, debian/0.4.3)")
 369     range_group.add_option("-a", "--auto", action="store_true", dest="auto", default=False,
 370                            help="autocomplete changelog from last snapshot or tag")
 371     version_group.add_option("-R", "--release", action="store_true", dest="release", default=False,
 372                              help="mark as release")
 373     version_group.add_option("-S", "--snapshot", action="store_true", dest="snapshot", default=False,
 374                              help="mark as snapshot build")
 375     version_group.add_option("-D", "--distribution", dest="distribution", help="Set distribution")
 376     version_group.add_option("--force-distribution", action="store_true", dest="force_distribution", default=False,
 377                              help="Force the provided distribution to be used, "
 378                              "even if it doesn't match the list of known distributions")
 379     version_group.add_option("-N", "--new-version", dest="new_version",
 380                              help="use this as base for the new version number")
 381     version_group.add_config_file_option("urgency", dest="urgency")
 382     version_group.add_option("--bpo", dest="bpo", action="store_true", default=False,
 383                              help="Increment the Debian release number for an upload to backports, "
 384                              "and add a backport upload changelog comment.")
 385     version_group.add_option("--nmu", dest="nmu", action="store_true", default=False,
 386                              help="Increment the Debian release number for a non-maintainer upload")
 387     version_group.add_option("--qa", dest="qa", action="store_true", default=False,
 388                              help="Increment the Debian release number for a Debian QA Team upload, "
 389                              "and add a QA upload changelog comment.")
 390     version_group.add_option("--team", dest="team", action="store_true", default=False,
 391                              help="Increment the Debian release number for a Debian Team upload, "
 392                              "and add a Team upload changelog comment.")
 393     version_group.add_option("--security", dest="security", action="store_true", default=False,
 394                              help="Increment the Debian release number for a security upload and "
 395                              "add a security upload changelog comment.")
 396     version_group.add_boolean_config_file_option(option_name="git-author", dest="use_git_author")
 397     commit_group.add_boolean_config_file_option(option_name="meta", dest="meta")
 398     commit_group.add_config_file_option(option_name="meta-closes", dest="meta_closes")
 399     commit_group.add_config_file_option(option_name="meta-closes-bugnum", dest="meta_closes_bugnum")
 400     commit_group.add_boolean_config_file_option(option_name="full", dest="full")
 401     commit_group.add_config_file_option(option_name="id-length", dest="idlen",
 402                                         help="include N digits of the commit id in the changelog entry, "
 403                                         "default is '%(id-length)s'",
 404                                         type="int", metavar="N")
 405     commit_group.add_config_file_option(option_name="ignore-regex", dest="ignore_regex",
 406                                         help="Ignore commit lines matching regex, "
 407                                         "default is '%(ignore-regex)s'")
 408     commit_group.add_boolean_config_file_option(option_name="multimaint", dest="multimaint")
 409     commit_group.add_boolean_config_file_option(option_name="multimaint-merge", dest="multimaint_merge")
 410     commit_group.add_config_file_option(option_name="spawn-editor", dest="spawn_editor")
 411     parser.add_config_file_option(option_name="commit-msg",
 412                                   dest="commit_msg")
 413     parser.add_option("-c", "--commit", action="store_true", dest="commit", default=False,
 414                       help="commit changelog file after generating")
 415     parser.add_config_file_option(option_name="dch-opt",
 416                                   dest="dch_opts", action="append",
 417                                   help="option to pass to dch verbatim, "
 418                                   "can be given multiple times",
 419                                   metavar="DCH_OPT")
 420
 421     help_msg = ('Load Python code from CUSTOMIZATION_FILE.  At the moment,'
 422                 ' the only useful thing the code can do is define a custom'
 423                 ' format_changelog_entry() function.')
 424     custom_group.add_config_file_option(option_name="customizations",
 425                                         dest="customization_file",
 426                                         help=help_msg)
 427     custom_group.add_config_file_option(option_name="postedit", dest="postedit",
 428                                         help="Hook to run after changes to the changelog file"
 429                                         "have been finalized default is '%(postedit)s'")
 430     return parser
 431
 432
 433 def parse_args(argv):
 434     parser = build_parser(argv[0])
 435     if not parser:
 436         return [None] * 4
 437
 438     (options, args) = parser.parse_args(argv[1:])
 439     gbp.log.setup(options.color, options.verbose, options.color_scheme)
 440     dch_options = process_options(options, parser)
 441     editor_cmd = process_editor_option(options)
 442     return options, args, dch_options, editor_cmd
 443
 444
 445 def main(argv):
 446     ret = 0
 447     changelog = 'debian/changelog'
 448     until = 'HEAD'
 449     found_snapshot_banner = False
 450     version_change = {}
 451     branch = None
 452
 453     options, args, dch_options, editor_cmd = parse_args(argv)
 454
 455     if not options:
 456         return ExitCodes.parse_error
 457
 458     try:
 459         old_cwd = os.path.abspath(os.path.curdir)
 460         try:
 461             repo = DebianGitRepository('.', toplevel=False)
 462             os.chdir(repo.path)
 463         except GitRepositoryError:
 464             raise GbpError("%s is not a git repository" % (os.path.abspath('.')))
 465
 466         get_customizations(options.customization_file)
 467         try:
 468             branch = repo.get_branch()
 469         except GitRepositoryError:
 470             # Not being on any branch is o.k. with --ignore-branch
 471             if not options.ignore_branch:
 472                 raise
 473
 474         if options.debian_branch != branch and not options.ignore_branch:
 475             gbp.log.err("You are not on branch '%s' but on '%s'" % (options.debian_branch, branch))
 476             raise GbpError("Use --ignore-branch to ignore or --debian-branch to set the branch name.")
 477
 478         source = DebianSource('.')
 479         cp = maybe_create_changelog(repo, source, options)
 480
 481         if options.since:
 482             since = options.since
 483         else:
 484             since = guess_documented_commit(cp, repo, options.debian_tag)
 485             if since:
 486                 msg = "Continuing from commit '%s'" % since
 487             else:
 488                 msg = "Starting from first commit"
 489                 gbp.log.info(msg)
 490             found_snapshot_banner = has_snapshot_banner(cp)
 491
 492         if args:
 493             gbp.log.info("Only looking for changes on '%s'" % " ".join(args))
 494         commits = repo.get_commits(since=since, until=until, paths=args,
 495                                    options=options.git_log.split(" "))
 496         commits.reverse()
 497
 498         add_section = False
 499         # add a new changelog section if:
 500         if (options.new_version or options.bpo or options.nmu or options.qa or
 501                 options.team or options.security):
 502             if options.bpo:
 503                 version_change['increment'] = '--bpo'
 504             elif options.nmu:
 505                 version_change['increment'] = '--nmu'
 506             elif options.qa:
 507                 version_change['increment'] = '--qa'
 508             elif options.team:
 509                 version_change['increment'] = '--team'
 510             elif options.security:
 511                 version_change['increment'] = '--security'
 512             else:
 513                 version_change['version'] = options.new_version
 514             # the user wants to force a new version
 515             add_section = True
 516         elif cp['Distribution'] != "UNRELEASED" and not found_snapshot_banner:
 517             if commits:
 518                 # the last version was a release and we have pending commits
 519                 add_section = True
 520             if options.snapshot:
 521                 # the user want to switch to snapshot mode
 522                 add_section = True
 523
 524         if add_section and not version_change and not source.is_native():
 525             # Get version from upstream if none provided
 526             v = guess_version_from_upstream(repo, options.upstream_tag,
 527                                             options.upstream_branch, cp)
 528             if v:
 529                 version_change['version'] = v
 530
 531         i = 0
 532         for c in commits:
 533             i += 1
 534             parsed = parse_commit(repo, c, options,
 535                                   last_commit=(i == len(commits)))
 536             commit_msg, (commit_author, commit_email) = parsed
 537             if not commit_msg:
 538                 # Some commits can be ignored
 539                 continue
 540
 541             if add_section:
 542                 # Add a section containing just this message (we can't
 543                 # add an empty section with dch)
 544                 cp.add_section(distribution="UNRELEASED", msg=commit_msg,
 545                                version=version_change,
 546                                author=commit_author,
 547                                email=commit_email,
 548                                dch_options=dch_options)
 549                 # Adding a section only needs to happen once.
 550                 add_section = False
 551             else:
 552                 cp.add_entry(commit_msg, commit_author, commit_email, dch_options)
 553
 554         # Show a message if there were no commits (not even ignored
 555         # commits).
 556         if not commits:
 557             gbp.log.info("No changes detected from %s to %s." % (since, until))
 558
 559         if add_section:
 560             # If we end up here, then there were no commits to include,
 561             # so we put a dummy message in the new section.
 562             cp.add_section(distribution="UNRELEASED", msg=["UNRELEASED"],
 563                            version=version_change,
 564                            dch_options=dch_options)
 565
 566         fixup_section(repo, use_git_author=options.use_git_author, options=options,
 567                       dch_options=dch_options)
 568
 569         if options.release:
 570             do_release(changelog, repo, cp, use_git_author=options.use_git_author,
 571                        dch_options=dch_options)
 572         elif options.snapshot:
 573             (snap, commit, version) = do_snapshot(changelog, repo, options.snapshot_number)
 574             gbp.log.info("Changelog %s (snapshot #%d) prepared up to %s" % (version, snap, commit[:7]))
 575
 576         if editor_cmd:
 577             gbpc.Command(editor_cmd, ["debian/changelog"])()
 578
 579         if options.postedit:
 580             cp = ChangeLog(filename=changelog)
 581             Hook('Postimport', options.postedit,
 582                  extra_env={'GBP_DEBIAN_VERSION': cp.version})()
 583
 584         if options.commit:
 585             # Get the version from the changelog file (since dch might
 586             # have incremented it, there's no way we can already know
 587             # the version).
 588             version = ChangeLog(filename=changelog).version
 589             # Commit the changes to the changelog file
 590             msg = changelog_commit_msg(options, version)
 591             repo.commit_files([changelog], msg)
 592             gbp.log.info("Changelog committed for version %s" % version)
 593     except KeyboardInterrupt:
 594         ret = 1
 595         gbp.log.err("Interrupted. Aborting.")
 596     except (gbpc.CommandExecFailed,
 597             GbpError,
 598             GitRepositoryError,
 599             DebianSourceError,
 600             NoChangeLogError) as err:
 601         if str(err):
 602             gbp.log.err(err)
 603         ret = 1
 604         maybe_debug_raise()
 605     finally:
 606         os.chdir(old_cwd)
 607     return ret
 608
 609
 610 if __name__ == "__main__":
 611     sys.exit(main(sys.argv))
 612
 613 # vim:et:ts=4:sw=4:et:sts=4:ai:set list listchars=tab\:»·,trail\:·: