gbp/scripts/common/pq.py

   1 # vim: set fileencoding=utf-8 :
   2 #
   3 # (C) 2011,2015,2017 Guido Günther <agx@sigxcpu.org>
   4 # (C) 2012 Intel Corporation <markus.lehtonen@linux.intel.com>
   5 #    This program is free software; you can redistribute it and/or modify
   6 #    it under the terms of the GNU General Public License as published by
   7 #    the Free Software Foundation; either version 2 of the License, or
   8 #    (at your option) any later version.
   9 #
  10 #    This program is distributed in the hope that it will be useful,
  11 #    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 #    GNU General Public License for more details.
  14 #
  15 #    You should have received a copy of the GNU General Public License
  16 #    along with this program; if not, please see
  17 #    <http://www.gnu.org/licenses/>
  18 #
  19 """Common functionality for Debian and RPM patchqueue management"""
  20
  21 import re
  22 import os
  23 import datetime
  24 import time
  25 from email.message import Message
  26 from email.header import Header
  27 from email.charset import Charset, QP
  28 from email.policy import Compat32
  29
  30 from gbp.git import GitRepositoryError
  31 from gbp.git.modifier import GitModifier, GitTz
  32 from gbp.errors import GbpError
  33 import gbp.log
  34
  35 PQ_BRANCH_PREFIX = "patch-queue/"
  36
  37
  38 def is_pq_branch(branch):
  39     """
  40     is branch a patch-queue branch?
  41
  42     >>> is_pq_branch("foo")
  43     False
  44     >>> is_pq_branch("patch-queue/foo")
  45     True
  46     """
  47     return [False, True][branch.startswith(PQ_BRANCH_PREFIX)]
  48
  49
  50 def pq_branch_name(branch):
  51     """
  52     get the patch queue branch corresponding to branch
  53
  54     >>> pq_branch_name("patch-queue/master")
  55     'patch-queue/master'
  56     >>> pq_branch_name("foo")
  57     'patch-queue/foo'
  58     """
  59     if not is_pq_branch(branch):
  60         return PQ_BRANCH_PREFIX + branch
  61     else:
  62         return branch
  63
  64
  65 def pq_branch_base(branch):
  66     """
  67     get the branch corresponding to the given patch queue branch
  68
  69     >>> pq_branch_base("patch-queue/master")
  70     'master'
  71     >>> pq_branch_base("foo")
  72     'foo'
  73     """
  74     if is_pq_branch(branch):
  75         return branch[len(PQ_BRANCH_PREFIX):]
  76     else:
  77         return branch
  78
  79
  80 def parse_gbp_commands(info, cmd_tag, noarg_cmds, arg_cmds, filter_cmds=None):
  81     """
  82     Parses gbp commands from commit message. Args with and wthout
  83     arguments are supported as is filtering out of commands from the
  84     commit body.
  85
  86     @param info: the commit into to parse for commands
  87     @param cmd_tag: the command tag
  88     @param noarg_cmds: commands without an argument
  89     @type  noarg_cmds: C{list} of C{str}
  90     @param arg_cmds: command with an argumnt
  91     @type  arg_cmds: C{list} of C{str}
  92     @param filter_cmds: commands to filter out of the passed in info
  93     @type  filter_cmds: C{list} of C{str}
  94     @returns: the parsed commands and the filtered commit body.
  95     """
  96     body = []
  97     cmd_re = re.compile(r'^%s:\s*(?P<cmd>[a-z-]+)(\s+(?P<args>\S.*))?' %
  98                         cmd_tag, flags=re.I)
  99     commands = {}
 100     for line in info['body'].splitlines():
 101         match = re.match(cmd_re, line)
 102         if match:
 103             cmd = match.group('cmd').lower()
 104             if arg_cmds and cmd in arg_cmds:
 105                 if match.group('args'):
 106                     commands[cmd] = match.group('args')
 107                 else:
 108                     gbp.log.warn("Ignoring gbp-command '%s' in commit %s: "
 109                                  "missing cmd arguments" % (line, info['id']))
 110             elif noarg_cmds and cmd in noarg_cmds:
 111                 commands[cmd] = match.group('args')
 112             else:
 113                 gbp.log.warn("Ignoring unknown gbp-command '%s' in commit %s"
 114                              % (line, info['id']))
 115             if filter_cmds is None or cmd not in filter_cmds:
 116                 body.append(line)
 117         else:
 118             body.append(line)
 119     msg = '\n'.join(body)
 120     return (commands, msg)
 121
 122
 123 def patch_path_filter(file_status, exclude_regex=None):
 124     """
 125     Create patch include paths, i.e. a "negation" of the exclude paths.
 126     """
 127     if exclude_regex:
 128         include_paths = []
 129         for file_list in file_status.values():
 130             for fname in file_list:
 131                 if not re.match(exclude_regex, fname):
 132                     include_paths.append(fname)
 133     else:
 134         include_paths = ['.']
 135
 136     return include_paths
 137
 138
 139 def write_patch_file(filename, commit_info, diff):
 140     """Write patch file"""
 141     if not diff:
 142         gbp.log.debug("I won't generate empty diff %s" % filename)
 143         return None
 144     try:
 145         with open(filename, 'wb') as patch:
 146             msg = Message()
 147             charset = Charset('utf-8')
 148             charset.body_encoding = None
 149             charset.header_encoding = QP
 150
 151             # Write headers
 152             name = commit_info['author']['name']
 153             email = commit_info['author']['email']
 154             # Git compat: put name in quotes if special characters found
 155             if re.search(r"[,.@()\[\]\\\:;]", name):
 156                 name = '"%s"' % name
 157             from_header = Header(header_name='from')
 158             try:
 159                 from_header.append(name, 'us-ascii')
 160             except UnicodeDecodeError:
 161                 from_header.append(name, charset)
 162             from_header.append('<%s>' % email)
 163             msg['From'] = from_header
 164             date = commit_info['author'].datetime
 165             datestr = date.strftime('%a, %-d %b %Y %H:%M:%S %z')
 166             msg['Date'] = Header(datestr, 'us-ascii', 'date')
 167             subject_header = Header(header_name='subject')
 168             try:
 169                 subject_header.append(commit_info['subject'], 'us-ascii')
 170             except UnicodeDecodeError:
 171                 subject_header.append(commit_info['subject'], charset)
 172             msg['Subject'] = subject_header
 173             # Write message body
 174             if commit_info['body']:
 175                 # Strip extra linefeeds
 176                 body = commit_info['body'].rstrip() + '\n'
 177                 try:
 178                     msg.set_payload(body.encode('us-ascii'))
 179                 except (UnicodeEncodeError):
 180                     msg.set_payload(body, charset)
 181             policy = Compat32(max_line_length=77)
 182             patch.write(msg.as_bytes(unixfrom=False, policy=policy))
 183
 184             # Write diff
 185             patch.write(b'---\n')
 186             patch.write(diff)
 187     except IOError as err:
 188         raise GbpError('Unable to create patch file: %s' % err)
 189     return filename
 190
 191
 192 DEFAULT_PATCH_NUM_PREFIX_FORMAT = "%04d-"
 193
 194
 195 def format_patch(outdir, repo, commit_info, series, abbrev, numbered=True,
 196                  path_exclude_regex=None, topic='', name=None, renumber=False,
 197                  patch_num_prefix_format=DEFAULT_PATCH_NUM_PREFIX_FORMAT):
 198     """Create patch of a single commit"""
 199
 200     # Determine filename and path
 201     outdir = os.path.join(outdir, topic)
 202     if not os.path.exists(outdir):
 203         os.makedirs(outdir)
 204
 205     try:
 206         num_prefix = str(patch_num_prefix_format) % (len(series) + 1) \
 207             if numbered else ''
 208     except Exception:
 209         gbp.log.warn("Bad format format string '%s', "
 210                      "falling back to default '%s'" %
 211                      (str(patch_num_prefix_format),
 212                       DEFAULT_PATCH_NUM_PREFIX_FORMAT))
 213         num_prefix = DEFAULT_PATCH_NUM_PREFIX_FORMAT % (len(series) + 1)
 214
 215     if name is not None:
 216         if renumber:
 217             # Remove any existing numeric prefix if the patch
 218             # should be renumbered
 219             name = re.sub(r'^\d+[-_]*', '', name)
 220         else:
 221             # Otherwise, clear proposed prefix
 222             num_prefix = ''
 223         (base, suffix) = os.path.splitext(name)
 224     else:
 225         suffix = '.patch'
 226         base_maxlen = 63 - len(num_prefix) - len(suffix)
 227         base = commit_info['patchname'][:base_maxlen]
 228
 229     filename = num_prefix + base + suffix
 230     filepath = os.path.join(outdir, filename)
 231     # Make sure that we don't overwrite existing patches in the series
 232     if filepath in series:
 233         presuffix = '-%d' % len([p for p in series
 234                                  if p.startswith(os.path.splitext(filepath)[0])])
 235         filename = num_prefix + base + presuffix + suffix
 236         filepath = os.path.join(outdir, filename)
 237
 238     # Determine files to include
 239     paths = patch_path_filter(commit_info['files'], path_exclude_regex)
 240
 241     # Finally, create the patch
 242     patch = None
 243     if paths:
 244         diff = repo.diff('%s^!' % commit_info['id'], paths=paths, stat=80,
 245                          summary=True, text=True, abbrev=abbrev, renames=False)
 246         patch = write_patch_file(filepath, commit_info, diff)
 247         if patch:
 248             series.append(patch)
 249     return patch
 250
 251
 252 def format_diff(outdir, filename, repo, start, end, abbrev, path_exclude_regex=None):
 253     """Create a patch of diff between two repository objects"""
 254
 255     info = {'author': repo.get_author_info()}
 256     now = datetime.datetime.now().replace(tzinfo=GitTz(-time.timezone))
 257     info['author'].set_date(now)
 258     info['subject'] = "Raw diff %s..%s" % (start, end)
 259     info['body'] = ("Raw diff between %s '%s' and\n%s '%s'\n" %
 260                     (repo.get_obj_type(start), start,
 261                      repo.get_obj_type(end), end))
 262     if not filename:
 263         filename = '%s-to-%s.diff' % (start, end)
 264     filename = os.path.join(outdir, filename)
 265
 266     file_status = repo.diff_status(start, end)
 267     paths = patch_path_filter(file_status, path_exclude_regex)
 268     if paths:
 269         diff = repo.diff(start, end, paths=paths, stat=80, summary=True,
 270                          text=True, abbrev=abbrev, renames=False)
 271         return write_patch_file(filename, info, diff)
 272     return None
 273
 274
 275 def get_maintainer_from_control(repo):
 276     """Get the maintainer from the control file"""
 277     control = os.path.join(repo.path, 'debian', 'control')
 278
 279     maint_re = re.compile('Maintainer: +(?P<name>.*[^ ]) *<(?P<email>.*)>')
 280     with open(control, encoding='utf-8') as f:
 281         for line in f:
 282             m = maint_re.match(line)
 283             if m:
 284                 return GitModifier(m.group('name'), m.group('email'))
 285     return GitModifier()
 286
 287
 288 def switch_to_pq_branch(repo, branch):
 289     """
 290     Switch to patch-queue branch if not already on it.
 291     doesn't exist yet
 292     """
 293     if is_pq_branch(branch):
 294         return
 295
 296     pq_branch = pq_branch_name(branch)
 297     if not repo.has_branch(pq_branch):
 298         raise GbpError("Branch '%s' does not exist, try "
 299                        "'import' instead" % pq_branch)
 300
 301     gbp.log.info("Switching to '%s'" % pq_branch)
 302     repo.set_branch(pq_branch)
 303
 304
 305 def apply_single_patch(repo, branch, patch, fallback_author, topic=None):
 306     switch_to_pq_branch(repo, branch)
 307     apply_and_commit_patch(repo, patch, fallback_author, topic)
 308     gbp.log.info("Applied %s" % os.path.basename(patch.path))
 309
 310
 311 def apply_and_commit_patch(repo, patch, fallback_author, topic=None, name=None):
 312     """apply a single patch 'patch', add topic 'topic' and commit it"""
 313     author = {'name': patch.author,
 314               'email': patch.email,
 315               'date': patch.date}
 316
 317     patch_fn = os.path.basename(patch.path)
 318     if not (author['name'] and author['email']):
 319         if fallback_author and fallback_author['name']:
 320             author = {}
 321             for key in 'name', 'email', 'date':
 322                 author[key] = fallback_author.get(key)
 323             gbp.log.warn("Patch '%s' has no authorship information, using "
 324                          "'%s <%s>'" % (patch_fn, author['name'],
 325                                         author['email']))
 326         else:
 327             gbp.log.warn("Patch '%s' has no authorship information" % patch_fn)
 328
 329     try:
 330         repo.apply_patch(patch.path, strip=patch.strip)
 331     except GitRepositoryError:
 332         gbp.log.warn("Patch %s failed to apply, retrying with whitespace fixup" % patch_fn)
 333         repo.apply_patch(patch.path, strip=patch.strip, fix_ws=True)
 334     tree = repo.write_tree()
 335     msg = "%s\n\n%s" % (patch.subject, patch.long_desc)
 336     if topic:
 337         msg += "\nGbp-Pq: Topic %s" % topic
 338     if name:
 339         msg += "\nGbp-Pq: Name %s" % name
 340     if author['name']:
 341         author['name'] = author['name'].encode('utf-8')
 342     commit = repo.commit_tree(tree, msg, [repo.head], author=author)
 343     repo.update_ref('HEAD', commit, msg="gbp-pq import %s" % patch.path)
 344
 345
 346 def drop_pq(repo, branch):
 347     repo.checkout(pq_branch_base(branch))
 348     pq_branch = pq_branch_name(branch)
 349     if repo.has_branch(pq_branch):
 350         repo.delete_branch(pq_branch)
 351         gbp.log.info("Dropped branch '%s'." % pq_branch)
 352     else:
 353         gbp.log.info("No patch queue branch found - doing nothing.")