1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
36 import xml.etree.ElementTree as XMLElementTree
38 from distutils.version import LooseVersion
39 from zipfile import ZipFile
42 from fdroidserver.asynchronousfilereader import AsynchronousFileReader
45 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
54 'sdk_path': "$ANDROID_HOME",
57 'r10e': "$ANDROID_NDK",
59 'build_tools': "23.0.1",
63 'accepted_formats': ['txt', 'yaml'],
64 'sync_from_local_copy_dir': False,
65 'per_app_repos': False,
66 'make_current_version_link': True,
67 'current_version_name_source': 'Name',
68 'update_stats': False,
72 'stats_to_carbon': False,
74 'build_server_always': False,
75 'keystore': 'keystore.jks',
76 'smartcardoptions': [],
82 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
83 'repo_name': "My First FDroid Repo Demo",
84 'repo_icon': "fdroid-icon.png",
85 'repo_description': '''
86 This is a repository of apps to be used with FDroid. Applications in this
87 repository are either official binaries built by the original application
88 developers, or are binaries built from source by the admin of f-droid.org
89 using the tools on https://gitlab.com/u/fdroid.
95 def setup_global_opts(parser):
96 parser.add_argument("-v", "--verbose", action="store_true", default=False,
97 help="Spew out even more information than normal")
98 parser.add_argument("-q", "--quiet", action="store_true", default=False,
99 help="Restrict output to warnings and errors")
102 def fill_config_defaults(thisconfig):
103 for k, v in default_config.items():
104 if k not in thisconfig:
107 # Expand paths (~users and $vars)
108 def expand_path(path):
112 path = os.path.expanduser(path)
113 path = os.path.expandvars(path)
118 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
123 thisconfig[k + '_orig'] = v
125 for k in ['ndk_paths']:
131 thisconfig[k][k2] = exp
132 thisconfig[k][k2 + '_orig'] = v
135 def regsub_file(pattern, repl, path):
136 with open(path, 'r') as f:
138 text = re.sub(pattern, repl, text)
139 with open(path, 'w') as f:
143 def read_config(opts, config_file='config.py'):
144 """Read the repository config
146 The config is read from config_file, which is in the current directory when
147 any of the repo management commands are used.
149 global config, options, env, orig_path
151 if config is not None:
153 if not os.path.isfile(config_file):
154 logging.critical("Missing config file - is this a repo directory?")
161 logging.debug("Reading %s" % config_file)
162 execfile(config_file, config)
164 # smartcardoptions must be a list since its command line args for Popen
165 if 'smartcardoptions' in config:
166 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
167 elif 'keystore' in config and config['keystore'] == 'NONE':
168 # keystore='NONE' means use smartcard, these are required defaults
169 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
170 'SunPKCS11-OpenSC', '-providerClass',
171 'sun.security.pkcs11.SunPKCS11',
172 '-providerArg', 'opensc-fdroid.cfg']
174 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
175 st = os.stat(config_file)
176 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
177 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
179 fill_config_defaults(config)
181 # There is no standard, so just set up the most common environment
184 orig_path = env['PATH']
185 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
186 env[n] = config['sdk_path']
188 for k in ["keystorepass", "keypass"]:
190 write_password_file(k)
192 for k in ["repo_description", "archive_description"]:
194 config[k] = clean_description(config[k])
196 if 'serverwebroot' in config:
197 if isinstance(config['serverwebroot'], basestring):
198 roots = [config['serverwebroot']]
199 elif all(isinstance(item, basestring) for item in config['serverwebroot']):
200 roots = config['serverwebroot']
202 raise TypeError('only accepts strings, lists, and tuples')
204 for rootstr in roots:
205 # since this is used with rsync, where trailing slashes have
206 # meaning, ensure there is always a trailing slash
207 if rootstr[-1] != '/':
209 rootlist.append(rootstr.replace('//', '/'))
210 config['serverwebroot'] = rootlist
215 def get_ndk_path(version):
217 version = 'r10e' # falls back to latest
218 paths = config['ndk_paths']
219 if version not in paths:
221 return paths[version] or ''
224 def find_sdk_tools_cmd(cmd):
225 '''find a working path to a tool from the Android SDK'''
228 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
229 # try to find a working path to this command, in all the recent possible paths
230 if 'build_tools' in config:
231 build_tools = os.path.join(config['sdk_path'], 'build-tools')
232 # if 'build_tools' was manually set and exists, check only that one
233 configed_build_tools = os.path.join(build_tools, config['build_tools'])
234 if os.path.exists(configed_build_tools):
235 tooldirs.append(configed_build_tools)
237 # no configed version, so hunt known paths for it
238 for f in sorted(os.listdir(build_tools), reverse=True):
239 if os.path.isdir(os.path.join(build_tools, f)):
240 tooldirs.append(os.path.join(build_tools, f))
241 tooldirs.append(build_tools)
242 sdk_tools = os.path.join(config['sdk_path'], 'tools')
243 if os.path.exists(sdk_tools):
244 tooldirs.append(sdk_tools)
245 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
246 if os.path.exists(sdk_platform_tools):
247 tooldirs.append(sdk_platform_tools)
248 tooldirs.append('/usr/bin')
250 if os.path.isfile(os.path.join(d, cmd)):
251 return os.path.join(d, cmd)
252 # did not find the command, exit with error message
253 ensure_build_tools_exists(config)
256 def test_sdk_exists(thisconfig):
257 if 'sdk_path' not in thisconfig:
258 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
261 logging.error("'sdk_path' not set in config.py!")
263 if thisconfig['sdk_path'] == default_config['sdk_path']:
264 logging.error('No Android SDK found!')
265 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
266 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
268 if not os.path.exists(thisconfig['sdk_path']):
269 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
271 if not os.path.isdir(thisconfig['sdk_path']):
272 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
274 for d in ['build-tools', 'platform-tools', 'tools']:
275 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
276 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
277 thisconfig['sdk_path'], d))
282 def ensure_build_tools_exists(thisconfig):
283 if not test_sdk_exists(thisconfig):
285 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
286 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
287 if not os.path.isdir(versioned_build_tools):
288 logging.critical('Android Build Tools path "'
289 + versioned_build_tools + '" does not exist!')
293 def write_password_file(pwtype, password=None):
295 writes out passwords to a protected file instead of passing passwords as
296 command line argments
298 filename = '.fdroid.' + pwtype + '.txt'
299 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
301 os.write(fd, config[pwtype])
303 os.write(fd, password)
305 config[pwtype + 'file'] = filename
308 # Given the arguments in the form of multiple appid:[vc] strings, this returns
309 # a dictionary with the set of vercodes specified for each package.
310 def read_pkg_args(args, allow_vercodes=False):
317 if allow_vercodes and ':' in p:
318 package, vercode = p.split(':')
320 package, vercode = p, None
321 if package not in vercodes:
322 vercodes[package] = [vercode] if vercode else []
324 elif vercode and vercode not in vercodes[package]:
325 vercodes[package] += [vercode] if vercode else []
330 # On top of what read_pkg_args does, this returns the whole app metadata, but
331 # limiting the builds list to the builds matching the vercodes specified.
332 def read_app_args(args, allapps, allow_vercodes=False):
334 vercodes = read_pkg_args(args, allow_vercodes)
340 for appid, app in allapps.iteritems():
341 if appid in vercodes:
344 if len(apps) != len(vercodes):
347 logging.critical("No such package: %s" % p)
348 raise FDroidException("Found invalid app ids in arguments")
350 raise FDroidException("No packages specified")
353 for appid, app in apps.iteritems():
357 app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
358 if len(app['builds']) != len(vercodes[appid]):
360 allvcs = [b['vercode'] for b in app['builds']]
361 for v in vercodes[appid]:
363 logging.critical("No such vercode %s for app %s" % (v, appid))
366 raise FDroidException("Found invalid vercodes for some apps")
371 def get_extension(filename):
372 _, ext = os.path.splitext(filename)
375 return ext.lower()[1:]
378 def has_extension(filename, ext):
379 return ext == get_extension(filename)
385 def clean_description(description):
386 'Remove unneeded newlines and spaces from a block of description text'
388 # this is split up by paragraph to make removing the newlines easier
389 for paragraph in re.split(r'\n\n', description):
390 paragraph = re.sub('\r', '', paragraph)
391 paragraph = re.sub('\n', ' ', paragraph)
392 paragraph = re.sub(' {2,}', ' ', paragraph)
393 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
394 returnstring += paragraph + '\n\n'
395 return returnstring.rstrip('\n')
398 def apknameinfo(filename):
400 filename = os.path.basename(filename)
401 if apk_regex is None:
402 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
403 m = apk_regex.match(filename)
405 result = (m.group(1), m.group(2))
406 except AttributeError:
407 raise FDroidException("Invalid apk name: %s" % filename)
411 def getapkname(app, build):
412 return "%s_%s.apk" % (app['id'], build['vercode'])
415 def getsrcname(app, build):
416 return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
423 return app['Auto Name']
428 return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
431 def getvcs(vcstype, remote, local):
433 return vcs_git(remote, local)
434 if vcstype == 'git-svn':
435 return vcs_gitsvn(remote, local)
437 return vcs_hg(remote, local)
439 return vcs_bzr(remote, local)
440 if vcstype == 'srclib':
441 if local != os.path.join('build', 'srclib', remote):
442 raise VCSException("Error: srclib paths are hard-coded!")
443 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
445 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
446 raise VCSException("Invalid vcs type " + vcstype)
449 def getsrclibvcs(name):
450 if name not in metadata.srclibs:
451 raise VCSException("Missing srclib " + name)
452 return metadata.srclibs[name]['Repo Type']
457 def __init__(self, remote, local):
459 # svn, git-svn and bzr may require auth
461 if self.repotype() in ('git-svn', 'bzr'):
463 if self.repotype == 'git-svn':
464 raise VCSException("Authentication is not supported for git-svn")
465 self.username, remote = remote.split('@')
466 if ':' not in self.username:
467 raise VCSException("Password required with username")
468 self.username, self.password = self.username.split(':')
472 self.clone_failed = False
473 self.refreshed = False
479 # Take the local repository to a clean version of the given revision, which
480 # is specificed in the VCS's native format. Beforehand, the repository can
481 # be dirty, or even non-existent. If the repository does already exist
482 # locally, it will be updated from the origin, but only once in the
483 # lifetime of the vcs object.
484 # None is acceptable for 'rev' if you know you are cloning a clean copy of
485 # the repo - otherwise it must specify a valid revision.
486 def gotorevision(self, rev, refresh=True):
488 if self.clone_failed:
489 raise VCSException("Downloading the repository already failed once, not trying again.")
491 # The .fdroidvcs-id file for a repo tells us what VCS type
492 # and remote that directory was created from, allowing us to drop it
493 # automatically if either of those things changes.
494 fdpath = os.path.join(self.local, '..',
495 '.fdroidvcs-' + os.path.basename(self.local))
496 cdata = self.repotype() + ' ' + self.remote
499 if os.path.exists(self.local):
500 if os.path.exists(fdpath):
501 with open(fdpath, 'r') as f:
502 fsdata = f.read().strip()
507 logging.info("Repository details for %s changed - deleting" % (
511 logging.info("Repository details for %s missing - deleting" % (
514 shutil.rmtree(self.local)
518 self.refreshed = True
521 self.gotorevisionx(rev)
522 except FDroidException, e:
525 # If necessary, write the .fdroidvcs file.
526 if writeback and not self.clone_failed:
527 with open(fdpath, 'w') as f:
533 # Derived classes need to implement this. It's called once basic checking
534 # has been performend.
535 def gotorevisionx(self, rev):
536 raise VCSException("This VCS type doesn't define gotorevisionx")
538 # Initialise and update submodules
539 def initsubmodules(self):
540 raise VCSException('Submodules not supported for this vcs type')
542 # Get a list of all known tags
544 if not self._gettags:
545 raise VCSException('gettags not supported for this vcs type')
547 for tag in self._gettags():
548 if re.match('[-A-Za-z0-9_. /]+$', tag):
552 def latesttags(self, tags, number):
553 """Get the most recent tags in a given list.
555 :param tags: a list of tags
556 :param number: the number to return
557 :returns: A list containing the most recent tags in the provided
558 list, up to the maximum number given.
560 raise VCSException('latesttags not supported for this vcs type')
562 # Get current commit reference (hash, revision, etc)
564 raise VCSException('getref not supported for this vcs type')
566 # Returns the srclib (name, path) used in setting up the current
577 # If the local directory exists, but is somehow not a git repository, git
578 # will traverse up the directory tree until it finds one that is (i.e.
579 # fdroidserver) and then we'll proceed to destroy it! This is called as
582 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
583 result = p.output.rstrip()
584 if not result.endswith(self.local):
585 raise VCSException('Repository mismatch')
587 def gotorevisionx(self, rev):
588 if not os.path.exists(self.local):
590 p = FDroidPopen(['git', 'clone', self.remote, self.local])
591 if p.returncode != 0:
592 self.clone_failed = True
593 raise VCSException("Git clone failed", p.output)
597 # Discard any working tree changes
598 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
599 'git', 'reset', '--hard'], cwd=self.local, output=False)
600 if p.returncode != 0:
601 raise VCSException("Git reset failed", p.output)
602 # Remove untracked files now, in case they're tracked in the target
603 # revision (it happens!)
604 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
605 'git', 'clean', '-dffx'], cwd=self.local, output=False)
606 if p.returncode != 0:
607 raise VCSException("Git clean failed", p.output)
608 if not self.refreshed:
609 # Get latest commits and tags from remote
610 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
611 if p.returncode != 0:
612 raise VCSException("Git fetch failed", p.output)
613 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
614 if p.returncode != 0:
615 raise VCSException("Git fetch failed", p.output)
616 # Recreate origin/HEAD as git clone would do it, in case it disappeared
617 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
618 if p.returncode != 0:
619 lines = p.output.splitlines()
620 if 'Multiple remote HEAD branches' not in lines[0]:
621 raise VCSException("Git remote set-head failed", p.output)
622 branch = lines[1].split(' ')[-1]
623 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
624 if p2.returncode != 0:
625 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
626 self.refreshed = True
627 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
628 # a github repo. Most of the time this is the same as origin/master.
629 rev = rev or 'origin/HEAD'
630 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
631 if p.returncode != 0:
632 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
633 # Get rid of any uncontrolled files left behind
634 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
635 if p.returncode != 0:
636 raise VCSException("Git clean failed", p.output)
638 def initsubmodules(self):
640 submfile = os.path.join(self.local, '.gitmodules')
641 if not os.path.isfile(submfile):
642 raise VCSException("No git submodules available")
644 # fix submodules not accessible without an account and public key auth
645 with open(submfile, 'r') as f:
646 lines = f.readlines()
647 with open(submfile, 'w') as f:
649 if 'git@github.com' in line:
650 line = line.replace('git@github.com:', 'https://github.com/')
653 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
654 if p.returncode != 0:
655 raise VCSException("Git submodule sync failed", p.output)
656 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
657 if p.returncode != 0:
658 raise VCSException("Git submodule update failed", p.output)
662 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
663 return p.output.splitlines()
665 def latesttags(self, tags, number):
670 ['git', 'show', '--format=format:%ct', '-s', tag],
671 cwd=self.local, output=False)
672 # Timestamp is on the last line. For a normal tag, it's the only
673 # line, but for annotated tags, the rest of the info precedes it.
674 ts = int(p.output.splitlines()[-1])
677 for _, t in sorted(tl)[-number:]:
682 class vcs_gitsvn(vcs):
687 # If the local directory exists, but is somehow not a git repository, git
688 # will traverse up the directory tree until it finds one that is (i.e.
689 # fdroidserver) and then we'll proceed to destory it! This is called as
692 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
693 result = p.output.rstrip()
694 if not result.endswith(self.local):
695 raise VCSException('Repository mismatch')
697 def gotorevisionx(self, rev):
698 if not os.path.exists(self.local):
700 gitsvn_args = ['git', 'svn', 'clone']
701 if ';' in self.remote:
702 remote_split = self.remote.split(';')
703 for i in remote_split[1:]:
704 if i.startswith('trunk='):
705 gitsvn_args.extend(['-T', i[6:]])
706 elif i.startswith('tags='):
707 gitsvn_args.extend(['-t', i[5:]])
708 elif i.startswith('branches='):
709 gitsvn_args.extend(['-b', i[9:]])
710 gitsvn_args.extend([remote_split[0], self.local])
711 p = FDroidPopen(gitsvn_args, output=False)
712 if p.returncode != 0:
713 self.clone_failed = True
714 raise VCSException("Git svn clone failed", p.output)
716 gitsvn_args.extend([self.remote, self.local])
717 p = FDroidPopen(gitsvn_args, output=False)
718 if p.returncode != 0:
719 self.clone_failed = True
720 raise VCSException("Git svn clone failed", p.output)
724 # Discard any working tree changes
725 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
726 if p.returncode != 0:
727 raise VCSException("Git reset failed", p.output)
728 # Remove untracked files now, in case they're tracked in the target
729 # revision (it happens!)
730 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
731 if p.returncode != 0:
732 raise VCSException("Git clean failed", p.output)
733 if not self.refreshed:
734 # Get new commits, branches and tags from repo
735 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
736 if p.returncode != 0:
737 raise VCSException("Git svn fetch failed")
738 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
739 if p.returncode != 0:
740 raise VCSException("Git svn rebase failed", p.output)
741 self.refreshed = True
743 rev = rev or 'master'
745 nospaces_rev = rev.replace(' ', '%20')
746 # Try finding a svn tag
747 for treeish in ['origin/', '']:
748 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
749 if p.returncode == 0:
751 if p.returncode != 0:
752 # No tag found, normal svn rev translation
753 # Translate svn rev into git format
754 rev_split = rev.split('/')
757 for treeish in ['origin/', '']:
758 if len(rev_split) > 1:
759 treeish += rev_split[0]
760 svn_rev = rev_split[1]
763 # if no branch is specified, then assume trunk (i.e. 'master' branch):
767 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
769 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
770 git_rev = p.output.rstrip()
772 if p.returncode == 0 and git_rev:
775 if p.returncode != 0 or not git_rev:
776 # Try a plain git checkout as a last resort
777 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
778 if p.returncode != 0:
779 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
781 # Check out the git rev equivalent to the svn rev
782 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
783 if p.returncode != 0:
784 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
786 # Get rid of any uncontrolled files left behind
787 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
788 if p.returncode != 0:
789 raise VCSException("Git clean failed", p.output)
793 for treeish in ['origin/', '']:
794 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
800 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
801 if p.returncode != 0:
803 return p.output.strip()
811 def gotorevisionx(self, rev):
812 if not os.path.exists(self.local):
813 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
814 if p.returncode != 0:
815 self.clone_failed = True
816 raise VCSException("Hg clone failed", p.output)
818 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
819 if p.returncode != 0:
820 raise VCSException("Hg status failed", p.output)
821 for line in p.output.splitlines():
822 if not line.startswith('? '):
823 raise VCSException("Unexpected output from hg status -uS: " + line)
824 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
825 if not self.refreshed:
826 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
827 if p.returncode != 0:
828 raise VCSException("Hg pull failed", p.output)
829 self.refreshed = True
831 rev = rev or 'default'
834 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
835 if p.returncode != 0:
836 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
837 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
838 # Also delete untracked files, we have to enable purge extension for that:
839 if "'purge' is provided by the following extension" in p.output:
840 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
841 myfile.write("\n[extensions]\nhgext.purge=\n")
842 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
843 if p.returncode != 0:
844 raise VCSException("HG purge failed", p.output)
845 elif p.returncode != 0:
846 raise VCSException("HG purge failed", p.output)
849 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
850 return p.output.splitlines()[1:]
858 def gotorevisionx(self, rev):
859 if not os.path.exists(self.local):
860 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
861 if p.returncode != 0:
862 self.clone_failed = True
863 raise VCSException("Bzr branch failed", p.output)
865 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
866 if p.returncode != 0:
867 raise VCSException("Bzr revert failed", p.output)
868 if not self.refreshed:
869 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
870 if p.returncode != 0:
871 raise VCSException("Bzr update failed", p.output)
872 self.refreshed = True
874 revargs = list(['-r', rev] if rev else [])
875 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
876 if p.returncode != 0:
877 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
880 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
881 return [tag.split(' ')[0].strip() for tag in
882 p.output.splitlines()]
885 def unescape_string(string):
886 if string[0] == '"' and string[-1] == '"':
889 return string.replace("\\'", "'")
892 def retrieve_string(app_dir, string, xmlfiles=None):
897 os.path.join(app_dir, 'res'),
898 os.path.join(app_dir, 'src', 'main', 'res'),
900 for r, d, f in os.walk(res_dir):
901 if os.path.basename(r) == 'values':
902 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
904 if not string.startswith('@string/'):
905 return unescape_string(string)
907 name = string[len('@string/'):]
909 for path in xmlfiles:
910 if not os.path.isfile(path):
912 xml = parse_xml(path)
913 element = xml.find('string[@name="' + name + '"]')
914 if element is not None and element.text is not None:
915 return retrieve_string(app_dir, element.text.encode('utf-8'), xmlfiles)
920 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
921 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
924 # Return list of existing files that will be used to find the highest vercode
925 def manifest_paths(app_dir, flavours):
927 possible_manifests = \
928 [os.path.join(app_dir, 'AndroidManifest.xml'),
929 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
930 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
931 os.path.join(app_dir, 'build.gradle')]
933 for flavour in flavours:
936 possible_manifests.append(
937 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
939 return [path for path in possible_manifests if os.path.isfile(path)]
942 # Retrieve the package name. Returns the name, or None if not found.
943 def fetch_real_name(app_dir, flavours):
944 for path in manifest_paths(app_dir, flavours):
945 if not has_extension(path, 'xml') or not os.path.isfile(path):
947 logging.debug("fetch_real_name: Checking manifest at " + path)
948 xml = parse_xml(path)
949 app = xml.find('application')
950 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
952 label = app.attrib["{http://schemas.android.com/apk/res/android}label"].encode('utf-8')
953 result = retrieve_string_singleline(app_dir, label)
955 result = result.strip()
960 def get_library_references(root_dir):
962 proppath = os.path.join(root_dir, 'project.properties')
963 if not os.path.isfile(proppath):
965 for line in file(proppath):
966 if not line.startswith('android.library.reference.'):
968 path = line.split('=')[1].strip()
969 relpath = os.path.join(root_dir, path)
970 if not os.path.isdir(relpath):
972 logging.debug("Found subproject at %s" % path)
973 libraries.append(path)
977 def ant_subprojects(root_dir):
978 subprojects = get_library_references(root_dir)
979 for subpath in subprojects:
980 subrelpath = os.path.join(root_dir, subpath)
981 for p in get_library_references(subrelpath):
982 relp = os.path.normpath(os.path.join(subpath, p))
983 if relp not in subprojects:
984 subprojects.insert(0, relp)
988 def remove_debuggable_flags(root_dir):
989 # Remove forced debuggable flags
990 logging.debug("Removing debuggable flags from %s" % root_dir)
991 for root, dirs, files in os.walk(root_dir):
992 if 'AndroidManifest.xml' in files:
993 regsub_file(r'android:debuggable="[^"]*"',
995 os.path.join(root, 'AndroidManifest.xml'))
998 # Extract some information from the AndroidManifest.xml at the given path.
999 # Returns (version, vercode, package), any or all of which might be None.
1000 # All values returned are strings.
1001 def parse_androidmanifests(paths, ignoreversions=None):
1004 return (None, None, None)
1006 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
1007 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1008 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1010 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1018 if not os.path.isfile(path):
1021 logging.debug("Parsing manifest at {0}".format(path))
1022 gradle = has_extension(path, 'gradle')
1028 for line in file(path):
1029 if gradle_comment.match(line):
1031 # Grab first occurence of each to avoid running into
1032 # alternative flavours and builds.
1034 matches = psearch_g(line)
1036 package = matches.group(2)
1038 matches = vnsearch_g(line)
1040 version = matches.group(2)
1042 matches = vcsearch_g(line)
1044 vercode = matches.group(1)
1046 xml = parse_xml(path)
1047 if "package" in xml.attrib:
1048 package = xml.attrib["package"].encode('utf-8')
1049 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1050 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"].encode('utf-8')
1051 base_dir = os.path.dirname(path)
1052 version = retrieve_string_singleline(base_dir, version)
1053 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1054 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"].encode('utf-8')
1055 if string_is_integer(a):
1058 # Remember package name, may be defined separately from version+vercode
1060 package = max_package
1062 logging.debug("..got package={0}, version={1}, vercode={2}"
1063 .format(package, version, vercode))
1065 # Always grab the package name and version name in case they are not
1066 # together with the highest version code
1067 if max_package is None and package is not None:
1068 max_package = package
1069 if max_version is None and version is not None:
1070 max_version = version
1072 if max_vercode is None or (vercode is not None and vercode > max_vercode):
1073 if not ignoresearch or not ignoresearch(version):
1074 if version is not None:
1075 max_version = version
1076 if vercode is not None:
1077 max_vercode = vercode
1078 if package is not None:
1079 max_package = package
1081 max_version = "Ignore"
1083 if max_version is None:
1084 max_version = "Unknown"
1086 if max_package and not is_valid_package_name(max_package):
1087 raise FDroidException("Invalid package name {0}".format(max_package))
1089 return (max_version, max_vercode, max_package)
1092 def is_valid_package_name(name):
1093 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1096 class FDroidException(Exception):
1098 def __init__(self, value, detail=None):
1100 self.detail = detail
1102 def limit_size(detail):
1103 if len(detail) < 16000:
1105 return '[...]\n' + detail[-16000:]
1107 def get_wikitext(self):
1108 ret = repr(self.value) + "\n"
1111 ret += "<pre>\n" + limit_size(self.detail) + "</pre>\n"
1117 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1121 class VCSException(FDroidException):
1125 class BuildException(FDroidException):
1129 # Get the specified source library.
1130 # Returns the path to it. Normally this is the path to be used when referencing
1131 # it, which may be a subdirectory of the actual project. If you want the base
1132 # directory of the project, pass 'basepath=True'.
1133 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1134 raw=False, prepare=True, preponly=False, refresh=True):
1142 name, ref = spec.split('@')
1144 number, name = name.split(':', 1)
1146 name, subdir = name.split('/', 1)
1148 if name not in metadata.srclibs:
1149 raise VCSException('srclib ' + name + ' not found.')
1151 srclib = metadata.srclibs[name]
1153 sdir = os.path.join(srclib_dir, name)
1156 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1157 vcs.srclib = (name, number, sdir)
1159 vcs.gotorevision(ref, refresh)
1166 libdir = os.path.join(sdir, subdir)
1167 elif srclib["Subdir"]:
1168 for subdir in srclib["Subdir"]:
1169 libdir_candidate = os.path.join(sdir, subdir)
1170 if os.path.exists(libdir_candidate):
1171 libdir = libdir_candidate
1177 remove_signing_keys(sdir)
1178 remove_debuggable_flags(sdir)
1182 if srclib["Prepare"]:
1183 cmd = replace_config_vars(srclib["Prepare"], None)
1185 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1186 if p.returncode != 0:
1187 raise BuildException("Error running prepare command for srclib %s"
1193 return (name, number, libdir)
1196 # Prepare the source code for a particular build
1197 # 'vcs' - the appropriate vcs object for the application
1198 # 'app' - the application details from the metadata
1199 # 'build' - the build details from the metadata
1200 # 'build_dir' - the path to the build directory, usually
1202 # 'srclib_dir' - the path to the source libraries directory, usually
1204 # 'extlib_dir' - the path to the external libraries directory, usually
1206 # Returns the (root, srclibpaths) where:
1207 # 'root' is the root directory, which may be the same as 'build_dir' or may
1208 # be a subdirectory of it.
1209 # 'srclibpaths' is information on the srclibs being used
1210 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1212 # Optionally, the actual app source can be in a subdirectory
1214 root_dir = os.path.join(build_dir, build['subdir'])
1216 root_dir = build_dir
1218 # Get a working copy of the right revision
1219 logging.info("Getting source for revision " + build['commit'])
1220 vcs.gotorevision(build['commit'], refresh)
1222 # Initialise submodules if required
1223 if build['submodules']:
1224 logging.info("Initialising submodules")
1225 vcs.initsubmodules()
1227 # Check that a subdir (if we're using one) exists. This has to happen
1228 # after the checkout, since it might not exist elsewhere
1229 if not os.path.exists(root_dir):
1230 raise BuildException('Missing subdir ' + root_dir)
1232 # Run an init command if one is required
1234 cmd = replace_config_vars(build['init'], build)
1235 logging.info("Running 'init' commands in %s" % root_dir)
1237 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1238 if p.returncode != 0:
1239 raise BuildException("Error running init command for %s:%s" %
1240 (app['id'], build['version']), p.output)
1242 # Apply patches if any
1244 logging.info("Applying patches")
1245 for patch in build['patch']:
1246 patch = patch.strip()
1247 logging.info("Applying " + patch)
1248 patch_path = os.path.join('metadata', app['id'], patch)
1249 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1250 if p.returncode != 0:
1251 raise BuildException("Failed to apply patch %s" % patch_path)
1253 # Get required source libraries
1255 if build['srclibs']:
1256 logging.info("Collecting source libraries")
1257 for lib in build['srclibs']:
1258 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
1260 for name, number, libpath in srclibpaths:
1261 place_srclib(root_dir, int(number) if number else None, libpath)
1263 basesrclib = vcs.getsrclib()
1264 # If one was used for the main source, add that too.
1266 srclibpaths.append(basesrclib)
1268 # Update the local.properties file
1269 localprops = [os.path.join(build_dir, 'local.properties')]
1271 localprops += [os.path.join(root_dir, 'local.properties')]
1272 for path in localprops:
1274 if os.path.isfile(path):
1275 logging.info("Updating local.properties file at %s" % path)
1276 with open(path, 'r') as f:
1280 logging.info("Creating local.properties file at %s" % path)
1281 # Fix old-fashioned 'sdk-location' by copying
1282 # from sdk.dir, if necessary
1283 if build['oldsdkloc']:
1284 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1285 re.S | re.M).group(1)
1286 props += "sdk-location=%s\n" % sdkloc
1288 props += "sdk.dir=%s\n" % config['sdk_path']
1289 props += "sdk-location=%s\n" % config['sdk_path']
1290 if build['ndk_path']:
1292 props += "ndk.dir=%s\n" % build['ndk_path']
1293 props += "ndk-location=%s\n" % build['ndk_path']
1294 # Add java.encoding if necessary
1295 if build['encoding']:
1296 props += "java.encoding=%s\n" % build['encoding']
1297 with open(path, 'w') as f:
1301 if build['type'] == 'gradle':
1302 flavours = build['gradle']
1304 version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1305 gradlepluginver = None
1307 gradle_dirs = [root_dir]
1309 # Parent dir build.gradle
1310 parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
1311 if parent_dir.startswith(build_dir):
1312 gradle_dirs.append(parent_dir)
1314 for dir_path in gradle_dirs:
1317 if not os.path.isdir(dir_path):
1319 for filename in os.listdir(dir_path):
1320 if not filename.endswith('.gradle'):
1322 path = os.path.join(dir_path, filename)
1323 if not os.path.isfile(path):
1325 for line in file(path):
1326 match = version_regex.match(line)
1328 gradlepluginver = match.group(1)
1332 build['gradlepluginver'] = LooseVersion(gradlepluginver)
1334 logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
1335 build['gradlepluginver'] = LooseVersion('0.11')
1338 n = build["target"].split('-')[1]
1339 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1340 r'compileSdkVersion %s' % n,
1341 os.path.join(root_dir, 'build.gradle'))
1343 # Remove forced debuggable flags
1344 remove_debuggable_flags(root_dir)
1346 # Insert version code and number into the manifest if necessary
1347 if build['forceversion']:
1348 logging.info("Changing the version name")
1349 for path in manifest_paths(root_dir, flavours):
1350 if not os.path.isfile(path):
1352 if has_extension(path, 'xml'):
1353 regsub_file(r'android:versionName="[^"]*"',
1354 r'android:versionName="%s"' % build['version'],
1356 elif has_extension(path, 'gradle'):
1357 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1358 r"""\1versionName '%s'""" % build['version'],
1361 if build['forcevercode']:
1362 logging.info("Changing the version code")
1363 for path in manifest_paths(root_dir, flavours):
1364 if not os.path.isfile(path):
1366 if has_extension(path, 'xml'):
1367 regsub_file(r'android:versionCode="[^"]*"',
1368 r'android:versionCode="%s"' % build['vercode'],
1370 elif has_extension(path, 'gradle'):
1371 regsub_file(r'versionCode[ =]+[0-9]+',
1372 r'versionCode %s' % build['vercode'],
1375 # Delete unwanted files
1377 logging.info("Removing specified files")
1378 for part in getpaths(build_dir, build, 'rm'):
1379 dest = os.path.join(build_dir, part)
1380 logging.info("Removing {0}".format(part))
1381 if os.path.lexists(dest):
1382 if os.path.islink(dest):
1383 FDroidPopen(['unlink', dest], output=False)
1385 FDroidPopen(['rm', '-rf', dest], output=False)
1387 logging.info("...but it didn't exist")
1389 remove_signing_keys(build_dir)
1391 # Add required external libraries
1392 if build['extlibs']:
1393 logging.info("Collecting prebuilt libraries")
1394 libsdir = os.path.join(root_dir, 'libs')
1395 if not os.path.exists(libsdir):
1397 for lib in build['extlibs']:
1399 logging.info("...installing extlib {0}".format(lib))
1400 libf = os.path.basename(lib)
1401 libsrc = os.path.join(extlib_dir, lib)
1402 if not os.path.exists(libsrc):
1403 raise BuildException("Missing extlib file {0}".format(libsrc))
1404 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1406 # Run a pre-build command if one is required
1407 if build['prebuild']:
1408 logging.info("Running 'prebuild' commands in %s" % root_dir)
1410 cmd = replace_config_vars(build['prebuild'], build)
1412 # Substitute source library paths into prebuild commands
1413 for name, number, libpath in srclibpaths:
1414 libpath = os.path.relpath(libpath, root_dir)
1415 cmd = cmd.replace('$$' + name + '$$', libpath)
1417 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1418 if p.returncode != 0:
1419 raise BuildException("Error running prebuild command for %s:%s" %
1420 (app['id'], build['version']), p.output)
1422 # Generate (or update) the ant build file, build.xml...
1423 if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
1424 parms = ['android', 'update', 'lib-project']
1425 lparms = ['android', 'update', 'project']
1428 parms += ['-t', build['target']]
1429 lparms += ['-t', build['target']]
1430 if build['update'] == ['auto']:
1431 update_dirs = ant_subprojects(root_dir) + ['.']
1433 update_dirs = build['update']
1435 for d in update_dirs:
1436 subdir = os.path.join(root_dir, d)
1438 logging.debug("Updating main project")
1439 cmd = parms + ['-p', d]
1441 logging.debug("Updating subproject %s" % d)
1442 cmd = lparms + ['-p', d]
1443 p = SdkToolsPopen(cmd, cwd=root_dir)
1444 # Check to see whether an error was returned without a proper exit
1445 # code (this is the case for the 'no target set or target invalid'
1447 if p.returncode != 0 or p.output.startswith("Error: "):
1448 raise BuildException("Failed to update project at %s" % d, p.output)
1449 # Clean update dirs via ant
1451 logging.info("Cleaning subproject %s" % d)
1452 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1454 return (root_dir, srclibpaths)
1457 # Split and extend via globbing the paths from a field
1458 def getpaths(build_dir, build, field):
1460 for p in build[field]:
1462 full_path = os.path.join(build_dir, p)
1463 full_path = os.path.normpath(full_path)
1464 paths += [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1469 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1475 self.path = os.path.join('stats', 'known_apks.txt')
1477 if os.path.isfile(self.path):
1478 for line in file(self.path):
1479 t = line.rstrip().split(' ')
1481 self.apks[t[0]] = (t[1], None)
1483 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1484 self.changed = False
1486 def writeifchanged(self):
1487 if not self.changed:
1490 if not os.path.exists('stats'):
1494 for apk, app in self.apks.iteritems():
1496 line = apk + ' ' + appid
1498 line += ' ' + time.strftime('%Y-%m-%d', added)
1501 with open(self.path, 'w') as f:
1502 for line in sorted(lst, key=natural_key):
1503 f.write(line + '\n')
1505 # Record an apk (if it's new, otherwise does nothing)
1506 # Returns the date it was added.
1507 def recordapk(self, apk, app):
1508 if apk not in self.apks:
1509 self.apks[apk] = (app, time.gmtime(time.time()))
1511 _, added = self.apks[apk]
1514 # Look up information - given the 'apkname', returns (app id, date added/None).
1515 # Or returns None for an unknown apk.
1516 def getapp(self, apkname):
1517 if apkname in self.apks:
1518 return self.apks[apkname]
1521 # Get the most recent 'num' apps added to the repo, as a list of package ids
1522 # with the most recent first.
1523 def getlatest(self, num):
1525 for apk, app in self.apks.iteritems():
1529 if apps[appid] > added:
1533 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1534 lst = [app for app, _ in sortedapps]
1539 def isApkDebuggable(apkfile, config):
1540 """Returns True if the given apk file is debuggable
1542 :param apkfile: full path to the apk to check"""
1544 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1546 if p.returncode != 0:
1547 logging.critical("Failed to get apk manifest information")
1549 for line in p.output.splitlines():
1550 if 'android:debuggable' in line and not line.endswith('0x0'):
1560 def SdkToolsPopen(commands, cwd=None, output=True):
1562 if cmd not in config:
1563 config[cmd] = find_sdk_tools_cmd(commands[0])
1564 return FDroidPopen([config[cmd]] + commands[1:],
1565 cwd=cwd, output=output)
1568 def FDroidPopen(commands, cwd=None, output=True):
1570 Run a command and capture the possibly huge output.
1572 :param commands: command and argument list like in subprocess.Popen
1573 :param cwd: optionally specifies a working directory
1574 :returns: A PopenResult.
1580 cwd = os.path.normpath(cwd)
1581 logging.debug("Directory: %s" % cwd)
1582 logging.debug("> %s" % ' '.join(commands))
1584 result = PopenResult()
1587 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
1588 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1590 raise BuildException("OSError while trying to execute " +
1591 ' '.join(commands) + ': ' + str(e))
1593 stdout_queue = Queue.Queue()
1594 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1596 # Check the queue for output (until there is no more to get)
1597 while not stdout_reader.eof():
1598 while not stdout_queue.empty():
1599 line = stdout_queue.get()
1600 if output and options.verbose:
1601 # Output directly to console
1602 sys.stderr.write(line)
1604 result.output += line
1608 result.returncode = p.wait()
1612 gradle_comment = re.compile(r'[ ]*//')
1615 def remove_signing_keys(build_dir):
1616 signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1618 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1619 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1620 re.compile(r'.*variant\.outputFile = .*'),
1621 re.compile(r'.*output\.outputFile = .*'),
1622 re.compile(r'.*\.readLine\(.*'),
1624 for root, dirs, files in os.walk(build_dir):
1625 if 'build.gradle' in files:
1626 path = os.path.join(root, 'build.gradle')
1628 with open(path, "r") as o:
1629 lines = o.readlines()
1635 with open(path, "w") as o:
1636 while i < len(lines):
1639 while line.endswith('\\\n'):
1640 line = line.rstrip('\\\n') + lines[i]
1643 if gradle_comment.match(line):
1648 opened += line.count('{')
1649 opened -= line.count('}')
1652 if signing_configs.match(line):
1657 if any(s.match(line) for s in line_matches):
1665 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1668 'project.properties',
1670 'default.properties',
1671 'ant.properties', ]:
1672 if propfile in files:
1673 path = os.path.join(root, propfile)
1675 with open(path, "r") as o:
1676 lines = o.readlines()
1680 with open(path, "w") as o:
1682 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1689 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1692 def reset_env_path():
1693 global env, orig_path
1694 env['PATH'] = orig_path
1697 def add_to_env_path(path):
1699 paths = env['PATH'].split(os.pathsep)
1703 env['PATH'] = os.pathsep.join(paths)
1706 def replace_config_vars(cmd, build):
1708 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1709 # env['ANDROID_NDK'] is set in build_local right before prepare_source
1710 cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
1711 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1712 if build is not None:
1713 cmd = cmd.replace('$$COMMIT$$', build['commit'])
1714 cmd = cmd.replace('$$VERSION$$', build['version'])
1715 cmd = cmd.replace('$$VERCODE$$', build['vercode'])
1719 def place_srclib(root_dir, number, libpath):
1722 relpath = os.path.relpath(libpath, root_dir)
1723 proppath = os.path.join(root_dir, 'project.properties')
1726 if os.path.isfile(proppath):
1727 with open(proppath, "r") as o:
1728 lines = o.readlines()
1730 with open(proppath, "w") as o:
1733 if line.startswith('android.library.reference.%d=' % number):
1734 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1739 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1742 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
1743 """Verify that two apks are the same
1745 One of the inputs is signed, the other is unsigned. The signature metadata
1746 is transferred from the signed to the unsigned apk, and then jarsigner is
1747 used to verify that the signature from the signed apk is also varlid for
1749 :param signed_apk: Path to a signed apk file
1750 :param unsigned_apk: Path to an unsigned apk file expected to match it
1751 :param tmp_dir: Path to directory for temporary files
1752 :returns: None if the verification is successful, otherwise a string
1753 describing what went wrong.
1755 sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
1756 with ZipFile(signed_apk) as signed_apk_as_zip:
1757 meta_inf_files = ['META-INF/MANIFEST.MF']
1758 for f in signed_apk_as_zip.namelist():
1759 if sigfile.match(f):
1760 meta_inf_files.append(f)
1761 if len(meta_inf_files) < 3:
1762 return "Signature files missing from {0}".format(signed_apk)
1763 signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
1764 with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
1765 for meta_inf_file in meta_inf_files:
1766 unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
1768 if subprocess.call(['jarsigner', '-verify', unsigned_apk]) != 0:
1769 logging.info("...NOT verified - {0}".format(signed_apk))
1770 return compare_apks(signed_apk, unsigned_apk, tmp_dir)
1771 logging.info("...successfully verified")
1775 def compare_apks(apk1, apk2, tmp_dir):
1778 Returns None if the apk content is the same (apart from the signing key),
1779 otherwise a string describing what's different, or what went wrong when
1780 trying to do the comparison.
1783 badchars = re.compile('''[/ :;'"]''')
1784 apk1dir = os.path.join(tmp_dir, badchars.sub('_', apk1[0:-4])) # trim .apk
1785 apk2dir = os.path.join(tmp_dir, badchars.sub('_', apk2[0:-4])) # trim .apk
1786 for d in [apk1dir, apk2dir]:
1787 if os.path.exists(d):
1790 os.mkdir(os.path.join(d, 'jar-xf'))
1792 if subprocess.call(['jar', 'xf',
1793 os.path.abspath(apk1)],
1794 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
1795 return("Failed to unpack " + apk1)
1796 if subprocess.call(['jar', 'xf',
1797 os.path.abspath(apk2)],
1798 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
1799 return("Failed to unpack " + apk2)
1801 # try to find apktool in the path, if it hasn't been manually configed
1802 if 'apktool' not in config:
1803 tmp = find_command('apktool')
1805 config['apktool'] = tmp
1806 if 'apktool' in config:
1807 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
1809 return("Failed to unpack " + apk1)
1810 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
1812 return("Failed to unpack " + apk2)
1814 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
1815 lines = p.output.splitlines()
1816 if len(lines) != 1 or 'META-INF' not in lines[0]:
1817 meld = find_command('meld')
1818 if meld is not None:
1819 p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
1820 return("Unexpected diff output - " + p.output)
1822 # since everything verifies, delete the comparison to keep cruft down
1823 shutil.rmtree(apk1dir)
1824 shutil.rmtree(apk2dir)
1826 # If we get here, it seems like they're the same!
1830 def find_command(command):
1831 '''find the full path of a command, or None if it can't be found in the PATH'''
1834 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
1836 fpath, fname = os.path.split(command)
1841 for path in os.environ["PATH"].split(os.pathsep):
1842 path = path.strip('"')
1843 exe_file = os.path.join(path, command)
1844 if is_exe(exe_file):
1851 '''generate a random password for when generating keys'''
1852 h = hashlib.sha256()
1853 h.update(os.urandom(16)) # salt
1854 h.update(bytes(socket.getfqdn()))
1855 return h.digest().encode('base64').strip()
1858 def genkeystore(localconfig):
1859 '''Generate a new key with random passwords and add it to new keystore'''
1860 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
1861 keystoredir = os.path.dirname(localconfig['keystore'])
1862 if keystoredir is None or keystoredir == '':
1863 keystoredir = os.path.join(os.getcwd(), keystoredir)
1864 if not os.path.exists(keystoredir):
1865 os.makedirs(keystoredir, mode=0o700)
1867 write_password_file("keystorepass", localconfig['keystorepass'])
1868 write_password_file("keypass", localconfig['keypass'])
1869 p = FDroidPopen(['keytool', '-genkey',
1870 '-keystore', localconfig['keystore'],
1871 '-alias', localconfig['repo_keyalias'],
1872 '-keyalg', 'RSA', '-keysize', '4096',
1873 '-sigalg', 'SHA256withRSA',
1874 '-validity', '10000',
1875 '-storepass:file', config['keystorepassfile'],
1876 '-keypass:file', config['keypassfile'],
1877 '-dname', localconfig['keydname']])
1878 # TODO keypass should be sent via stdin
1879 if p.returncode != 0:
1880 raise BuildException("Failed to generate key", p.output)
1881 os.chmod(localconfig['keystore'], 0o0600)
1882 # now show the lovely key that was just generated
1883 p = FDroidPopen(['keytool', '-list', '-v',
1884 '-keystore', localconfig['keystore'],
1885 '-alias', localconfig['repo_keyalias'],
1886 '-storepass:file', config['keystorepassfile']])
1887 logging.info(p.output.strip() + '\n\n')
1890 def write_to_config(thisconfig, key, value=None):
1891 '''write a key/value to the local config.py'''
1893 origkey = key + '_orig'
1894 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
1895 with open('config.py', 'r') as f:
1897 pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
1898 repl = '\n' + key + ' = "' + value + '"'
1899 data = re.sub(pattern, repl, data)
1900 # if this key is not in the file, append it
1901 if not re.match('\s*' + key + '\s*=\s*"', data):
1903 # make sure the file ends with a carraige return
1904 if not re.match('\n$', data):
1906 with open('config.py', 'w') as f:
1910 def parse_xml(path):
1911 return XMLElementTree.parse(path).getroot()
1914 def string_is_integer(string):
1922 def get_per_app_repos():
1923 '''per-app repos are dirs named with the packageName of a single app'''
1925 # Android packageNames are Java packages, they may contain uppercase or
1926 # lowercase letters ('A' through 'Z'), numbers, and underscores
1927 # ('_'). However, individual package name parts may only start with
1928 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
1929 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
1932 for root, dirs, files in os.walk(os.getcwd()):
1934 print 'checking', root, 'for', d
1935 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
1936 # standard parts of an fdroid repo, so never packageNames
1939 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):