1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
36 import xml.etree.ElementTree as XMLElementTree
38 from distutils.version import LooseVersion
39 from zipfile import ZipFile
42 from fdroidserver.asynchronousfilereader import AsynchronousFileReader
45 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
54 'sdk_path': "$ANDROID_HOME",
57 'r10e': "$ANDROID_NDK",
59 'build_tools': "23.0.1",
63 'accepted_formats': ['txt', 'yaml'],
64 'sync_from_local_copy_dir': False,
65 'per_app_repos': False,
66 'make_current_version_link': True,
67 'current_version_name_source': 'Name',
68 'update_stats': False,
72 'stats_to_carbon': False,
74 'build_server_always': False,
75 'keystore': 'keystore.jks',
76 'smartcardoptions': [],
82 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
83 'repo_name': "My First FDroid Repo Demo",
84 'repo_icon': "fdroid-icon.png",
85 'repo_description': '''
86 This is a repository of apps to be used with FDroid. Applications in this
87 repository are either official binaries built by the original application
88 developers, or are binaries built from source by the admin of f-droid.org
89 using the tools on https://gitlab.com/u/fdroid.
95 def setup_global_opts(parser):
96 parser.add_argument("-v", "--verbose", action="store_true", default=False,
97 help="Spew out even more information than normal")
98 parser.add_argument("-q", "--quiet", action="store_true", default=False,
99 help="Restrict output to warnings and errors")
102 def fill_config_defaults(thisconfig):
103 for k, v in default_config.items():
104 if k not in thisconfig:
107 # Expand paths (~users and $vars)
108 def expand_path(path):
112 path = os.path.expanduser(path)
113 path = os.path.expandvars(path)
118 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
123 thisconfig[k + '_orig'] = v
125 for k in ['ndk_paths']:
131 thisconfig[k][k2] = exp
132 thisconfig[k][k2 + '_orig'] = v
135 def regsub_file(pattern, repl, path):
136 with open(path, 'r') as f:
138 text = re.sub(pattern, repl, text)
139 with open(path, 'w') as f:
143 def read_config(opts, config_file='config.py'):
144 """Read the repository config
146 The config is read from config_file, which is in the current directory when
147 any of the repo management commands are used.
149 global config, options, env, orig_path
151 if config is not None:
153 if not os.path.isfile(config_file):
154 logging.critical("Missing config file - is this a repo directory?")
161 logging.debug("Reading %s" % config_file)
162 execfile(config_file, config)
164 # smartcardoptions must be a list since its command line args for Popen
165 if 'smartcardoptions' in config:
166 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
167 elif 'keystore' in config and config['keystore'] == 'NONE':
168 # keystore='NONE' means use smartcard, these are required defaults
169 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
170 'SunPKCS11-OpenSC', '-providerClass',
171 'sun.security.pkcs11.SunPKCS11',
172 '-providerArg', 'opensc-fdroid.cfg']
174 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
175 st = os.stat(config_file)
176 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
177 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
179 fill_config_defaults(config)
181 # There is no standard, so just set up the most common environment
184 orig_path = env['PATH']
185 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
186 env[n] = config['sdk_path']
188 for k in ["keystorepass", "keypass"]:
190 write_password_file(k)
192 for k in ["repo_description", "archive_description"]:
194 config[k] = clean_description(config[k])
196 if 'serverwebroot' in config:
197 if isinstance(config['serverwebroot'], basestring):
198 roots = [config['serverwebroot']]
199 elif all(isinstance(item, basestring) for item in config['serverwebroot']):
200 roots = config['serverwebroot']
202 raise TypeError('only accepts strings, lists, and tuples')
204 for rootstr in roots:
205 # since this is used with rsync, where trailing slashes have
206 # meaning, ensure there is always a trailing slash
207 if rootstr[-1] != '/':
209 rootlist.append(rootstr.replace('//', '/'))
210 config['serverwebroot'] = rootlist
215 def get_ndk_path(version):
217 version = 'r10e' # falls back to latest
218 paths = config['ndk_paths']
219 if version not in paths:
221 return paths[version] or ''
224 def find_sdk_tools_cmd(cmd):
225 '''find a working path to a tool from the Android SDK'''
228 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
229 # try to find a working path to this command, in all the recent possible paths
230 if 'build_tools' in config:
231 build_tools = os.path.join(config['sdk_path'], 'build-tools')
232 # if 'build_tools' was manually set and exists, check only that one
233 configed_build_tools = os.path.join(build_tools, config['build_tools'])
234 if os.path.exists(configed_build_tools):
235 tooldirs.append(configed_build_tools)
237 # no configed version, so hunt known paths for it
238 for f in sorted(os.listdir(build_tools), reverse=True):
239 if os.path.isdir(os.path.join(build_tools, f)):
240 tooldirs.append(os.path.join(build_tools, f))
241 tooldirs.append(build_tools)
242 sdk_tools = os.path.join(config['sdk_path'], 'tools')
243 if os.path.exists(sdk_tools):
244 tooldirs.append(sdk_tools)
245 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
246 if os.path.exists(sdk_platform_tools):
247 tooldirs.append(sdk_platform_tools)
248 tooldirs.append('/usr/bin')
250 if os.path.isfile(os.path.join(d, cmd)):
251 return os.path.join(d, cmd)
252 # did not find the command, exit with error message
253 ensure_build_tools_exists(config)
256 def test_sdk_exists(thisconfig):
257 if 'sdk_path' not in thisconfig:
258 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
261 logging.error("'sdk_path' not set in config.py!")
263 if thisconfig['sdk_path'] == default_config['sdk_path']:
264 logging.error('No Android SDK found!')
265 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
266 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
268 if not os.path.exists(thisconfig['sdk_path']):
269 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
271 if not os.path.isdir(thisconfig['sdk_path']):
272 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
274 for d in ['build-tools', 'platform-tools', 'tools']:
275 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
276 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
277 thisconfig['sdk_path'], d))
282 def ensure_build_tools_exists(thisconfig):
283 if not test_sdk_exists(thisconfig):
285 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
286 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
287 if not os.path.isdir(versioned_build_tools):
288 logging.critical('Android Build Tools path "'
289 + versioned_build_tools + '" does not exist!')
293 def write_password_file(pwtype, password=None):
295 writes out passwords to a protected file instead of passing passwords as
296 command line argments
298 filename = '.fdroid.' + pwtype + '.txt'
299 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
301 os.write(fd, config[pwtype])
303 os.write(fd, password)
305 config[pwtype + 'file'] = filename
308 # Given the arguments in the form of multiple appid:[vc] strings, this returns
309 # a dictionary with the set of vercodes specified for each package.
310 def read_pkg_args(args, allow_vercodes=False):
317 if allow_vercodes and ':' in p:
318 package, vercode = p.split(':')
320 package, vercode = p, None
321 if package not in vercodes:
322 vercodes[package] = [vercode] if vercode else []
324 elif vercode and vercode not in vercodes[package]:
325 vercodes[package] += [vercode] if vercode else []
330 # On top of what read_pkg_args does, this returns the whole app metadata, but
331 # limiting the builds list to the builds matching the vercodes specified.
332 def read_app_args(args, allapps, allow_vercodes=False):
334 vercodes = read_pkg_args(args, allow_vercodes)
340 for appid, app in allapps.iteritems():
341 if appid in vercodes:
344 if len(apps) != len(vercodes):
347 logging.critical("No such package: %s" % p)
348 raise FDroidException("Found invalid app ids in arguments")
350 raise FDroidException("No packages specified")
353 for appid, app in apps.iteritems():
357 app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
358 if len(app['builds']) != len(vercodes[appid]):
360 allvcs = [b['vercode'] for b in app['builds']]
361 for v in vercodes[appid]:
363 logging.critical("No such vercode %s for app %s" % (v, appid))
366 raise FDroidException("Found invalid vercodes for some apps")
371 def get_extension(filename):
372 _, ext = os.path.splitext(filename)
375 return ext.lower()[1:]
378 def has_extension(filename, ext):
379 return ext == get_extension(filename)
385 def clean_description(description):
386 'Remove unneeded newlines and spaces from a block of description text'
388 # this is split up by paragraph to make removing the newlines easier
389 for paragraph in re.split(r'\n\n', description):
390 paragraph = re.sub('\r', '', paragraph)
391 paragraph = re.sub('\n', ' ', paragraph)
392 paragraph = re.sub(' {2,}', ' ', paragraph)
393 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
394 returnstring += paragraph + '\n\n'
395 return returnstring.rstrip('\n')
398 def apknameinfo(filename):
400 filename = os.path.basename(filename)
401 if apk_regex is None:
402 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
403 m = apk_regex.match(filename)
405 result = (m.group(1), m.group(2))
406 except AttributeError:
407 raise FDroidException("Invalid apk name: %s" % filename)
411 def getapkname(app, build):
412 return "%s_%s.apk" % (app['id'], build['vercode'])
415 def getsrcname(app, build):
416 return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
423 return app['Auto Name']
428 return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
431 def getvcs(vcstype, remote, local):
433 return vcs_git(remote, local)
434 if vcstype == 'git-svn':
435 return vcs_gitsvn(remote, local)
437 return vcs_hg(remote, local)
439 return vcs_bzr(remote, local)
440 if vcstype == 'srclib':
441 if local != os.path.join('build', 'srclib', remote):
442 raise VCSException("Error: srclib paths are hard-coded!")
443 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
445 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
446 raise VCSException("Invalid vcs type " + vcstype)
449 def getsrclibvcs(name):
450 if name not in metadata.srclibs:
451 raise VCSException("Missing srclib " + name)
452 return metadata.srclibs[name]['Repo Type']
457 def __init__(self, remote, local):
459 # svn, git-svn and bzr may require auth
461 if self.repotype() in ('git-svn', 'bzr'):
463 if self.repotype == 'git-svn':
464 raise VCSException("Authentication is not supported for git-svn")
465 self.username, remote = remote.split('@')
466 if ':' not in self.username:
467 raise VCSException("Password required with username")
468 self.username, self.password = self.username.split(':')
472 self.clone_failed = False
473 self.refreshed = False
479 # Take the local repository to a clean version of the given revision, which
480 # is specificed in the VCS's native format. Beforehand, the repository can
481 # be dirty, or even non-existent. If the repository does already exist
482 # locally, it will be updated from the origin, but only once in the
483 # lifetime of the vcs object.
484 # None is acceptable for 'rev' if you know you are cloning a clean copy of
485 # the repo - otherwise it must specify a valid revision.
486 def gotorevision(self, rev, refresh=True):
488 if self.clone_failed:
489 raise VCSException("Downloading the repository already failed once, not trying again.")
491 # The .fdroidvcs-id file for a repo tells us what VCS type
492 # and remote that directory was created from, allowing us to drop it
493 # automatically if either of those things changes.
494 fdpath = os.path.join(self.local, '..',
495 '.fdroidvcs-' + os.path.basename(self.local))
496 cdata = self.repotype() + ' ' + self.remote
499 if os.path.exists(self.local):
500 if os.path.exists(fdpath):
501 with open(fdpath, 'r') as f:
502 fsdata = f.read().strip()
507 logging.info("Repository details for %s changed - deleting" % (
511 logging.info("Repository details for %s missing - deleting" % (
514 shutil.rmtree(self.local)
518 self.refreshed = True
521 self.gotorevisionx(rev)
522 except FDroidException, e:
525 # If necessary, write the .fdroidvcs file.
526 if writeback and not self.clone_failed:
527 with open(fdpath, 'w') as f:
533 # Derived classes need to implement this. It's called once basic checking
534 # has been performend.
535 def gotorevisionx(self, rev):
536 raise VCSException("This VCS type doesn't define gotorevisionx")
538 # Initialise and update submodules
539 def initsubmodules(self):
540 raise VCSException('Submodules not supported for this vcs type')
542 # Get a list of all known tags
544 if not self._gettags:
545 raise VCSException('gettags not supported for this vcs type')
547 for tag in self._gettags():
548 if re.match('[-A-Za-z0-9_. /]+$', tag):
552 def latesttags(self, tags, number):
553 """Get the most recent tags in a given list.
555 :param tags: a list of tags
556 :param number: the number to return
557 :returns: A list containing the most recent tags in the provided
558 list, up to the maximum number given.
560 raise VCSException('latesttags not supported for this vcs type')
562 # Get current commit reference (hash, revision, etc)
564 raise VCSException('getref not supported for this vcs type')
566 # Returns the srclib (name, path) used in setting up the current
577 # If the local directory exists, but is somehow not a git repository, git
578 # will traverse up the directory tree until it finds one that is (i.e.
579 # fdroidserver) and then we'll proceed to destroy it! This is called as
582 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
583 result = p.output.rstrip()
584 if not result.endswith(self.local):
585 raise VCSException('Repository mismatch')
587 def gotorevisionx(self, rev):
588 if not os.path.exists(self.local):
590 p = FDroidPopen(['git', 'clone', self.remote, self.local])
591 if p.returncode != 0:
592 self.clone_failed = True
593 raise VCSException("Git clone failed", p.output)
597 # Discard any working tree changes
598 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
599 'git', 'reset', '--hard'], cwd=self.local, output=False)
600 if p.returncode != 0:
601 raise VCSException("Git reset failed", p.output)
602 # Remove untracked files now, in case they're tracked in the target
603 # revision (it happens!)
604 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
605 'git', 'clean', '-dffx'], cwd=self.local, output=False)
606 if p.returncode != 0:
607 raise VCSException("Git clean failed", p.output)
608 if not self.refreshed:
609 # Get latest commits and tags from remote
610 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
611 if p.returncode != 0:
612 raise VCSException("Git fetch failed", p.output)
613 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
614 if p.returncode != 0:
615 raise VCSException("Git fetch failed", p.output)
616 # Recreate origin/HEAD as git clone would do it, in case it disappeared
617 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
618 if p.returncode != 0:
619 lines = p.output.splitlines()
620 if 'Multiple remote HEAD branches' not in lines[0]:
621 raise VCSException("Git remote set-head failed", p.output)
622 branch = lines[1].split(' ')[-1]
623 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
624 if p2.returncode != 0:
625 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
626 self.refreshed = True
627 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
628 # a github repo. Most of the time this is the same as origin/master.
629 rev = rev or 'origin/HEAD'
630 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
631 if p.returncode != 0:
632 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
633 # Get rid of any uncontrolled files left behind
634 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
635 if p.returncode != 0:
636 raise VCSException("Git clean failed", p.output)
638 def initsubmodules(self):
640 submfile = os.path.join(self.local, '.gitmodules')
641 if not os.path.isfile(submfile):
642 raise VCSException("No git submodules available")
644 # fix submodules not accessible without an account and public key auth
645 with open(submfile, 'r') as f:
646 lines = f.readlines()
647 with open(submfile, 'w') as f:
649 if 'git@github.com' in line:
650 line = line.replace('git@github.com:', 'https://github.com/')
653 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
654 if p.returncode != 0:
655 raise VCSException("Git submodule sync failed", p.output)
656 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
657 if p.returncode != 0:
658 raise VCSException("Git submodule update failed", p.output)
662 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
663 return p.output.splitlines()
665 def latesttags(self, tags, number):
670 ['git', 'show', '--format=format:%ct', '-s', tag],
671 cwd=self.local, output=False)
672 # Timestamp is on the last line. For a normal tag, it's the only
673 # line, but for annotated tags, the rest of the info precedes it.
674 ts = int(p.output.splitlines()[-1])
677 for _, t in sorted(tl)[-number:]:
682 class vcs_gitsvn(vcs):
687 # If the local directory exists, but is somehow not a git repository, git
688 # will traverse up the directory tree until it finds one that is (i.e.
689 # fdroidserver) and then we'll proceed to destory it! This is called as
692 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
693 result = p.output.rstrip()
694 if not result.endswith(self.local):
695 raise VCSException('Repository mismatch')
697 def gotorevisionx(self, rev):
698 if not os.path.exists(self.local):
700 gitsvn_args = ['git', 'svn', 'clone']
701 if ';' in self.remote:
702 remote_split = self.remote.split(';')
703 for i in remote_split[1:]:
704 if i.startswith('trunk='):
705 gitsvn_args.extend(['-T', i[6:]])
706 elif i.startswith('tags='):
707 gitsvn_args.extend(['-t', i[5:]])
708 elif i.startswith('branches='):
709 gitsvn_args.extend(['-b', i[9:]])
710 gitsvn_args.extend([remote_split[0], self.local])
711 p = FDroidPopen(gitsvn_args, output=False)
712 if p.returncode != 0:
713 self.clone_failed = True
714 raise VCSException("Git svn clone failed", p.output)
716 gitsvn_args.extend([self.remote, self.local])
717 p = FDroidPopen(gitsvn_args, output=False)
718 if p.returncode != 0:
719 self.clone_failed = True
720 raise VCSException("Git svn clone failed", p.output)
724 # Discard any working tree changes
725 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
726 if p.returncode != 0:
727 raise VCSException("Git reset failed", p.output)
728 # Remove untracked files now, in case they're tracked in the target
729 # revision (it happens!)
730 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
731 if p.returncode != 0:
732 raise VCSException("Git clean failed", p.output)
733 if not self.refreshed:
734 # Get new commits, branches and tags from repo
735 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
736 if p.returncode != 0:
737 raise VCSException("Git svn fetch failed")
738 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
739 if p.returncode != 0:
740 raise VCSException("Git svn rebase failed", p.output)
741 self.refreshed = True
743 rev = rev or 'master'
745 nospaces_rev = rev.replace(' ', '%20')
746 # Try finding a svn tag
747 for treeish in ['origin/', '']:
748 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
749 if p.returncode == 0:
751 if p.returncode != 0:
752 # No tag found, normal svn rev translation
753 # Translate svn rev into git format
754 rev_split = rev.split('/')
757 for treeish in ['origin/', '']:
758 if len(rev_split) > 1:
759 treeish += rev_split[0]
760 svn_rev = rev_split[1]
763 # if no branch is specified, then assume trunk (i.e. 'master' branch):
767 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
769 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
770 git_rev = p.output.rstrip()
772 if p.returncode == 0 and git_rev:
775 if p.returncode != 0 or not git_rev:
776 # Try a plain git checkout as a last resort
777 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
778 if p.returncode != 0:
779 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
781 # Check out the git rev equivalent to the svn rev
782 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
783 if p.returncode != 0:
784 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
786 # Get rid of any uncontrolled files left behind
787 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
788 if p.returncode != 0:
789 raise VCSException("Git clean failed", p.output)
793 for treeish in ['origin/', '']:
794 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
800 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
801 if p.returncode != 0:
803 return p.output.strip()
811 def gotorevisionx(self, rev):
812 if not os.path.exists(self.local):
813 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
814 if p.returncode != 0:
815 self.clone_failed = True
816 raise VCSException("Hg clone failed", p.output)
818 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
819 if p.returncode != 0:
820 raise VCSException("Hg status failed", p.output)
821 for line in p.output.splitlines():
822 if not line.startswith('? '):
823 raise VCSException("Unexpected output from hg status -uS: " + line)
824 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
825 if not self.refreshed:
826 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
827 if p.returncode != 0:
828 raise VCSException("Hg pull failed", p.output)
829 self.refreshed = True
831 rev = rev or 'default'
834 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
835 if p.returncode != 0:
836 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
837 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
838 # Also delete untracked files, we have to enable purge extension for that:
839 if "'purge' is provided by the following extension" in p.output:
840 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
841 myfile.write("\n[extensions]\nhgext.purge=\n")
842 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
843 if p.returncode != 0:
844 raise VCSException("HG purge failed", p.output)
845 elif p.returncode != 0:
846 raise VCSException("HG purge failed", p.output)
849 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
850 return p.output.splitlines()[1:]
858 def gotorevisionx(self, rev):
859 if not os.path.exists(self.local):
860 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
861 if p.returncode != 0:
862 self.clone_failed = True
863 raise VCSException("Bzr branch failed", p.output)
865 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
866 if p.returncode != 0:
867 raise VCSException("Bzr revert failed", p.output)
868 if not self.refreshed:
869 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
870 if p.returncode != 0:
871 raise VCSException("Bzr update failed", p.output)
872 self.refreshed = True
874 revargs = list(['-r', rev] if rev else [])
875 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
876 if p.returncode != 0:
877 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
880 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
881 return [tag.split(' ')[0].strip() for tag in
882 p.output.splitlines()]
885 def unescape_string(string):
886 if string[0] == '"' and string[-1] == '"':
889 return string.replace("\\'", "'")
892 def retrieve_string(app_dir, string, xmlfiles=None):
897 os.path.join(app_dir, 'res'),
898 os.path.join(app_dir, 'src', 'main', 'res'),
900 for r, d, f in os.walk(res_dir):
901 if os.path.basename(r) == 'values':
902 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
904 if not string.startswith('@string/'):
905 return unescape_string(string)
907 name = string[len('@string/'):]
909 for path in xmlfiles:
910 if not os.path.isfile(path):
912 xml = parse_xml(path)
913 element = xml.find('string[@name="' + name + '"]')
914 if element is not None and element.text is not None:
915 return retrieve_string(app_dir, element.text.encode('utf-8'), xmlfiles)
920 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
921 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
924 # Return list of existing files that will be used to find the highest vercode
925 def manifest_paths(app_dir, flavours):
927 possible_manifests = \
928 [os.path.join(app_dir, 'AndroidManifest.xml'),
929 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
930 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
931 os.path.join(app_dir, 'build.gradle')]
933 for flavour in flavours:
936 possible_manifests.append(
937 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
939 return [path for path in possible_manifests if os.path.isfile(path)]
942 # Retrieve the package name. Returns the name, or None if not found.
943 def fetch_real_name(app_dir, flavours):
944 for path in manifest_paths(app_dir, flavours):
945 if not has_extension(path, 'xml') or not os.path.isfile(path):
947 logging.debug("fetch_real_name: Checking manifest at " + path)
948 xml = parse_xml(path)
949 app = xml.find('application')
950 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
952 label = app.attrib["{http://schemas.android.com/apk/res/android}label"].encode('utf-8')
953 result = retrieve_string_singleline(app_dir, label)
955 result = result.strip()
960 def get_library_references(root_dir):
962 proppath = os.path.join(root_dir, 'project.properties')
963 if not os.path.isfile(proppath):
965 for line in file(proppath):
966 if not line.startswith('android.library.reference.'):
968 path = line.split('=')[1].strip()
969 relpath = os.path.join(root_dir, path)
970 if not os.path.isdir(relpath):
972 logging.debug("Found subproject at %s" % path)
973 libraries.append(path)
977 def ant_subprojects(root_dir):
978 subprojects = get_library_references(root_dir)
979 for subpath in subprojects:
980 subrelpath = os.path.join(root_dir, subpath)
981 for p in get_library_references(subrelpath):
982 relp = os.path.normpath(os.path.join(subpath, p))
983 if relp not in subprojects:
984 subprojects.insert(0, relp)
988 def remove_debuggable_flags(root_dir):
989 # Remove forced debuggable flags
990 logging.debug("Removing debuggable flags from %s" % root_dir)
991 for root, dirs, files in os.walk(root_dir):
992 if 'AndroidManifest.xml' in files:
993 regsub_file(r'android:debuggable="[^"]*"',
995 os.path.join(root, 'AndroidManifest.xml'))
998 # Extract some information from the AndroidManifest.xml at the given path.
999 # Returns (version, vercode, package), any or all of which might be None.
1000 # All values returned are strings.
1001 def parse_androidmanifests(paths, ignoreversions=None):
1004 return (None, None, None)
1006 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
1007 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1008 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1010 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1018 if not os.path.isfile(path):
1021 logging.debug("Parsing manifest at {0}".format(path))
1022 gradle = has_extension(path, 'gradle')
1028 for line in file(path):
1029 # Grab first occurence of each to avoid running into
1030 # alternative flavours and builds.
1032 matches = psearch_g(line)
1034 package = matches.group(2)
1036 matches = vnsearch_g(line)
1038 version = matches.group(2)
1040 matches = vcsearch_g(line)
1042 vercode = matches.group(1)
1044 xml = parse_xml(path)
1045 if "package" in xml.attrib:
1046 package = xml.attrib["package"].encode('utf-8')
1047 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1048 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"].encode('utf-8')
1049 base_dir = os.path.dirname(path)
1050 version = retrieve_string_singleline(base_dir, version)
1051 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1052 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"].encode('utf-8')
1053 if string_is_integer(a):
1056 # Remember package name, may be defined separately from version+vercode
1058 package = max_package
1060 logging.debug("..got package={0}, version={1}, vercode={2}"
1061 .format(package, version, vercode))
1063 # Always grab the package name and version name in case they are not
1064 # together with the highest version code
1065 if max_package is None and package is not None:
1066 max_package = package
1067 if max_version is None and version is not None:
1068 max_version = version
1070 if max_vercode is None or (vercode is not None and vercode > max_vercode):
1071 if not ignoresearch or not ignoresearch(version):
1072 if version is not None:
1073 max_version = version
1074 if vercode is not None:
1075 max_vercode = vercode
1076 if package is not None:
1077 max_package = package
1079 max_version = "Ignore"
1081 if max_version is None:
1082 max_version = "Unknown"
1084 if max_package and not is_valid_package_name(max_package):
1085 raise FDroidException("Invalid package name {0}".format(max_package))
1087 return (max_version, max_vercode, max_package)
1090 def is_valid_package_name(name):
1091 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1094 class FDroidException(Exception):
1096 def __init__(self, value, detail=None):
1098 self.detail = detail
1100 def get_wikitext(self):
1101 ret = repr(self.value) + "\n"
1105 txt = self.detail[-8192:] if len(self.detail) > 8192 else self.detail
1113 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1117 class VCSException(FDroidException):
1121 class BuildException(FDroidException):
1125 # Get the specified source library.
1126 # Returns the path to it. Normally this is the path to be used when referencing
1127 # it, which may be a subdirectory of the actual project. If you want the base
1128 # directory of the project, pass 'basepath=True'.
1129 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1130 raw=False, prepare=True, preponly=False, refresh=True):
1138 name, ref = spec.split('@')
1140 number, name = name.split(':', 1)
1142 name, subdir = name.split('/', 1)
1144 if name not in metadata.srclibs:
1145 raise VCSException('srclib ' + name + ' not found.')
1147 srclib = metadata.srclibs[name]
1149 sdir = os.path.join(srclib_dir, name)
1152 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1153 vcs.srclib = (name, number, sdir)
1155 vcs.gotorevision(ref, refresh)
1162 libdir = os.path.join(sdir, subdir)
1163 elif srclib["Subdir"]:
1164 for subdir in srclib["Subdir"]:
1165 libdir_candidate = os.path.join(sdir, subdir)
1166 if os.path.exists(libdir_candidate):
1167 libdir = libdir_candidate
1173 remove_signing_keys(sdir)
1174 remove_debuggable_flags(sdir)
1178 if srclib["Prepare"]:
1179 cmd = replace_config_vars(srclib["Prepare"], None)
1181 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1182 if p.returncode != 0:
1183 raise BuildException("Error running prepare command for srclib %s"
1189 return (name, number, libdir)
1192 # Prepare the source code for a particular build
1193 # 'vcs' - the appropriate vcs object for the application
1194 # 'app' - the application details from the metadata
1195 # 'build' - the build details from the metadata
1196 # 'build_dir' - the path to the build directory, usually
1198 # 'srclib_dir' - the path to the source libraries directory, usually
1200 # 'extlib_dir' - the path to the external libraries directory, usually
1202 # Returns the (root, srclibpaths) where:
1203 # 'root' is the root directory, which may be the same as 'build_dir' or may
1204 # be a subdirectory of it.
1205 # 'srclibpaths' is information on the srclibs being used
1206 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1208 # Optionally, the actual app source can be in a subdirectory
1210 root_dir = os.path.join(build_dir, build['subdir'])
1212 root_dir = build_dir
1214 # Get a working copy of the right revision
1215 logging.info("Getting source for revision " + build['commit'])
1216 vcs.gotorevision(build['commit'], refresh)
1218 # Initialise submodules if required
1219 if build['submodules']:
1220 logging.info("Initialising submodules")
1221 vcs.initsubmodules()
1223 # Check that a subdir (if we're using one) exists. This has to happen
1224 # after the checkout, since it might not exist elsewhere
1225 if not os.path.exists(root_dir):
1226 raise BuildException('Missing subdir ' + root_dir)
1228 # Run an init command if one is required
1230 cmd = replace_config_vars(build['init'], build)
1231 logging.info("Running 'init' commands in %s" % root_dir)
1233 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1234 if p.returncode != 0:
1235 raise BuildException("Error running init command for %s:%s" %
1236 (app['id'], build['version']), p.output)
1238 # Apply patches if any
1240 logging.info("Applying patches")
1241 for patch in build['patch']:
1242 patch = patch.strip()
1243 logging.info("Applying " + patch)
1244 patch_path = os.path.join('metadata', app['id'], patch)
1245 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1246 if p.returncode != 0:
1247 raise BuildException("Failed to apply patch %s" % patch_path)
1249 # Get required source libraries
1251 if build['srclibs']:
1252 logging.info("Collecting source libraries")
1253 for lib in build['srclibs']:
1254 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
1256 for name, number, libpath in srclibpaths:
1257 place_srclib(root_dir, int(number) if number else None, libpath)
1259 basesrclib = vcs.getsrclib()
1260 # If one was used for the main source, add that too.
1262 srclibpaths.append(basesrclib)
1264 # Update the local.properties file
1265 localprops = [os.path.join(build_dir, 'local.properties')]
1267 localprops += [os.path.join(root_dir, 'local.properties')]
1268 for path in localprops:
1270 if os.path.isfile(path):
1271 logging.info("Updating local.properties file at %s" % path)
1272 with open(path, 'r') as f:
1276 logging.info("Creating local.properties file at %s" % path)
1277 # Fix old-fashioned 'sdk-location' by copying
1278 # from sdk.dir, if necessary
1279 if build['oldsdkloc']:
1280 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1281 re.S | re.M).group(1)
1282 props += "sdk-location=%s\n" % sdkloc
1284 props += "sdk.dir=%s\n" % config['sdk_path']
1285 props += "sdk-location=%s\n" % config['sdk_path']
1286 if build['ndk_path']:
1288 props += "ndk.dir=%s\n" % build['ndk_path']
1289 props += "ndk-location=%s\n" % build['ndk_path']
1290 # Add java.encoding if necessary
1291 if build['encoding']:
1292 props += "java.encoding=%s\n" % build['encoding']
1293 with open(path, 'w') as f:
1297 if build['type'] == 'gradle':
1298 flavours = build['gradle']
1300 version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1301 gradlepluginver = None
1303 gradle_dirs = [root_dir]
1305 # Parent dir build.gradle
1306 parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
1307 if parent_dir.startswith(build_dir):
1308 gradle_dirs.append(parent_dir)
1310 for dir_path in gradle_dirs:
1313 if not os.path.isdir(dir_path):
1315 for filename in os.listdir(dir_path):
1316 if not filename.endswith('.gradle'):
1318 path = os.path.join(dir_path, filename)
1319 if not os.path.isfile(path):
1321 for line in file(path):
1322 match = version_regex.match(line)
1324 gradlepluginver = match.group(1)
1328 build['gradlepluginver'] = LooseVersion(gradlepluginver)
1330 logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
1331 build['gradlepluginver'] = LooseVersion('0.11')
1334 n = build["target"].split('-')[1]
1335 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1336 r'compileSdkVersion %s' % n,
1337 os.path.join(root_dir, 'build.gradle'))
1339 # Remove forced debuggable flags
1340 remove_debuggable_flags(root_dir)
1342 # Insert version code and number into the manifest if necessary
1343 if build['forceversion']:
1344 logging.info("Changing the version name")
1345 for path in manifest_paths(root_dir, flavours):
1346 if not os.path.isfile(path):
1348 if has_extension(path, 'xml'):
1349 regsub_file(r'android:versionName="[^"]*"',
1350 r'android:versionName="%s"' % build['version'],
1352 elif has_extension(path, 'gradle'):
1353 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1354 r"""\1versionName '%s'""" % build['version'],
1357 if build['forcevercode']:
1358 logging.info("Changing the version code")
1359 for path in manifest_paths(root_dir, flavours):
1360 if not os.path.isfile(path):
1362 if has_extension(path, 'xml'):
1363 regsub_file(r'android:versionCode="[^"]*"',
1364 r'android:versionCode="%s"' % build['vercode'],
1366 elif has_extension(path, 'gradle'):
1367 regsub_file(r'versionCode[ =]+[0-9]+',
1368 r'versionCode %s' % build['vercode'],
1371 # Delete unwanted files
1373 logging.info("Removing specified files")
1374 for part in getpaths(build_dir, build, 'rm'):
1375 dest = os.path.join(build_dir, part)
1376 logging.info("Removing {0}".format(part))
1377 if os.path.lexists(dest):
1378 if os.path.islink(dest):
1379 FDroidPopen(['unlink', dest], output=False)
1381 FDroidPopen(['rm', '-rf', dest], output=False)
1383 logging.info("...but it didn't exist")
1385 remove_signing_keys(build_dir)
1387 # Add required external libraries
1388 if build['extlibs']:
1389 logging.info("Collecting prebuilt libraries")
1390 libsdir = os.path.join(root_dir, 'libs')
1391 if not os.path.exists(libsdir):
1393 for lib in build['extlibs']:
1395 logging.info("...installing extlib {0}".format(lib))
1396 libf = os.path.basename(lib)
1397 libsrc = os.path.join(extlib_dir, lib)
1398 if not os.path.exists(libsrc):
1399 raise BuildException("Missing extlib file {0}".format(libsrc))
1400 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1402 # Run a pre-build command if one is required
1403 if build['prebuild']:
1404 logging.info("Running 'prebuild' commands in %s" % root_dir)
1406 cmd = replace_config_vars(build['prebuild'], build)
1408 # Substitute source library paths into prebuild commands
1409 for name, number, libpath in srclibpaths:
1410 libpath = os.path.relpath(libpath, root_dir)
1411 cmd = cmd.replace('$$' + name + '$$', libpath)
1413 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1414 if p.returncode != 0:
1415 raise BuildException("Error running prebuild command for %s:%s" %
1416 (app['id'], build['version']), p.output)
1418 # Generate (or update) the ant build file, build.xml...
1419 if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
1420 parms = ['android', 'update', 'lib-project']
1421 lparms = ['android', 'update', 'project']
1424 parms += ['-t', build['target']]
1425 lparms += ['-t', build['target']]
1426 if build['update'] == ['auto']:
1427 update_dirs = ant_subprojects(root_dir) + ['.']
1429 update_dirs = build['update']
1431 for d in update_dirs:
1432 subdir = os.path.join(root_dir, d)
1434 logging.debug("Updating main project")
1435 cmd = parms + ['-p', d]
1437 logging.debug("Updating subproject %s" % d)
1438 cmd = lparms + ['-p', d]
1439 p = SdkToolsPopen(cmd, cwd=root_dir)
1440 # Check to see whether an error was returned without a proper exit
1441 # code (this is the case for the 'no target set or target invalid'
1443 if p.returncode != 0 or p.output.startswith("Error: "):
1444 raise BuildException("Failed to update project at %s" % d, p.output)
1445 # Clean update dirs via ant
1447 logging.info("Cleaning subproject %s" % d)
1448 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1450 return (root_dir, srclibpaths)
1453 # Split and extend via globbing the paths from a field
1454 def getpaths(build_dir, build, field):
1456 for p in build[field]:
1458 full_path = os.path.join(build_dir, p)
1459 full_path = os.path.normpath(full_path)
1460 paths += [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1465 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1471 self.path = os.path.join('stats', 'known_apks.txt')
1473 if os.path.isfile(self.path):
1474 for line in file(self.path):
1475 t = line.rstrip().split(' ')
1477 self.apks[t[0]] = (t[1], None)
1479 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1480 self.changed = False
1482 def writeifchanged(self):
1483 if not self.changed:
1486 if not os.path.exists('stats'):
1490 for apk, app in self.apks.iteritems():
1492 line = apk + ' ' + appid
1494 line += ' ' + time.strftime('%Y-%m-%d', added)
1497 with open(self.path, 'w') as f:
1498 for line in sorted(lst, key=natural_key):
1499 f.write(line + '\n')
1501 # Record an apk (if it's new, otherwise does nothing)
1502 # Returns the date it was added.
1503 def recordapk(self, apk, app):
1504 if apk not in self.apks:
1505 self.apks[apk] = (app, time.gmtime(time.time()))
1507 _, added = self.apks[apk]
1510 # Look up information - given the 'apkname', returns (app id, date added/None).
1511 # Or returns None for an unknown apk.
1512 def getapp(self, apkname):
1513 if apkname in self.apks:
1514 return self.apks[apkname]
1517 # Get the most recent 'num' apps added to the repo, as a list of package ids
1518 # with the most recent first.
1519 def getlatest(self, num):
1521 for apk, app in self.apks.iteritems():
1525 if apps[appid] > added:
1529 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1530 lst = [app for app, _ in sortedapps]
1535 def isApkDebuggable(apkfile, config):
1536 """Returns True if the given apk file is debuggable
1538 :param apkfile: full path to the apk to check"""
1540 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1542 if p.returncode != 0:
1543 logging.critical("Failed to get apk manifest information")
1545 for line in p.output.splitlines():
1546 if 'android:debuggable' in line and not line.endswith('0x0'):
1556 def SdkToolsPopen(commands, cwd=None, output=True):
1558 if cmd not in config:
1559 config[cmd] = find_sdk_tools_cmd(commands[0])
1560 return FDroidPopen([config[cmd]] + commands[1:],
1561 cwd=cwd, output=output)
1564 def FDroidPopen(commands, cwd=None, output=True):
1566 Run a command and capture the possibly huge output.
1568 :param commands: command and argument list like in subprocess.Popen
1569 :param cwd: optionally specifies a working directory
1570 :returns: A PopenResult.
1576 cwd = os.path.normpath(cwd)
1577 logging.debug("Directory: %s" % cwd)
1578 logging.debug("> %s" % ' '.join(commands))
1580 result = PopenResult()
1583 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
1584 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1586 raise BuildException("OSError while trying to execute " +
1587 ' '.join(commands) + ': ' + str(e))
1589 stdout_queue = Queue.Queue()
1590 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1592 # Check the queue for output (until there is no more to get)
1593 while not stdout_reader.eof():
1594 while not stdout_queue.empty():
1595 line = stdout_queue.get()
1596 if output and options.verbose:
1597 # Output directly to console
1598 sys.stderr.write(line)
1600 result.output += line
1604 result.returncode = p.wait()
1608 def remove_signing_keys(build_dir):
1609 comment = re.compile(r'[ ]*//')
1610 signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1612 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1613 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1614 re.compile(r'.*variant\.outputFile = .*'),
1615 re.compile(r'.*output\.outputFile = .*'),
1616 re.compile(r'.*\.readLine\(.*'),
1618 for root, dirs, files in os.walk(build_dir):
1619 if 'build.gradle' in files:
1620 path = os.path.join(root, 'build.gradle')
1622 with open(path, "r") as o:
1623 lines = o.readlines()
1629 with open(path, "w") as o:
1630 while i < len(lines):
1633 while line.endswith('\\\n'):
1634 line = line.rstrip('\\\n') + lines[i]
1637 if comment.match(line):
1641 opened += line.count('{')
1642 opened -= line.count('}')
1645 if signing_configs.match(line):
1650 if any(s.match(line) for s in line_matches):
1658 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1661 'project.properties',
1663 'default.properties',
1664 'ant.properties', ]:
1665 if propfile in files:
1666 path = os.path.join(root, propfile)
1668 with open(path, "r") as o:
1669 lines = o.readlines()
1673 with open(path, "w") as o:
1675 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1682 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1685 def reset_env_path():
1686 global env, orig_path
1687 env['PATH'] = orig_path
1690 def add_to_env_path(path):
1692 paths = env['PATH'].split(os.pathsep)
1696 env['PATH'] = os.pathsep.join(paths)
1699 def replace_config_vars(cmd, build):
1701 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1702 # env['ANDROID_NDK'] is set in build_local right before prepare_source
1703 cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
1704 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1705 if build is not None:
1706 cmd = cmd.replace('$$COMMIT$$', build['commit'])
1707 cmd = cmd.replace('$$VERSION$$', build['version'])
1708 cmd = cmd.replace('$$VERCODE$$', build['vercode'])
1712 def place_srclib(root_dir, number, libpath):
1715 relpath = os.path.relpath(libpath, root_dir)
1716 proppath = os.path.join(root_dir, 'project.properties')
1719 if os.path.isfile(proppath):
1720 with open(proppath, "r") as o:
1721 lines = o.readlines()
1723 with open(proppath, "w") as o:
1726 if line.startswith('android.library.reference.%d=' % number):
1727 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1732 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1735 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
1736 """Verify that two apks are the same
1738 One of the inputs is signed, the other is unsigned. The signature metadata
1739 is transferred from the signed to the unsigned apk, and then jarsigner is
1740 used to verify that the signature from the signed apk is also varlid for
1742 :param signed_apk: Path to a signed apk file
1743 :param unsigned_apk: Path to an unsigned apk file expected to match it
1744 :param tmp_dir: Path to directory for temporary files
1745 :returns: None if the verification is successful, otherwise a string
1746 describing what went wrong.
1748 sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
1749 with ZipFile(signed_apk) as signed_apk_as_zip:
1750 meta_inf_files = ['META-INF/MANIFEST.MF']
1751 for f in signed_apk_as_zip.namelist():
1752 if sigfile.match(f):
1753 meta_inf_files.append(f)
1754 if len(meta_inf_files) < 3:
1755 return "Signature files missing from {0}".format(signed_apk)
1756 signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
1757 with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
1758 for meta_inf_file in meta_inf_files:
1759 unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
1761 if subprocess.call(['jarsigner', '-verify', unsigned_apk]) != 0:
1762 logging.info("...NOT verified - {0}".format(signed_apk))
1763 return compare_apks(signed_apk, unsigned_apk, tmp_dir)
1764 logging.info("...successfully verified")
1768 def compare_apks(apk1, apk2, tmp_dir):
1771 Returns None if the apk content is the same (apart from the signing key),
1772 otherwise a string describing what's different, or what went wrong when
1773 trying to do the comparison.
1776 badchars = re.compile('''[/ :;'"]''')
1777 apk1dir = os.path.join(tmp_dir, badchars.sub('_', apk1[0:-4])) # trim .apk
1778 apk2dir = os.path.join(tmp_dir, badchars.sub('_', apk2[0:-4])) # trim .apk
1779 for d in [apk1dir, apk2dir]:
1780 if os.path.exists(d):
1783 os.mkdir(os.path.join(d, 'jar-xf'))
1785 if subprocess.call(['jar', 'xf',
1786 os.path.abspath(apk1)],
1787 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
1788 return("Failed to unpack " + apk1)
1789 if subprocess.call(['jar', 'xf',
1790 os.path.abspath(apk2)],
1791 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
1792 return("Failed to unpack " + apk2)
1794 # try to find apktool in the path, if it hasn't been manually configed
1795 if 'apktool' not in config:
1796 tmp = find_command('apktool')
1798 config['apktool'] = tmp
1799 if 'apktool' in config:
1800 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
1802 return("Failed to unpack " + apk1)
1803 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
1805 return("Failed to unpack " + apk2)
1807 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
1808 lines = p.output.splitlines()
1809 if len(lines) != 1 or 'META-INF' not in lines[0]:
1810 meld = find_command('meld')
1811 if meld is not None:
1812 p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
1813 return("Unexpected diff output - " + p.output)
1815 # since everything verifies, delete the comparison to keep cruft down
1816 shutil.rmtree(apk1dir)
1817 shutil.rmtree(apk2dir)
1819 # If we get here, it seems like they're the same!
1823 def find_command(command):
1824 '''find the full path of a command, or None if it can't be found in the PATH'''
1827 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
1829 fpath, fname = os.path.split(command)
1834 for path in os.environ["PATH"].split(os.pathsep):
1835 path = path.strip('"')
1836 exe_file = os.path.join(path, command)
1837 if is_exe(exe_file):
1844 '''generate a random password for when generating keys'''
1845 h = hashlib.sha256()
1846 h.update(os.urandom(16)) # salt
1847 h.update(bytes(socket.getfqdn()))
1848 return h.digest().encode('base64').strip()
1851 def genkeystore(localconfig):
1852 '''Generate a new key with random passwords and add it to new keystore'''
1853 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
1854 keystoredir = os.path.dirname(localconfig['keystore'])
1855 if keystoredir is None or keystoredir == '':
1856 keystoredir = os.path.join(os.getcwd(), keystoredir)
1857 if not os.path.exists(keystoredir):
1858 os.makedirs(keystoredir, mode=0o700)
1860 write_password_file("keystorepass", localconfig['keystorepass'])
1861 write_password_file("keypass", localconfig['keypass'])
1862 p = FDroidPopen(['keytool', '-genkey',
1863 '-keystore', localconfig['keystore'],
1864 '-alias', localconfig['repo_keyalias'],
1865 '-keyalg', 'RSA', '-keysize', '4096',
1866 '-sigalg', 'SHA256withRSA',
1867 '-validity', '10000',
1868 '-storepass:file', config['keystorepassfile'],
1869 '-keypass:file', config['keypassfile'],
1870 '-dname', localconfig['keydname']])
1871 # TODO keypass should be sent via stdin
1872 if p.returncode != 0:
1873 raise BuildException("Failed to generate key", p.output)
1874 os.chmod(localconfig['keystore'], 0o0600)
1875 # now show the lovely key that was just generated
1876 p = FDroidPopen(['keytool', '-list', '-v',
1877 '-keystore', localconfig['keystore'],
1878 '-alias', localconfig['repo_keyalias'],
1879 '-storepass:file', config['keystorepassfile']])
1880 logging.info(p.output.strip() + '\n\n')
1883 def write_to_config(thisconfig, key, value=None):
1884 '''write a key/value to the local config.py'''
1886 origkey = key + '_orig'
1887 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
1888 with open('config.py', 'r') as f:
1890 pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
1891 repl = '\n' + key + ' = "' + value + '"'
1892 data = re.sub(pattern, repl, data)
1893 # if this key is not in the file, append it
1894 if not re.match('\s*' + key + '\s*=\s*"', data):
1896 # make sure the file ends with a carraige return
1897 if not re.match('\n$', data):
1899 with open('config.py', 'w') as f:
1903 def parse_xml(path):
1904 return XMLElementTree.parse(path).getroot()
1907 def string_is_integer(string):
1915 def get_per_app_repos():
1916 '''per-app repos are dirs named with the packageName of a single app'''
1918 # Android packageNames are Java packages, they may contain uppercase or
1919 # lowercase letters ('A' through 'Z'), numbers, and underscores
1920 # ('_'). However, individual package name parts may only start with
1921 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
1922 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
1925 for root, dirs, files in os.walk(os.getcwd()):
1927 print 'checking', root, 'for', d
1928 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
1929 # standard parts of an fdroid repo, so never packageNames
1932 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):