1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
36 import xml.etree.ElementTree as XMLElementTree
38 from distutils.version import LooseVersion
39 from zipfile import ZipFile
42 from fdroidserver.asynchronousfilereader import AsynchronousFileReader
45 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
54 'sdk_path': "$ANDROID_HOME",
57 'r10e': "$ANDROID_NDK",
59 'build_tools': "23.0.1",
63 'accepted_formats': ['txt', 'yaml'],
64 'sync_from_local_copy_dir': False,
65 'per_app_repos': False,
66 'make_current_version_link': True,
67 'current_version_name_source': 'Name',
68 'update_stats': False,
72 'stats_to_carbon': False,
74 'build_server_always': False,
75 'keystore': 'keystore.jks',
76 'smartcardoptions': [],
82 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
83 'repo_name': "My First FDroid Repo Demo",
84 'repo_icon': "fdroid-icon.png",
85 'repo_description': '''
86 This is a repository of apps to be used with FDroid. Applications in this
87 repository are either official binaries built by the original application
88 developers, or are binaries built from source by the admin of f-droid.org
89 using the tools on https://gitlab.com/u/fdroid.
95 def setup_global_opts(parser):
96 parser.add_argument("-v", "--verbose", action="store_true", default=False,
97 help="Spew out even more information than normal")
98 parser.add_argument("-q", "--quiet", action="store_true", default=False,
99 help="Restrict output to warnings and errors")
102 def fill_config_defaults(thisconfig):
103 for k, v in default_config.items():
104 if k not in thisconfig:
107 # Expand paths (~users and $vars)
108 def expand_path(path):
112 path = os.path.expanduser(path)
113 path = os.path.expandvars(path)
118 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
123 thisconfig[k + '_orig'] = v
125 for k in ['ndk_paths']:
131 thisconfig[k][k2] = exp
132 thisconfig[k][k2 + '_orig'] = v
135 def regsub_file(pattern, repl, path):
136 with open(path, 'r') as f:
138 text = re.sub(pattern, repl, text)
139 with open(path, 'w') as f:
143 def read_config(opts, config_file='config.py'):
144 """Read the repository config
146 The config is read from config_file, which is in the current directory when
147 any of the repo management commands are used.
149 global config, options, env, orig_path
151 if config is not None:
153 if not os.path.isfile(config_file):
154 logging.critical("Missing config file - is this a repo directory?")
161 logging.debug("Reading %s" % config_file)
162 execfile(config_file, config)
164 # smartcardoptions must be a list since its command line args for Popen
165 if 'smartcardoptions' in config:
166 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
167 elif 'keystore' in config and config['keystore'] == 'NONE':
168 # keystore='NONE' means use smartcard, these are required defaults
169 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
170 'SunPKCS11-OpenSC', '-providerClass',
171 'sun.security.pkcs11.SunPKCS11',
172 '-providerArg', 'opensc-fdroid.cfg']
174 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
175 st = os.stat(config_file)
176 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
177 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
179 fill_config_defaults(config)
181 # There is no standard, so just set up the most common environment
184 orig_path = env['PATH']
185 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
186 env[n] = config['sdk_path']
188 for k in ["keystorepass", "keypass"]:
190 write_password_file(k)
192 for k in ["repo_description", "archive_description"]:
194 config[k] = clean_description(config[k])
196 if 'serverwebroot' in config:
197 if isinstance(config['serverwebroot'], basestring):
198 roots = [config['serverwebroot']]
199 elif all(isinstance(item, basestring) for item in config['serverwebroot']):
200 roots = config['serverwebroot']
202 raise TypeError('only accepts strings, lists, and tuples')
204 for rootstr in roots:
205 # since this is used with rsync, where trailing slashes have
206 # meaning, ensure there is always a trailing slash
207 if rootstr[-1] != '/':
209 rootlist.append(rootstr.replace('//', '/'))
210 config['serverwebroot'] = rootlist
215 def get_ndk_path(version):
217 version = 'r10e' # falls back to latest
218 paths = config['ndk_paths']
219 if version not in paths:
221 return paths[version] or ''
224 def find_sdk_tools_cmd(cmd):
225 '''find a working path to a tool from the Android SDK'''
228 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
229 # try to find a working path to this command, in all the recent possible paths
230 if 'build_tools' in config:
231 build_tools = os.path.join(config['sdk_path'], 'build-tools')
232 # if 'build_tools' was manually set and exists, check only that one
233 configed_build_tools = os.path.join(build_tools, config['build_tools'])
234 if os.path.exists(configed_build_tools):
235 tooldirs.append(configed_build_tools)
237 # no configed version, so hunt known paths for it
238 for f in sorted(os.listdir(build_tools), reverse=True):
239 if os.path.isdir(os.path.join(build_tools, f)):
240 tooldirs.append(os.path.join(build_tools, f))
241 tooldirs.append(build_tools)
242 sdk_tools = os.path.join(config['sdk_path'], 'tools')
243 if os.path.exists(sdk_tools):
244 tooldirs.append(sdk_tools)
245 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
246 if os.path.exists(sdk_platform_tools):
247 tooldirs.append(sdk_platform_tools)
248 tooldirs.append('/usr/bin')
250 if os.path.isfile(os.path.join(d, cmd)):
251 return os.path.join(d, cmd)
252 # did not find the command, exit with error message
253 ensure_build_tools_exists(config)
256 def test_sdk_exists(thisconfig):
257 if 'sdk_path' not in thisconfig:
258 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
261 logging.error("'sdk_path' not set in config.py!")
263 if thisconfig['sdk_path'] == default_config['sdk_path']:
264 logging.error('No Android SDK found!')
265 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
266 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
268 if not os.path.exists(thisconfig['sdk_path']):
269 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
271 if not os.path.isdir(thisconfig['sdk_path']):
272 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
274 for d in ['build-tools', 'platform-tools', 'tools']:
275 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
276 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
277 thisconfig['sdk_path'], d))
282 def ensure_build_tools_exists(thisconfig):
283 if not test_sdk_exists(thisconfig):
285 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
286 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
287 if not os.path.isdir(versioned_build_tools):
288 logging.critical('Android Build Tools path "'
289 + versioned_build_tools + '" does not exist!')
293 def write_password_file(pwtype, password=None):
295 writes out passwords to a protected file instead of passing passwords as
296 command line argments
298 filename = '.fdroid.' + pwtype + '.txt'
299 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
301 os.write(fd, config[pwtype])
303 os.write(fd, password)
305 config[pwtype + 'file'] = filename
308 # Given the arguments in the form of multiple appid:[vc] strings, this returns
309 # a dictionary with the set of vercodes specified for each package.
310 def read_pkg_args(args, allow_vercodes=False):
317 if allow_vercodes and ':' in p:
318 package, vercode = p.split(':')
320 package, vercode = p, None
321 if package not in vercodes:
322 vercodes[package] = [vercode] if vercode else []
324 elif vercode and vercode not in vercodes[package]:
325 vercodes[package] += [vercode] if vercode else []
330 # On top of what read_pkg_args does, this returns the whole app metadata, but
331 # limiting the builds list to the builds matching the vercodes specified.
332 def read_app_args(args, allapps, allow_vercodes=False):
334 vercodes = read_pkg_args(args, allow_vercodes)
340 for appid, app in allapps.iteritems():
341 if appid in vercodes:
344 if len(apps) != len(vercodes):
347 logging.critical("No such package: %s" % p)
348 raise FDroidException("Found invalid app ids in arguments")
350 raise FDroidException("No packages specified")
353 for appid, app in apps.iteritems():
357 app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
358 if len(app['builds']) != len(vercodes[appid]):
360 allvcs = [b['vercode'] for b in app['builds']]
361 for v in vercodes[appid]:
363 logging.critical("No such vercode %s for app %s" % (v, appid))
366 raise FDroidException("Found invalid vercodes for some apps")
371 def get_extension(filename):
372 _, ext = os.path.splitext(filename)
375 return ext.lower()[1:]
378 def has_extension(filename, ext):
379 return ext == get_extension(filename)
385 def clean_description(description):
386 'Remove unneeded newlines and spaces from a block of description text'
388 # this is split up by paragraph to make removing the newlines easier
389 for paragraph in re.split(r'\n\n', description):
390 paragraph = re.sub('\r', '', paragraph)
391 paragraph = re.sub('\n', ' ', paragraph)
392 paragraph = re.sub(' {2,}', ' ', paragraph)
393 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
394 returnstring += paragraph + '\n\n'
395 return returnstring.rstrip('\n')
398 def apknameinfo(filename):
400 filename = os.path.basename(filename)
401 if apk_regex is None:
402 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
403 m = apk_regex.match(filename)
405 result = (m.group(1), m.group(2))
406 except AttributeError:
407 raise FDroidException("Invalid apk name: %s" % filename)
411 def getapkname(app, build):
412 return "%s_%s.apk" % (app['id'], build['vercode'])
415 def getsrcname(app, build):
416 return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
423 return app['Auto Name']
428 return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
431 def getvcs(vcstype, remote, local):
433 return vcs_git(remote, local)
434 if vcstype == 'git-svn':
435 return vcs_gitsvn(remote, local)
437 return vcs_hg(remote, local)
439 return vcs_bzr(remote, local)
440 if vcstype == 'srclib':
441 if local != os.path.join('build', 'srclib', remote):
442 raise VCSException("Error: srclib paths are hard-coded!")
443 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
445 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
446 raise VCSException("Invalid vcs type " + vcstype)
449 def getsrclibvcs(name):
450 if name not in metadata.srclibs:
451 raise VCSException("Missing srclib " + name)
452 return metadata.srclibs[name]['Repo Type']
457 def __init__(self, remote, local):
459 # svn, git-svn and bzr may require auth
461 if self.repotype() in ('git-svn', 'bzr'):
463 if self.repotype == 'git-svn':
464 raise VCSException("Authentication is not supported for git-svn")
465 self.username, remote = remote.split('@')
466 if ':' not in self.username:
467 raise VCSException("Password required with username")
468 self.username, self.password = self.username.split(':')
472 self.clone_failed = False
473 self.refreshed = False
479 # Take the local repository to a clean version of the given revision, which
480 # is specificed in the VCS's native format. Beforehand, the repository can
481 # be dirty, or even non-existent. If the repository does already exist
482 # locally, it will be updated from the origin, but only once in the
483 # lifetime of the vcs object.
484 # None is acceptable for 'rev' if you know you are cloning a clean copy of
485 # the repo - otherwise it must specify a valid revision.
486 def gotorevision(self, rev, refresh=True):
488 if self.clone_failed:
489 raise VCSException("Downloading the repository already failed once, not trying again.")
491 # The .fdroidvcs-id file for a repo tells us what VCS type
492 # and remote that directory was created from, allowing us to drop it
493 # automatically if either of those things changes.
494 fdpath = os.path.join(self.local, '..',
495 '.fdroidvcs-' + os.path.basename(self.local))
496 cdata = self.repotype() + ' ' + self.remote
499 if os.path.exists(self.local):
500 if os.path.exists(fdpath):
501 with open(fdpath, 'r') as f:
502 fsdata = f.read().strip()
507 logging.info("Repository details for %s changed - deleting" % (
511 logging.info("Repository details for %s missing - deleting" % (
514 shutil.rmtree(self.local)
518 self.refreshed = True
521 self.gotorevisionx(rev)
522 except FDroidException, e:
525 # If necessary, write the .fdroidvcs file.
526 if writeback and not self.clone_failed:
527 with open(fdpath, 'w') as f:
533 # Derived classes need to implement this. It's called once basic checking
534 # has been performend.
535 def gotorevisionx(self, rev):
536 raise VCSException("This VCS type doesn't define gotorevisionx")
538 # Initialise and update submodules
539 def initsubmodules(self):
540 raise VCSException('Submodules not supported for this vcs type')
542 # Get a list of all known tags
544 if not self._gettags:
545 raise VCSException('gettags not supported for this vcs type')
547 for tag in self._gettags():
548 if re.match('[-A-Za-z0-9_. /]+$', tag):
552 def latesttags(self, tags, number):
553 """Get the most recent tags in a given list.
555 :param tags: a list of tags
556 :param number: the number to return
557 :returns: A list containing the most recent tags in the provided
558 list, up to the maximum number given.
560 raise VCSException('latesttags not supported for this vcs type')
562 # Get current commit reference (hash, revision, etc)
564 raise VCSException('getref not supported for this vcs type')
566 # Returns the srclib (name, path) used in setting up the current
577 # If the local directory exists, but is somehow not a git repository, git
578 # will traverse up the directory tree until it finds one that is (i.e.
579 # fdroidserver) and then we'll proceed to destroy it! This is called as
582 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
583 result = p.output.rstrip()
584 if not result.endswith(self.local):
585 raise VCSException('Repository mismatch')
587 def gotorevisionx(self, rev):
588 if not os.path.exists(self.local):
590 p = FDroidPopen(['git', 'clone', self.remote, self.local])
591 if p.returncode != 0:
592 self.clone_failed = True
593 raise VCSException("Git clone failed", p.output)
597 # Discard any working tree changes
598 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
599 'git', 'reset', '--hard'], cwd=self.local, output=False)
600 if p.returncode != 0:
601 raise VCSException("Git reset failed", p.output)
602 # Remove untracked files now, in case they're tracked in the target
603 # revision (it happens!)
604 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
605 'git', 'clean', '-dffx'], cwd=self.local, output=False)
606 if p.returncode != 0:
607 raise VCSException("Git clean failed", p.output)
608 if not self.refreshed:
609 # Get latest commits and tags from remote
610 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
611 if p.returncode != 0:
612 raise VCSException("Git fetch failed", p.output)
613 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
614 if p.returncode != 0:
615 raise VCSException("Git fetch failed", p.output)
616 # Recreate origin/HEAD as git clone would do it, in case it disappeared
617 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
618 if p.returncode != 0:
619 lines = p.output.splitlines()
620 if 'Multiple remote HEAD branches' not in lines[0]:
621 raise VCSException("Git remote set-head failed", p.output)
622 branch = lines[1].split(' ')[-1]
623 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
624 if p2.returncode != 0:
625 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
626 self.refreshed = True
627 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
628 # a github repo. Most of the time this is the same as origin/master.
629 rev = rev or 'origin/HEAD'
630 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
631 if p.returncode != 0:
632 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
633 # Get rid of any uncontrolled files left behind
634 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
635 if p.returncode != 0:
636 raise VCSException("Git clean failed", p.output)
638 def initsubmodules(self):
640 submfile = os.path.join(self.local, '.gitmodules')
641 if not os.path.isfile(submfile):
642 raise VCSException("No git submodules available")
644 # fix submodules not accessible without an account and public key auth
645 with open(submfile, 'r') as f:
646 lines = f.readlines()
647 with open(submfile, 'w') as f:
649 if 'git@github.com' in line:
650 line = line.replace('git@github.com:', 'https://github.com/')
653 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
654 if p.returncode != 0:
655 raise VCSException("Git submodule sync failed", p.output)
656 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
657 if p.returncode != 0:
658 raise VCSException("Git submodule update failed", p.output)
662 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
663 return p.output.splitlines()
665 def latesttags(self, tags, number):
670 ['git', 'show', '--format=format:%ct', '-s', tag],
671 cwd=self.local, output=False)
672 # Timestamp is on the last line. For a normal tag, it's the only
673 # line, but for annotated tags, the rest of the info precedes it.
674 ts = int(p.output.splitlines()[-1])
677 for _, t in sorted(tl)[-number:]:
682 class vcs_gitsvn(vcs):
687 # If the local directory exists, but is somehow not a git repository, git
688 # will traverse up the directory tree until it finds one that is (i.e.
689 # fdroidserver) and then we'll proceed to destory it! This is called as
692 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
693 result = p.output.rstrip()
694 if not result.endswith(self.local):
695 raise VCSException('Repository mismatch')
697 def gotorevisionx(self, rev):
698 if not os.path.exists(self.local):
700 gitsvn_args = ['git', 'svn', 'clone']
701 if ';' in self.remote:
702 remote_split = self.remote.split(';')
703 for i in remote_split[1:]:
704 if i.startswith('trunk='):
705 gitsvn_args.extend(['-T', i[6:]])
706 elif i.startswith('tags='):
707 gitsvn_args.extend(['-t', i[5:]])
708 elif i.startswith('branches='):
709 gitsvn_args.extend(['-b', i[9:]])
710 gitsvn_args.extend([remote_split[0], self.local])
711 p = FDroidPopen(gitsvn_args, output=False)
712 if p.returncode != 0:
713 self.clone_failed = True
714 raise VCSException("Git svn clone failed", p.output)
716 gitsvn_args.extend([self.remote, self.local])
717 p = FDroidPopen(gitsvn_args, output=False)
718 if p.returncode != 0:
719 self.clone_failed = True
720 raise VCSException("Git svn clone failed", p.output)
724 # Discard any working tree changes
725 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
726 if p.returncode != 0:
727 raise VCSException("Git reset failed", p.output)
728 # Remove untracked files now, in case they're tracked in the target
729 # revision (it happens!)
730 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
731 if p.returncode != 0:
732 raise VCSException("Git clean failed", p.output)
733 if not self.refreshed:
734 # Get new commits, branches and tags from repo
735 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
736 if p.returncode != 0:
737 raise VCSException("Git svn fetch failed")
738 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
739 if p.returncode != 0:
740 raise VCSException("Git svn rebase failed", p.output)
741 self.refreshed = True
743 rev = rev or 'master'
745 nospaces_rev = rev.replace(' ', '%20')
746 # Try finding a svn tag
747 for treeish in ['origin/', '']:
748 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
749 if p.returncode == 0:
751 if p.returncode != 0:
752 # No tag found, normal svn rev translation
753 # Translate svn rev into git format
754 rev_split = rev.split('/')
757 for treeish in ['origin/', '']:
758 if len(rev_split) > 1:
759 treeish += rev_split[0]
760 svn_rev = rev_split[1]
763 # if no branch is specified, then assume trunk (i.e. 'master' branch):
767 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
769 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
770 git_rev = p.output.rstrip()
772 if p.returncode == 0 and git_rev:
775 if p.returncode != 0 or not git_rev:
776 # Try a plain git checkout as a last resort
777 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
778 if p.returncode != 0:
779 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
781 # Check out the git rev equivalent to the svn rev
782 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
783 if p.returncode != 0:
784 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
786 # Get rid of any uncontrolled files left behind
787 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
788 if p.returncode != 0:
789 raise VCSException("Git clean failed", p.output)
793 for treeish in ['origin/', '']:
794 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
800 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
801 if p.returncode != 0:
803 return p.output.strip()
811 def gotorevisionx(self, rev):
812 if not os.path.exists(self.local):
813 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
814 if p.returncode != 0:
815 self.clone_failed = True
816 raise VCSException("Hg clone failed", p.output)
818 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
819 if p.returncode != 0:
820 raise VCSException("Hg status failed", p.output)
821 for line in p.output.splitlines():
822 if not line.startswith('? '):
823 raise VCSException("Unexpected output from hg status -uS: " + line)
824 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
825 if not self.refreshed:
826 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
827 if p.returncode != 0:
828 raise VCSException("Hg pull failed", p.output)
829 self.refreshed = True
831 rev = rev or 'default'
834 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
835 if p.returncode != 0:
836 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
837 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
838 # Also delete untracked files, we have to enable purge extension for that:
839 if "'purge' is provided by the following extension" in p.output:
840 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
841 myfile.write("\n[extensions]\nhgext.purge=\n")
842 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
843 if p.returncode != 0:
844 raise VCSException("HG purge failed", p.output)
845 elif p.returncode != 0:
846 raise VCSException("HG purge failed", p.output)
849 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
850 return p.output.splitlines()[1:]
858 def gotorevisionx(self, rev):
859 if not os.path.exists(self.local):
860 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
861 if p.returncode != 0:
862 self.clone_failed = True
863 raise VCSException("Bzr branch failed", p.output)
865 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
866 if p.returncode != 0:
867 raise VCSException("Bzr revert failed", p.output)
868 if not self.refreshed:
869 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
870 if p.returncode != 0:
871 raise VCSException("Bzr update failed", p.output)
872 self.refreshed = True
874 revargs = list(['-r', rev] if rev else [])
875 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
876 if p.returncode != 0:
877 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
880 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
881 return [tag.split(' ')[0].strip() for tag in
882 p.output.splitlines()]
885 def unescape_string(string):
888 if string[0] == '"' and string[-1] == '"':
891 return string.replace("\\'", "'")
894 def retrieve_string(app_dir, string, xmlfiles=None):
896 if not string.startswith('@string/'):
897 return unescape_string(string)
902 os.path.join(app_dir, 'res'),
903 os.path.join(app_dir, 'src', 'main', 'res'),
905 for r, d, f in os.walk(res_dir):
906 if os.path.basename(r) == 'values':
907 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
909 name = string[len('@string/'):]
911 def element_content(element):
912 if element.text is None:
914 return element.text.encode('utf-8')
916 for path in xmlfiles:
917 if not os.path.isfile(path):
919 xml = parse_xml(path)
920 element = xml.find('string[@name="' + name + '"]')
921 if element is not None:
922 content = element_content(element)
923 return retrieve_string(app_dir, content, xmlfiles)
928 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
929 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
932 # Return list of existing files that will be used to find the highest vercode
933 def manifest_paths(app_dir, flavours):
935 possible_manifests = \
936 [os.path.join(app_dir, 'AndroidManifest.xml'),
937 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
938 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
939 os.path.join(app_dir, 'build.gradle')]
941 for flavour in flavours:
944 possible_manifests.append(
945 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
947 return [path for path in possible_manifests if os.path.isfile(path)]
950 # Retrieve the package name. Returns the name, or None if not found.
951 def fetch_real_name(app_dir, flavours):
952 for path in manifest_paths(app_dir, flavours):
953 if not has_extension(path, 'xml') or not os.path.isfile(path):
955 logging.debug("fetch_real_name: Checking manifest at " + path)
956 xml = parse_xml(path)
957 app = xml.find('application')
958 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
960 label = app.attrib["{http://schemas.android.com/apk/res/android}label"].encode('utf-8')
961 result = retrieve_string_singleline(app_dir, label)
963 result = result.strip()
968 def get_library_references(root_dir):
970 proppath = os.path.join(root_dir, 'project.properties')
971 if not os.path.isfile(proppath):
973 for line in file(proppath):
974 if not line.startswith('android.library.reference.'):
976 path = line.split('=')[1].strip()
977 relpath = os.path.join(root_dir, path)
978 if not os.path.isdir(relpath):
980 logging.debug("Found subproject at %s" % path)
981 libraries.append(path)
985 def ant_subprojects(root_dir):
986 subprojects = get_library_references(root_dir)
987 for subpath in subprojects:
988 subrelpath = os.path.join(root_dir, subpath)
989 for p in get_library_references(subrelpath):
990 relp = os.path.normpath(os.path.join(subpath, p))
991 if relp not in subprojects:
992 subprojects.insert(0, relp)
996 def remove_debuggable_flags(root_dir):
997 # Remove forced debuggable flags
998 logging.debug("Removing debuggable flags from %s" % root_dir)
999 for root, dirs, files in os.walk(root_dir):
1000 if 'AndroidManifest.xml' in files:
1001 regsub_file(r'android:debuggable="[^"]*"',
1003 os.path.join(root, 'AndroidManifest.xml'))
1006 # Extract some information from the AndroidManifest.xml at the given path.
1007 # Returns (version, vercode, package), any or all of which might be None.
1008 # All values returned are strings.
1009 def parse_androidmanifests(paths, ignoreversions=None):
1012 return (None, None, None)
1014 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
1015 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1016 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1018 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1026 if not os.path.isfile(path):
1029 logging.debug("Parsing manifest at {0}".format(path))
1030 gradle = has_extension(path, 'gradle')
1036 for line in file(path):
1037 if gradle_comment.match(line):
1039 # Grab first occurence of each to avoid running into
1040 # alternative flavours and builds.
1042 matches = psearch_g(line)
1044 package = matches.group(2)
1046 matches = vnsearch_g(line)
1048 version = matches.group(2)
1050 matches = vcsearch_g(line)
1052 vercode = matches.group(1)
1054 xml = parse_xml(path)
1055 if "package" in xml.attrib:
1056 package = xml.attrib["package"].encode('utf-8')
1057 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1058 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"].encode('utf-8')
1059 base_dir = os.path.dirname(path)
1060 version = retrieve_string_singleline(base_dir, version)
1061 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1062 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"].encode('utf-8')
1063 if string_is_integer(a):
1066 # Remember package name, may be defined separately from version+vercode
1068 package = max_package
1070 logging.debug("..got package={0}, version={1}, vercode={2}"
1071 .format(package, version, vercode))
1073 # Always grab the package name and version name in case they are not
1074 # together with the highest version code
1075 if max_package is None and package is not None:
1076 max_package = package
1077 if max_version is None and version is not None:
1078 max_version = version
1080 if max_vercode is None or (vercode is not None and vercode > max_vercode):
1081 if not ignoresearch or not ignoresearch(version):
1082 if version is not None:
1083 max_version = version
1084 if vercode is not None:
1085 max_vercode = vercode
1086 if package is not None:
1087 max_package = package
1089 max_version = "Ignore"
1091 if max_version is None:
1092 max_version = "Unknown"
1094 if max_package and not is_valid_package_name(max_package):
1095 raise FDroidException("Invalid package name {0}".format(max_package))
1097 return (max_version, max_vercode, max_package)
1100 def is_valid_package_name(name):
1101 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1104 class FDroidException(Exception):
1106 def __init__(self, value, detail=None):
1108 self.detail = detail
1110 def shortened_detail(self):
1111 if len(self.detail) < 16000:
1113 return '[...]\n' + self.detail[-16000:]
1115 def get_wikitext(self):
1116 ret = repr(self.value) + "\n"
1119 ret += "<pre>\n" + self.shortened_detail() + "</pre>\n"
1125 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1129 class VCSException(FDroidException):
1133 class BuildException(FDroidException):
1137 # Get the specified source library.
1138 # Returns the path to it. Normally this is the path to be used when referencing
1139 # it, which may be a subdirectory of the actual project. If you want the base
1140 # directory of the project, pass 'basepath=True'.
1141 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1142 raw=False, prepare=True, preponly=False, refresh=True):
1150 name, ref = spec.split('@')
1152 number, name = name.split(':', 1)
1154 name, subdir = name.split('/', 1)
1156 if name not in metadata.srclibs:
1157 raise VCSException('srclib ' + name + ' not found.')
1159 srclib = metadata.srclibs[name]
1161 sdir = os.path.join(srclib_dir, name)
1164 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1165 vcs.srclib = (name, number, sdir)
1167 vcs.gotorevision(ref, refresh)
1174 libdir = os.path.join(sdir, subdir)
1175 elif srclib["Subdir"]:
1176 for subdir in srclib["Subdir"]:
1177 libdir_candidate = os.path.join(sdir, subdir)
1178 if os.path.exists(libdir_candidate):
1179 libdir = libdir_candidate
1185 remove_signing_keys(sdir)
1186 remove_debuggable_flags(sdir)
1190 if srclib["Prepare"]:
1191 cmd = replace_config_vars(srclib["Prepare"], None)
1193 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1194 if p.returncode != 0:
1195 raise BuildException("Error running prepare command for srclib %s"
1201 return (name, number, libdir)
1204 # Prepare the source code for a particular build
1205 # 'vcs' - the appropriate vcs object for the application
1206 # 'app' - the application details from the metadata
1207 # 'build' - the build details from the metadata
1208 # 'build_dir' - the path to the build directory, usually
1210 # 'srclib_dir' - the path to the source libraries directory, usually
1212 # 'extlib_dir' - the path to the external libraries directory, usually
1214 # Returns the (root, srclibpaths) where:
1215 # 'root' is the root directory, which may be the same as 'build_dir' or may
1216 # be a subdirectory of it.
1217 # 'srclibpaths' is information on the srclibs being used
1218 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1220 # Optionally, the actual app source can be in a subdirectory
1222 root_dir = os.path.join(build_dir, build['subdir'])
1224 root_dir = build_dir
1226 # Get a working copy of the right revision
1227 logging.info("Getting source for revision " + build['commit'])
1228 vcs.gotorevision(build['commit'], refresh)
1230 # Initialise submodules if required
1231 if build['submodules']:
1232 logging.info("Initialising submodules")
1233 vcs.initsubmodules()
1235 # Check that a subdir (if we're using one) exists. This has to happen
1236 # after the checkout, since it might not exist elsewhere
1237 if not os.path.exists(root_dir):
1238 raise BuildException('Missing subdir ' + root_dir)
1240 # Run an init command if one is required
1242 cmd = replace_config_vars(build['init'], build)
1243 logging.info("Running 'init' commands in %s" % root_dir)
1245 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1246 if p.returncode != 0:
1247 raise BuildException("Error running init command for %s:%s" %
1248 (app['id'], build['version']), p.output)
1250 # Apply patches if any
1252 logging.info("Applying patches")
1253 for patch in build['patch']:
1254 patch = patch.strip()
1255 logging.info("Applying " + patch)
1256 patch_path = os.path.join('metadata', app['id'], patch)
1257 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1258 if p.returncode != 0:
1259 raise BuildException("Failed to apply patch %s" % patch_path)
1261 # Get required source libraries
1263 if build['srclibs']:
1264 logging.info("Collecting source libraries")
1265 for lib in build['srclibs']:
1266 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
1268 for name, number, libpath in srclibpaths:
1269 place_srclib(root_dir, int(number) if number else None, libpath)
1271 basesrclib = vcs.getsrclib()
1272 # If one was used for the main source, add that too.
1274 srclibpaths.append(basesrclib)
1276 # Update the local.properties file
1277 localprops = [os.path.join(build_dir, 'local.properties')]
1279 localprops += [os.path.join(root_dir, 'local.properties')]
1280 for path in localprops:
1282 if os.path.isfile(path):
1283 logging.info("Updating local.properties file at %s" % path)
1284 with open(path, 'r') as f:
1288 logging.info("Creating local.properties file at %s" % path)
1289 # Fix old-fashioned 'sdk-location' by copying
1290 # from sdk.dir, if necessary
1291 if build['oldsdkloc']:
1292 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1293 re.S | re.M).group(1)
1294 props += "sdk-location=%s\n" % sdkloc
1296 props += "sdk.dir=%s\n" % config['sdk_path']
1297 props += "sdk-location=%s\n" % config['sdk_path']
1298 if build['ndk_path']:
1300 props += "ndk.dir=%s\n" % build['ndk_path']
1301 props += "ndk-location=%s\n" % build['ndk_path']
1302 # Add java.encoding if necessary
1303 if build['encoding']:
1304 props += "java.encoding=%s\n" % build['encoding']
1305 with open(path, 'w') as f:
1309 if build['type'] == 'gradle':
1310 flavours = build['gradle']
1312 version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1313 gradlepluginver = None
1315 gradle_dirs = [root_dir]
1317 # Parent dir build.gradle
1318 parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
1319 if parent_dir.startswith(build_dir):
1320 gradle_dirs.append(parent_dir)
1322 for dir_path in gradle_dirs:
1325 if not os.path.isdir(dir_path):
1327 for filename in os.listdir(dir_path):
1328 if not filename.endswith('.gradle'):
1330 path = os.path.join(dir_path, filename)
1331 if not os.path.isfile(path):
1333 for line in file(path):
1334 match = version_regex.match(line)
1336 gradlepluginver = match.group(1)
1340 build['gradlepluginver'] = LooseVersion(gradlepluginver)
1342 logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
1343 build['gradlepluginver'] = LooseVersion('0.11')
1346 n = build["target"].split('-')[1]
1347 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1348 r'compileSdkVersion %s' % n,
1349 os.path.join(root_dir, 'build.gradle'))
1351 # Remove forced debuggable flags
1352 remove_debuggable_flags(root_dir)
1354 # Insert version code and number into the manifest if necessary
1355 if build['forceversion']:
1356 logging.info("Changing the version name")
1357 for path in manifest_paths(root_dir, flavours):
1358 if not os.path.isfile(path):
1360 if has_extension(path, 'xml'):
1361 regsub_file(r'android:versionName="[^"]*"',
1362 r'android:versionName="%s"' % build['version'],
1364 elif has_extension(path, 'gradle'):
1365 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1366 r"""\1versionName '%s'""" % build['version'],
1369 if build['forcevercode']:
1370 logging.info("Changing the version code")
1371 for path in manifest_paths(root_dir, flavours):
1372 if not os.path.isfile(path):
1374 if has_extension(path, 'xml'):
1375 regsub_file(r'android:versionCode="[^"]*"',
1376 r'android:versionCode="%s"' % build['vercode'],
1378 elif has_extension(path, 'gradle'):
1379 regsub_file(r'versionCode[ =]+[0-9]+',
1380 r'versionCode %s' % build['vercode'],
1383 # Delete unwanted files
1385 logging.info("Removing specified files")
1386 for part in getpaths(build_dir, build, 'rm'):
1387 dest = os.path.join(build_dir, part)
1388 logging.info("Removing {0}".format(part))
1389 if os.path.lexists(dest):
1390 if os.path.islink(dest):
1391 FDroidPopen(['unlink', dest], output=False)
1393 FDroidPopen(['rm', '-rf', dest], output=False)
1395 logging.info("...but it didn't exist")
1397 remove_signing_keys(build_dir)
1399 # Add required external libraries
1400 if build['extlibs']:
1401 logging.info("Collecting prebuilt libraries")
1402 libsdir = os.path.join(root_dir, 'libs')
1403 if not os.path.exists(libsdir):
1405 for lib in build['extlibs']:
1407 logging.info("...installing extlib {0}".format(lib))
1408 libf = os.path.basename(lib)
1409 libsrc = os.path.join(extlib_dir, lib)
1410 if not os.path.exists(libsrc):
1411 raise BuildException("Missing extlib file {0}".format(libsrc))
1412 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1414 # Run a pre-build command if one is required
1415 if build['prebuild']:
1416 logging.info("Running 'prebuild' commands in %s" % root_dir)
1418 cmd = replace_config_vars(build['prebuild'], build)
1420 # Substitute source library paths into prebuild commands
1421 for name, number, libpath in srclibpaths:
1422 libpath = os.path.relpath(libpath, root_dir)
1423 cmd = cmd.replace('$$' + name + '$$', libpath)
1425 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1426 if p.returncode != 0:
1427 raise BuildException("Error running prebuild command for %s:%s" %
1428 (app['id'], build['version']), p.output)
1430 # Generate (or update) the ant build file, build.xml...
1431 if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
1432 parms = ['android', 'update', 'lib-project']
1433 lparms = ['android', 'update', 'project']
1436 parms += ['-t', build['target']]
1437 lparms += ['-t', build['target']]
1438 if build['update'] == ['auto']:
1439 update_dirs = ant_subprojects(root_dir) + ['.']
1441 update_dirs = build['update']
1443 for d in update_dirs:
1444 subdir = os.path.join(root_dir, d)
1446 logging.debug("Updating main project")
1447 cmd = parms + ['-p', d]
1449 logging.debug("Updating subproject %s" % d)
1450 cmd = lparms + ['-p', d]
1451 p = SdkToolsPopen(cmd, cwd=root_dir)
1452 # Check to see whether an error was returned without a proper exit
1453 # code (this is the case for the 'no target set or target invalid'
1455 if p.returncode != 0 or p.output.startswith("Error: "):
1456 raise BuildException("Failed to update project at %s" % d, p.output)
1457 # Clean update dirs via ant
1459 logging.info("Cleaning subproject %s" % d)
1460 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1462 return (root_dir, srclibpaths)
1465 # Split and extend via globbing the paths from a field
1466 def getpaths(build_dir, build, field):
1468 for p in build[field]:
1470 full_path = os.path.join(build_dir, p)
1471 full_path = os.path.normpath(full_path)
1472 paths += [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1477 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1483 self.path = os.path.join('stats', 'known_apks.txt')
1485 if os.path.isfile(self.path):
1486 for line in file(self.path):
1487 t = line.rstrip().split(' ')
1489 self.apks[t[0]] = (t[1], None)
1491 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1492 self.changed = False
1494 def writeifchanged(self):
1495 if not self.changed:
1498 if not os.path.exists('stats'):
1502 for apk, app in self.apks.iteritems():
1504 line = apk + ' ' + appid
1506 line += ' ' + time.strftime('%Y-%m-%d', added)
1509 with open(self.path, 'w') as f:
1510 for line in sorted(lst, key=natural_key):
1511 f.write(line + '\n')
1513 # Record an apk (if it's new, otherwise does nothing)
1514 # Returns the date it was added.
1515 def recordapk(self, apk, app):
1516 if apk not in self.apks:
1517 self.apks[apk] = (app, time.gmtime(time.time()))
1519 _, added = self.apks[apk]
1522 # Look up information - given the 'apkname', returns (app id, date added/None).
1523 # Or returns None for an unknown apk.
1524 def getapp(self, apkname):
1525 if apkname in self.apks:
1526 return self.apks[apkname]
1529 # Get the most recent 'num' apps added to the repo, as a list of package ids
1530 # with the most recent first.
1531 def getlatest(self, num):
1533 for apk, app in self.apks.iteritems():
1537 if apps[appid] > added:
1541 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1542 lst = [app for app, _ in sortedapps]
1547 def isApkDebuggable(apkfile, config):
1548 """Returns True if the given apk file is debuggable
1550 :param apkfile: full path to the apk to check"""
1552 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1554 if p.returncode != 0:
1555 logging.critical("Failed to get apk manifest information")
1557 for line in p.output.splitlines():
1558 if 'android:debuggable' in line and not line.endswith('0x0'):
1568 def SdkToolsPopen(commands, cwd=None, output=True):
1570 if cmd not in config:
1571 config[cmd] = find_sdk_tools_cmd(commands[0])
1572 return FDroidPopen([config[cmd]] + commands[1:],
1573 cwd=cwd, output=output)
1576 def FDroidPopen(commands, cwd=None, output=True):
1578 Run a command and capture the possibly huge output.
1580 :param commands: command and argument list like in subprocess.Popen
1581 :param cwd: optionally specifies a working directory
1582 :returns: A PopenResult.
1588 cwd = os.path.normpath(cwd)
1589 logging.debug("Directory: %s" % cwd)
1590 logging.debug("> %s" % ' '.join(commands))
1592 result = PopenResult()
1595 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
1596 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1598 raise BuildException("OSError while trying to execute " +
1599 ' '.join(commands) + ': ' + str(e))
1601 stdout_queue = Queue.Queue()
1602 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1604 # Check the queue for output (until there is no more to get)
1605 while not stdout_reader.eof():
1606 while not stdout_queue.empty():
1607 line = stdout_queue.get()
1608 if output and options.verbose:
1609 # Output directly to console
1610 sys.stderr.write(line)
1612 result.output += line
1616 result.returncode = p.wait()
1620 gradle_comment = re.compile(r'[ ]*//')
1623 def remove_signing_keys(build_dir):
1624 signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1626 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1627 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1628 re.compile(r'.*variant\.outputFile = .*'),
1629 re.compile(r'.*output\.outputFile = .*'),
1630 re.compile(r'.*\.readLine\(.*'),
1632 for root, dirs, files in os.walk(build_dir):
1633 if 'build.gradle' in files:
1634 path = os.path.join(root, 'build.gradle')
1636 with open(path, "r") as o:
1637 lines = o.readlines()
1643 with open(path, "w") as o:
1644 while i < len(lines):
1647 while line.endswith('\\\n'):
1648 line = line.rstrip('\\\n') + lines[i]
1651 if gradle_comment.match(line):
1656 opened += line.count('{')
1657 opened -= line.count('}')
1660 if signing_configs.match(line):
1665 if any(s.match(line) for s in line_matches):
1673 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1676 'project.properties',
1678 'default.properties',
1679 'ant.properties', ]:
1680 if propfile in files:
1681 path = os.path.join(root, propfile)
1683 with open(path, "r") as o:
1684 lines = o.readlines()
1688 with open(path, "w") as o:
1690 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1697 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1700 def reset_env_path():
1701 global env, orig_path
1702 env['PATH'] = orig_path
1705 def add_to_env_path(path):
1707 paths = env['PATH'].split(os.pathsep)
1711 env['PATH'] = os.pathsep.join(paths)
1714 def replace_config_vars(cmd, build):
1716 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1717 # env['ANDROID_NDK'] is set in build_local right before prepare_source
1718 cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
1719 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1720 if build is not None:
1721 cmd = cmd.replace('$$COMMIT$$', build['commit'])
1722 cmd = cmd.replace('$$VERSION$$', build['version'])
1723 cmd = cmd.replace('$$VERCODE$$', build['vercode'])
1727 def place_srclib(root_dir, number, libpath):
1730 relpath = os.path.relpath(libpath, root_dir)
1731 proppath = os.path.join(root_dir, 'project.properties')
1734 if os.path.isfile(proppath):
1735 with open(proppath, "r") as o:
1736 lines = o.readlines()
1738 with open(proppath, "w") as o:
1741 if line.startswith('android.library.reference.%d=' % number):
1742 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1747 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1750 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
1751 """Verify that two apks are the same
1753 One of the inputs is signed, the other is unsigned. The signature metadata
1754 is transferred from the signed to the unsigned apk, and then jarsigner is
1755 used to verify that the signature from the signed apk is also varlid for
1757 :param signed_apk: Path to a signed apk file
1758 :param unsigned_apk: Path to an unsigned apk file expected to match it
1759 :param tmp_dir: Path to directory for temporary files
1760 :returns: None if the verification is successful, otherwise a string
1761 describing what went wrong.
1763 sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
1764 with ZipFile(signed_apk) as signed_apk_as_zip:
1765 meta_inf_files = ['META-INF/MANIFEST.MF']
1766 for f in signed_apk_as_zip.namelist():
1767 if sigfile.match(f):
1768 meta_inf_files.append(f)
1769 if len(meta_inf_files) < 3:
1770 return "Signature files missing from {0}".format(signed_apk)
1771 signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
1772 with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
1773 for meta_inf_file in meta_inf_files:
1774 unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
1776 if subprocess.call(['jarsigner', '-verify', unsigned_apk]) != 0:
1777 logging.info("...NOT verified - {0}".format(signed_apk))
1778 return compare_apks(signed_apk, unsigned_apk, tmp_dir)
1779 logging.info("...successfully verified")
1783 def compare_apks(apk1, apk2, tmp_dir):
1786 Returns None if the apk content is the same (apart from the signing key),
1787 otherwise a string describing what's different, or what went wrong when
1788 trying to do the comparison.
1791 badchars = re.compile('''[/ :;'"]''')
1792 apk1dir = os.path.join(tmp_dir, badchars.sub('_', apk1[0:-4])) # trim .apk
1793 apk2dir = os.path.join(tmp_dir, badchars.sub('_', apk2[0:-4])) # trim .apk
1794 for d in [apk1dir, apk2dir]:
1795 if os.path.exists(d):
1798 os.mkdir(os.path.join(d, 'jar-xf'))
1800 if subprocess.call(['jar', 'xf',
1801 os.path.abspath(apk1)],
1802 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
1803 return("Failed to unpack " + apk1)
1804 if subprocess.call(['jar', 'xf',
1805 os.path.abspath(apk2)],
1806 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
1807 return("Failed to unpack " + apk2)
1809 # try to find apktool in the path, if it hasn't been manually configed
1810 if 'apktool' not in config:
1811 tmp = find_command('apktool')
1813 config['apktool'] = tmp
1814 if 'apktool' in config:
1815 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
1817 return("Failed to unpack " + apk1)
1818 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
1820 return("Failed to unpack " + apk2)
1822 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
1823 lines = p.output.splitlines()
1824 if len(lines) != 1 or 'META-INF' not in lines[0]:
1825 meld = find_command('meld')
1826 if meld is not None:
1827 p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
1828 return("Unexpected diff output - " + p.output)
1830 # since everything verifies, delete the comparison to keep cruft down
1831 shutil.rmtree(apk1dir)
1832 shutil.rmtree(apk2dir)
1834 # If we get here, it seems like they're the same!
1838 def find_command(command):
1839 '''find the full path of a command, or None if it can't be found in the PATH'''
1842 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
1844 fpath, fname = os.path.split(command)
1849 for path in os.environ["PATH"].split(os.pathsep):
1850 path = path.strip('"')
1851 exe_file = os.path.join(path, command)
1852 if is_exe(exe_file):
1859 '''generate a random password for when generating keys'''
1860 h = hashlib.sha256()
1861 h.update(os.urandom(16)) # salt
1862 h.update(bytes(socket.getfqdn()))
1863 return h.digest().encode('base64').strip()
1866 def genkeystore(localconfig):
1867 '''Generate a new key with random passwords and add it to new keystore'''
1868 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
1869 keystoredir = os.path.dirname(localconfig['keystore'])
1870 if keystoredir is None or keystoredir == '':
1871 keystoredir = os.path.join(os.getcwd(), keystoredir)
1872 if not os.path.exists(keystoredir):
1873 os.makedirs(keystoredir, mode=0o700)
1875 write_password_file("keystorepass", localconfig['keystorepass'])
1876 write_password_file("keypass", localconfig['keypass'])
1877 p = FDroidPopen(['keytool', '-genkey',
1878 '-keystore', localconfig['keystore'],
1879 '-alias', localconfig['repo_keyalias'],
1880 '-keyalg', 'RSA', '-keysize', '4096',
1881 '-sigalg', 'SHA256withRSA',
1882 '-validity', '10000',
1883 '-storepass:file', config['keystorepassfile'],
1884 '-keypass:file', config['keypassfile'],
1885 '-dname', localconfig['keydname']])
1886 # TODO keypass should be sent via stdin
1887 if p.returncode != 0:
1888 raise BuildException("Failed to generate key", p.output)
1889 os.chmod(localconfig['keystore'], 0o0600)
1890 # now show the lovely key that was just generated
1891 p = FDroidPopen(['keytool', '-list', '-v',
1892 '-keystore', localconfig['keystore'],
1893 '-alias', localconfig['repo_keyalias'],
1894 '-storepass:file', config['keystorepassfile']])
1895 logging.info(p.output.strip() + '\n\n')
1898 def write_to_config(thisconfig, key, value=None):
1899 '''write a key/value to the local config.py'''
1901 origkey = key + '_orig'
1902 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
1903 with open('config.py', 'r') as f:
1905 pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
1906 repl = '\n' + key + ' = "' + value + '"'
1907 data = re.sub(pattern, repl, data)
1908 # if this key is not in the file, append it
1909 if not re.match('\s*' + key + '\s*=\s*"', data):
1911 # make sure the file ends with a carraige return
1912 if not re.match('\n$', data):
1914 with open('config.py', 'w') as f:
1918 def parse_xml(path):
1919 return XMLElementTree.parse(path).getroot()
1922 def string_is_integer(string):
1930 def get_per_app_repos():
1931 '''per-app repos are dirs named with the packageName of a single app'''
1933 # Android packageNames are Java packages, they may contain uppercase or
1934 # lowercase letters ('A' through 'Z'), numbers, and underscores
1935 # ('_'). However, individual package name parts may only start with
1936 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
1937 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
1940 for root, dirs, files in os.walk(os.getcwd()):
1942 print 'checking', root, 'for', d
1943 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
1944 # standard parts of an fdroid repo, so never packageNames
1947 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):