1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
36 import xml.etree.ElementTree as XMLElementTree
38 from distutils.version import LooseVersion
39 from zipfile import ZipFile
42 from fdroidserver.asynchronousfilereader import AsynchronousFileReader
45 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
54 'sdk_path': "$ANDROID_HOME",
57 'r10e': "$ANDROID_NDK",
59 'build_tools': "23.0.1",
63 'accepted_formats': ['txt', 'yaml'],
64 'sync_from_local_copy_dir': False,
65 'per_app_repos': False,
66 'make_current_version_link': True,
67 'current_version_name_source': 'Name',
68 'update_stats': False,
72 'stats_to_carbon': False,
74 'build_server_always': False,
75 'keystore': 'keystore.jks',
76 'smartcardoptions': [],
82 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
83 'repo_name': "My First FDroid Repo Demo",
84 'repo_icon': "fdroid-icon.png",
85 'repo_description': '''
86 This is a repository of apps to be used with FDroid. Applications in this
87 repository are either official binaries built by the original application
88 developers, or are binaries built from source by the admin of f-droid.org
89 using the tools on https://gitlab.com/u/fdroid.
95 def setup_global_opts(parser):
96 parser.add_argument("-v", "--verbose", action="store_true", default=False,
97 help="Spew out even more information than normal")
98 parser.add_argument("-q", "--quiet", action="store_true", default=False,
99 help="Restrict output to warnings and errors")
102 def fill_config_defaults(thisconfig):
103 for k, v in default_config.items():
104 if k not in thisconfig:
107 # Expand paths (~users and $vars)
108 def expand_path(path):
112 path = os.path.expanduser(path)
113 path = os.path.expandvars(path)
118 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
123 thisconfig[k + '_orig'] = v
125 for k in ['ndk_paths']:
131 thisconfig[k][k2] = exp
132 thisconfig[k][k2 + '_orig'] = v
135 def regsub_file(pattern, repl, path):
136 with open(path, 'r') as f:
138 text = re.sub(pattern, repl, text)
139 with open(path, 'w') as f:
143 def read_config(opts, config_file='config.py'):
144 """Read the repository config
146 The config is read from config_file, which is in the current directory when
147 any of the repo management commands are used.
149 global config, options, env, orig_path
151 if config is not None:
153 if not os.path.isfile(config_file):
154 logging.critical("Missing config file - is this a repo directory?")
161 logging.debug("Reading %s" % config_file)
162 execfile(config_file, config)
164 # smartcardoptions must be a list since its command line args for Popen
165 if 'smartcardoptions' in config:
166 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
167 elif 'keystore' in config and config['keystore'] == 'NONE':
168 # keystore='NONE' means use smartcard, these are required defaults
169 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
170 'SunPKCS11-OpenSC', '-providerClass',
171 'sun.security.pkcs11.SunPKCS11',
172 '-providerArg', 'opensc-fdroid.cfg']
174 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
175 st = os.stat(config_file)
176 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
177 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
179 fill_config_defaults(config)
181 # There is no standard, so just set up the most common environment
184 orig_path = env['PATH']
185 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
186 env[n] = config['sdk_path']
188 for k in ["keystorepass", "keypass"]:
190 write_password_file(k)
192 for k in ["repo_description", "archive_description"]:
194 config[k] = clean_description(config[k])
196 if 'serverwebroot' in config:
197 if isinstance(config['serverwebroot'], basestring):
198 roots = [config['serverwebroot']]
199 elif all(isinstance(item, basestring) for item in config['serverwebroot']):
200 roots = config['serverwebroot']
202 raise TypeError('only accepts strings, lists, and tuples')
204 for rootstr in roots:
205 # since this is used with rsync, where trailing slashes have
206 # meaning, ensure there is always a trailing slash
207 if rootstr[-1] != '/':
209 rootlist.append(rootstr.replace('//', '/'))
210 config['serverwebroot'] = rootlist
215 def get_ndk_path(version):
217 version = 'r10e' # falls back to latest
218 paths = config['ndk_paths']
219 if version not in paths:
221 return paths[version] or ''
224 def find_sdk_tools_cmd(cmd):
225 '''find a working path to a tool from the Android SDK'''
228 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
229 # try to find a working path to this command, in all the recent possible paths
230 if 'build_tools' in config:
231 build_tools = os.path.join(config['sdk_path'], 'build-tools')
232 # if 'build_tools' was manually set and exists, check only that one
233 configed_build_tools = os.path.join(build_tools, config['build_tools'])
234 if os.path.exists(configed_build_tools):
235 tooldirs.append(configed_build_tools)
237 # no configed version, so hunt known paths for it
238 for f in sorted(os.listdir(build_tools), reverse=True):
239 if os.path.isdir(os.path.join(build_tools, f)):
240 tooldirs.append(os.path.join(build_tools, f))
241 tooldirs.append(build_tools)
242 sdk_tools = os.path.join(config['sdk_path'], 'tools')
243 if os.path.exists(sdk_tools):
244 tooldirs.append(sdk_tools)
245 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
246 if os.path.exists(sdk_platform_tools):
247 tooldirs.append(sdk_platform_tools)
248 tooldirs.append('/usr/bin')
250 if os.path.isfile(os.path.join(d, cmd)):
251 return os.path.join(d, cmd)
252 # did not find the command, exit with error message
253 ensure_build_tools_exists(config)
256 def test_sdk_exists(thisconfig):
257 if 'sdk_path' not in thisconfig:
258 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
261 logging.error("'sdk_path' not set in config.py!")
263 if thisconfig['sdk_path'] == default_config['sdk_path']:
264 logging.error('No Android SDK found!')
265 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
266 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
268 if not os.path.exists(thisconfig['sdk_path']):
269 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
271 if not os.path.isdir(thisconfig['sdk_path']):
272 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
274 for d in ['build-tools', 'platform-tools', 'tools']:
275 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
276 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
277 thisconfig['sdk_path'], d))
282 def ensure_build_tools_exists(thisconfig):
283 if not test_sdk_exists(thisconfig):
285 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
286 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
287 if not os.path.isdir(versioned_build_tools):
288 logging.critical('Android Build Tools path "'
289 + versioned_build_tools + '" does not exist!')
293 def write_password_file(pwtype, password=None):
295 writes out passwords to a protected file instead of passing passwords as
296 command line argments
298 filename = '.fdroid.' + pwtype + '.txt'
299 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
301 os.write(fd, config[pwtype])
303 os.write(fd, password)
305 config[pwtype + 'file'] = filename
308 # Given the arguments in the form of multiple appid:[vc] strings, this returns
309 # a dictionary with the set of vercodes specified for each package.
310 def read_pkg_args(args, allow_vercodes=False):
317 if allow_vercodes and ':' in p:
318 package, vercode = p.split(':')
320 package, vercode = p, None
321 if package not in vercodes:
322 vercodes[package] = [vercode] if vercode else []
324 elif vercode and vercode not in vercodes[package]:
325 vercodes[package] += [vercode] if vercode else []
330 # On top of what read_pkg_args does, this returns the whole app metadata, but
331 # limiting the builds list to the builds matching the vercodes specified.
332 def read_app_args(args, allapps, allow_vercodes=False):
334 vercodes = read_pkg_args(args, allow_vercodes)
340 for appid, app in allapps.iteritems():
341 if appid in vercodes:
344 if len(apps) != len(vercodes):
347 logging.critical("No such package: %s" % p)
348 raise FDroidException("Found invalid app ids in arguments")
350 raise FDroidException("No packages specified")
353 for appid, app in apps.iteritems():
357 app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
358 if len(app['builds']) != len(vercodes[appid]):
360 allvcs = [b['vercode'] for b in app['builds']]
361 for v in vercodes[appid]:
363 logging.critical("No such vercode %s for app %s" % (v, appid))
366 raise FDroidException("Found invalid vercodes for some apps")
371 def get_extension(filename):
372 _, ext = os.path.splitext(filename)
375 return ext.lower()[1:]
378 def has_extension(filename, ext):
379 return ext == get_extension(filename)
385 def clean_description(description):
386 'Remove unneeded newlines and spaces from a block of description text'
388 # this is split up by paragraph to make removing the newlines easier
389 for paragraph in re.split(r'\n\n', description):
390 paragraph = re.sub('\r', '', paragraph)
391 paragraph = re.sub('\n', ' ', paragraph)
392 paragraph = re.sub(' {2,}', ' ', paragraph)
393 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
394 returnstring += paragraph + '\n\n'
395 return returnstring.rstrip('\n')
398 def apknameinfo(filename):
400 filename = os.path.basename(filename)
401 if apk_regex is None:
402 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
403 m = apk_regex.match(filename)
405 result = (m.group(1), m.group(2))
406 except AttributeError:
407 raise FDroidException("Invalid apk name: %s" % filename)
411 def getapkname(app, build):
412 return "%s_%s.apk" % (app['id'], build['vercode'])
415 def getsrcname(app, build):
416 return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
423 return app['Auto Name']
428 return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
431 def getvcs(vcstype, remote, local):
433 return vcs_git(remote, local)
434 if vcstype == 'git-svn':
435 return vcs_gitsvn(remote, local)
437 return vcs_hg(remote, local)
439 return vcs_bzr(remote, local)
440 if vcstype == 'srclib':
441 if local != os.path.join('build', 'srclib', remote):
442 raise VCSException("Error: srclib paths are hard-coded!")
443 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
445 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
446 raise VCSException("Invalid vcs type " + vcstype)
449 def getsrclibvcs(name):
450 if name not in metadata.srclibs:
451 raise VCSException("Missing srclib " + name)
452 return metadata.srclibs[name]['Repo Type']
457 def __init__(self, remote, local):
459 # svn, git-svn and bzr may require auth
461 if self.repotype() in ('git-svn', 'bzr'):
463 if self.repotype == 'git-svn':
464 raise VCSException("Authentication is not supported for git-svn")
465 self.username, remote = remote.split('@')
466 if ':' not in self.username:
467 raise VCSException("Password required with username")
468 self.username, self.password = self.username.split(':')
472 self.clone_failed = False
473 self.refreshed = False
479 # Take the local repository to a clean version of the given revision, which
480 # is specificed in the VCS's native format. Beforehand, the repository can
481 # be dirty, or even non-existent. If the repository does already exist
482 # locally, it will be updated from the origin, but only once in the
483 # lifetime of the vcs object.
484 # None is acceptable for 'rev' if you know you are cloning a clean copy of
485 # the repo - otherwise it must specify a valid revision.
486 def gotorevision(self, rev, refresh=True):
488 if self.clone_failed:
489 raise VCSException("Downloading the repository already failed once, not trying again.")
491 # The .fdroidvcs-id file for a repo tells us what VCS type
492 # and remote that directory was created from, allowing us to drop it
493 # automatically if either of those things changes.
494 fdpath = os.path.join(self.local, '..',
495 '.fdroidvcs-' + os.path.basename(self.local))
496 cdata = self.repotype() + ' ' + self.remote
499 if os.path.exists(self.local):
500 if os.path.exists(fdpath):
501 with open(fdpath, 'r') as f:
502 fsdata = f.read().strip()
507 logging.info("Repository details for %s changed - deleting" % (
511 logging.info("Repository details for %s missing - deleting" % (
514 shutil.rmtree(self.local)
518 self.refreshed = True
521 self.gotorevisionx(rev)
522 except FDroidException, e:
525 # If necessary, write the .fdroidvcs file.
526 if writeback and not self.clone_failed:
527 with open(fdpath, 'w') as f:
533 # Derived classes need to implement this. It's called once basic checking
534 # has been performend.
535 def gotorevisionx(self, rev):
536 raise VCSException("This VCS type doesn't define gotorevisionx")
538 # Initialise and update submodules
539 def initsubmodules(self):
540 raise VCSException('Submodules not supported for this vcs type')
542 # Get a list of all known tags
544 if not self._gettags:
545 raise VCSException('gettags not supported for this vcs type')
547 for tag in self._gettags():
548 if re.match('[-A-Za-z0-9_. /]+$', tag):
552 def latesttags(self, tags, number):
553 """Get the most recent tags in a given list.
555 :param tags: a list of tags
556 :param number: the number to return
557 :returns: A list containing the most recent tags in the provided
558 list, up to the maximum number given.
560 raise VCSException('latesttags not supported for this vcs type')
562 # Get current commit reference (hash, revision, etc)
564 raise VCSException('getref not supported for this vcs type')
566 # Returns the srclib (name, path) used in setting up the current
577 # If the local directory exists, but is somehow not a git repository, git
578 # will traverse up the directory tree until it finds one that is (i.e.
579 # fdroidserver) and then we'll proceed to destroy it! This is called as
582 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
583 result = p.output.rstrip()
584 if not result.endswith(self.local):
585 raise VCSException('Repository mismatch')
587 def gotorevisionx(self, rev):
588 if not os.path.exists(self.local):
590 p = FDroidPopen(['git', 'clone', self.remote, self.local])
591 if p.returncode != 0:
592 self.clone_failed = True
593 raise VCSException("Git clone failed", p.output)
597 # Discard any working tree changes
598 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
599 'git', 'reset', '--hard'], cwd=self.local, output=False)
600 if p.returncode != 0:
601 raise VCSException("Git reset failed", p.output)
602 # Remove untracked files now, in case they're tracked in the target
603 # revision (it happens!)
604 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
605 'git', 'clean', '-dffx'], cwd=self.local, output=False)
606 if p.returncode != 0:
607 raise VCSException("Git clean failed", p.output)
608 if not self.refreshed:
609 # Get latest commits and tags from remote
610 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
611 if p.returncode != 0:
612 raise VCSException("Git fetch failed", p.output)
613 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
614 if p.returncode != 0:
615 raise VCSException("Git fetch failed", p.output)
616 # Recreate origin/HEAD as git clone would do it, in case it disappeared
617 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
618 if p.returncode != 0:
619 lines = p.output.splitlines()
620 if 'Multiple remote HEAD branches' not in lines[0]:
621 raise VCSException("Git remote set-head failed", p.output)
622 branch = lines[1].split(' ')[-1]
623 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
624 if p2.returncode != 0:
625 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
626 self.refreshed = True
627 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
628 # a github repo. Most of the time this is the same as origin/master.
629 rev = rev or 'origin/HEAD'
630 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
631 if p.returncode != 0:
632 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
633 # Get rid of any uncontrolled files left behind
634 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
635 if p.returncode != 0:
636 raise VCSException("Git clean failed", p.output)
638 def initsubmodules(self):
640 submfile = os.path.join(self.local, '.gitmodules')
641 if not os.path.isfile(submfile):
642 raise VCSException("No git submodules available")
644 # fix submodules not accessible without an account and public key auth
645 with open(submfile, 'r') as f:
646 lines = f.readlines()
647 with open(submfile, 'w') as f:
649 if 'git@github.com' in line:
650 line = line.replace('git@github.com:', 'https://github.com/')
653 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
654 if p.returncode != 0:
655 raise VCSException("Git submodule sync failed", p.output)
656 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
657 if p.returncode != 0:
658 raise VCSException("Git submodule update failed", p.output)
662 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
663 return p.output.splitlines()
665 def latesttags(self, tags, number):
670 ['git', 'show', '--format=format:%ct', '-s', tag],
671 cwd=self.local, output=False)
672 # Timestamp is on the last line. For a normal tag, it's the only
673 # line, but for annotated tags, the rest of the info precedes it.
674 ts = int(p.output.splitlines()[-1])
677 for _, t in sorted(tl)[-number:]:
682 class vcs_gitsvn(vcs):
687 # If the local directory exists, but is somehow not a git repository, git
688 # will traverse up the directory tree until it finds one that is (i.e.
689 # fdroidserver) and then we'll proceed to destory it! This is called as
692 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
693 result = p.output.rstrip()
694 if not result.endswith(self.local):
695 raise VCSException('Repository mismatch')
697 def gotorevisionx(self, rev):
698 if not os.path.exists(self.local):
700 gitsvn_args = ['git', 'svn', 'clone']
701 if ';' in self.remote:
702 remote_split = self.remote.split(';')
703 for i in remote_split[1:]:
704 if i.startswith('trunk='):
705 gitsvn_args.extend(['-T', i[6:]])
706 elif i.startswith('tags='):
707 gitsvn_args.extend(['-t', i[5:]])
708 elif i.startswith('branches='):
709 gitsvn_args.extend(['-b', i[9:]])
710 gitsvn_args.extend([remote_split[0], self.local])
711 p = FDroidPopen(gitsvn_args, output=False)
712 if p.returncode != 0:
713 self.clone_failed = True
714 raise VCSException("Git svn clone failed", p.output)
716 gitsvn_args.extend([self.remote, self.local])
717 p = FDroidPopen(gitsvn_args, output=False)
718 if p.returncode != 0:
719 self.clone_failed = True
720 raise VCSException("Git svn clone failed", p.output)
724 # Discard any working tree changes
725 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
726 if p.returncode != 0:
727 raise VCSException("Git reset failed", p.output)
728 # Remove untracked files now, in case they're tracked in the target
729 # revision (it happens!)
730 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
731 if p.returncode != 0:
732 raise VCSException("Git clean failed", p.output)
733 if not self.refreshed:
734 # Get new commits, branches and tags from repo
735 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
736 if p.returncode != 0:
737 raise VCSException("Git svn fetch failed")
738 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
739 if p.returncode != 0:
740 raise VCSException("Git svn rebase failed", p.output)
741 self.refreshed = True
743 rev = rev or 'master'
745 nospaces_rev = rev.replace(' ', '%20')
746 # Try finding a svn tag
747 for treeish in ['origin/', '']:
748 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
749 if p.returncode == 0:
751 if p.returncode != 0:
752 # No tag found, normal svn rev translation
753 # Translate svn rev into git format
754 rev_split = rev.split('/')
757 for treeish in ['origin/', '']:
758 if len(rev_split) > 1:
759 treeish += rev_split[0]
760 svn_rev = rev_split[1]
763 # if no branch is specified, then assume trunk (i.e. 'master' branch):
767 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
769 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
770 git_rev = p.output.rstrip()
772 if p.returncode == 0 and git_rev:
775 if p.returncode != 0 or not git_rev:
776 # Try a plain git checkout as a last resort
777 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
778 if p.returncode != 0:
779 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
781 # Check out the git rev equivalent to the svn rev
782 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
783 if p.returncode != 0:
784 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
786 # Get rid of any uncontrolled files left behind
787 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
788 if p.returncode != 0:
789 raise VCSException("Git clean failed", p.output)
793 for treeish in ['origin/', '']:
794 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
800 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
801 if p.returncode != 0:
803 return p.output.strip()
811 def gotorevisionx(self, rev):
812 if not os.path.exists(self.local):
813 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
814 if p.returncode != 0:
815 self.clone_failed = True
816 raise VCSException("Hg clone failed", p.output)
818 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
819 if p.returncode != 0:
820 raise VCSException("Hg status failed", p.output)
821 for line in p.output.splitlines():
822 if not line.startswith('? '):
823 raise VCSException("Unexpected output from hg status -uS: " + line)
824 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
825 if not self.refreshed:
826 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
827 if p.returncode != 0:
828 raise VCSException("Hg pull failed", p.output)
829 self.refreshed = True
831 rev = rev or 'default'
834 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
835 if p.returncode != 0:
836 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
837 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
838 # Also delete untracked files, we have to enable purge extension for that:
839 if "'purge' is provided by the following extension" in p.output:
840 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
841 myfile.write("\n[extensions]\nhgext.purge=\n")
842 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
843 if p.returncode != 0:
844 raise VCSException("HG purge failed", p.output)
845 elif p.returncode != 0:
846 raise VCSException("HG purge failed", p.output)
849 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
850 return p.output.splitlines()[1:]
858 def gotorevisionx(self, rev):
859 if not os.path.exists(self.local):
860 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
861 if p.returncode != 0:
862 self.clone_failed = True
863 raise VCSException("Bzr branch failed", p.output)
865 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
866 if p.returncode != 0:
867 raise VCSException("Bzr revert failed", p.output)
868 if not self.refreshed:
869 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
870 if p.returncode != 0:
871 raise VCSException("Bzr update failed", p.output)
872 self.refreshed = True
874 revargs = list(['-r', rev] if rev else [])
875 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
876 if p.returncode != 0:
877 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
880 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
881 return [tag.split(' ')[0].strip() for tag in
882 p.output.splitlines()]
885 def unescape_string(string):
886 if string[0] == '"' and string[-1] == '"':
889 return string.replace("\\'", "'")
892 def retrieve_string(app_dir, string, xmlfiles=None):
897 os.path.join(app_dir, 'res'),
898 os.path.join(app_dir, 'src', 'main', 'res'),
900 for r, d, f in os.walk(res_dir):
901 if os.path.basename(r) == 'values':
902 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
904 if not string.startswith('@string/'):
905 return unescape_string(string)
907 name = string[len('@string/'):]
909 for path in xmlfiles:
910 if not os.path.isfile(path):
912 xml = parse_xml(path)
913 element = xml.find('string[@name="' + name + '"]')
914 if element is not None and element.text is not None:
915 return retrieve_string(app_dir, element.text.encode('utf-8'), xmlfiles)
920 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
921 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
924 # Return list of existing files that will be used to find the highest vercode
925 def manifest_paths(app_dir, flavours):
927 possible_manifests = \
928 [os.path.join(app_dir, 'AndroidManifest.xml'),
929 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
930 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
931 os.path.join(app_dir, 'build.gradle')]
933 for flavour in flavours:
936 possible_manifests.append(
937 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
939 return [path for path in possible_manifests if os.path.isfile(path)]
942 # Retrieve the package name. Returns the name, or None if not found.
943 def fetch_real_name(app_dir, flavours):
944 for path in manifest_paths(app_dir, flavours):
945 if not has_extension(path, 'xml') or not os.path.isfile(path):
947 logging.debug("fetch_real_name: Checking manifest at " + path)
948 xml = parse_xml(path)
949 app = xml.find('application')
950 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
952 label = app.attrib["{http://schemas.android.com/apk/res/android}label"].encode('utf-8')
953 result = retrieve_string_singleline(app_dir, label)
955 result = result.strip()
960 def get_library_references(root_dir):
962 proppath = os.path.join(root_dir, 'project.properties')
963 if not os.path.isfile(proppath):
965 for line in file(proppath):
966 if not line.startswith('android.library.reference.'):
968 path = line.split('=')[1].strip()
969 relpath = os.path.join(root_dir, path)
970 if not os.path.isdir(relpath):
972 logging.debug("Found subproject at %s" % path)
973 libraries.append(path)
977 def ant_subprojects(root_dir):
978 subprojects = get_library_references(root_dir)
979 for subpath in subprojects:
980 subrelpath = os.path.join(root_dir, subpath)
981 for p in get_library_references(subrelpath):
982 relp = os.path.normpath(os.path.join(subpath, p))
983 if relp not in subprojects:
984 subprojects.insert(0, relp)
988 def remove_debuggable_flags(root_dir):
989 # Remove forced debuggable flags
990 logging.debug("Removing debuggable flags from %s" % root_dir)
991 for root, dirs, files in os.walk(root_dir):
992 if 'AndroidManifest.xml' in files:
993 regsub_file(r'android:debuggable="[^"]*"',
995 os.path.join(root, 'AndroidManifest.xml'))
998 # Extract some information from the AndroidManifest.xml at the given path.
999 # Returns (version, vercode, package), any or all of which might be None.
1000 # All values returned are strings.
1001 def parse_androidmanifests(paths, ignoreversions=None):
1004 return (None, None, None)
1006 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
1007 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1008 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1010 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1018 if not os.path.isfile(path):
1021 logging.debug("Parsing manifest at {0}".format(path))
1022 gradle = has_extension(path, 'gradle')
1028 for line in file(path):
1029 if gradle_comment.match(line):
1031 # Grab first occurence of each to avoid running into
1032 # alternative flavours and builds.
1034 matches = psearch_g(line)
1036 package = matches.group(2)
1038 matches = vnsearch_g(line)
1040 version = matches.group(2)
1042 matches = vcsearch_g(line)
1044 vercode = matches.group(1)
1046 xml = parse_xml(path)
1047 if "package" in xml.attrib:
1048 package = xml.attrib["package"].encode('utf-8')
1049 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1050 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"].encode('utf-8')
1051 base_dir = os.path.dirname(path)
1052 version = retrieve_string_singleline(base_dir, version)
1053 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1054 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"].encode('utf-8')
1055 if string_is_integer(a):
1058 # Remember package name, may be defined separately from version+vercode
1060 package = max_package
1062 logging.debug("..got package={0}, version={1}, vercode={2}"
1063 .format(package, version, vercode))
1065 # Always grab the package name and version name in case they are not
1066 # together with the highest version code
1067 if max_package is None and package is not None:
1068 max_package = package
1069 if max_version is None and version is not None:
1070 max_version = version
1072 if max_vercode is None or (vercode is not None and vercode > max_vercode):
1073 if not ignoresearch or not ignoresearch(version):
1074 if version is not None:
1075 max_version = version
1076 if vercode is not None:
1077 max_vercode = vercode
1078 if package is not None:
1079 max_package = package
1081 max_version = "Ignore"
1083 if max_version is None:
1084 max_version = "Unknown"
1086 if max_package and not is_valid_package_name(max_package):
1087 raise FDroidException("Invalid package name {0}".format(max_package))
1089 return (max_version, max_vercode, max_package)
1092 def is_valid_package_name(name):
1093 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1096 class FDroidException(Exception):
1098 def __init__(self, value, detail=None):
1100 self.detail = detail
1102 def get_wikitext(self):
1103 ret = repr(self.value) + "\n"
1107 txt = self.detail[-8192:] if len(self.detail) > 8192 else self.detail
1115 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1119 class VCSException(FDroidException):
1123 class BuildException(FDroidException):
1127 # Get the specified source library.
1128 # Returns the path to it. Normally this is the path to be used when referencing
1129 # it, which may be a subdirectory of the actual project. If you want the base
1130 # directory of the project, pass 'basepath=True'.
1131 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1132 raw=False, prepare=True, preponly=False, refresh=True):
1140 name, ref = spec.split('@')
1142 number, name = name.split(':', 1)
1144 name, subdir = name.split('/', 1)
1146 if name not in metadata.srclibs:
1147 raise VCSException('srclib ' + name + ' not found.')
1149 srclib = metadata.srclibs[name]
1151 sdir = os.path.join(srclib_dir, name)
1154 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1155 vcs.srclib = (name, number, sdir)
1157 vcs.gotorevision(ref, refresh)
1164 libdir = os.path.join(sdir, subdir)
1165 elif srclib["Subdir"]:
1166 for subdir in srclib["Subdir"]:
1167 libdir_candidate = os.path.join(sdir, subdir)
1168 if os.path.exists(libdir_candidate):
1169 libdir = libdir_candidate
1175 remove_signing_keys(sdir)
1176 remove_debuggable_flags(sdir)
1180 if srclib["Prepare"]:
1181 cmd = replace_config_vars(srclib["Prepare"], None)
1183 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1184 if p.returncode != 0:
1185 raise BuildException("Error running prepare command for srclib %s"
1191 return (name, number, libdir)
1194 # Prepare the source code for a particular build
1195 # 'vcs' - the appropriate vcs object for the application
1196 # 'app' - the application details from the metadata
1197 # 'build' - the build details from the metadata
1198 # 'build_dir' - the path to the build directory, usually
1200 # 'srclib_dir' - the path to the source libraries directory, usually
1202 # 'extlib_dir' - the path to the external libraries directory, usually
1204 # Returns the (root, srclibpaths) where:
1205 # 'root' is the root directory, which may be the same as 'build_dir' or may
1206 # be a subdirectory of it.
1207 # 'srclibpaths' is information on the srclibs being used
1208 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1210 # Optionally, the actual app source can be in a subdirectory
1212 root_dir = os.path.join(build_dir, build['subdir'])
1214 root_dir = build_dir
1216 # Get a working copy of the right revision
1217 logging.info("Getting source for revision " + build['commit'])
1218 vcs.gotorevision(build['commit'], refresh)
1220 # Initialise submodules if required
1221 if build['submodules']:
1222 logging.info("Initialising submodules")
1223 vcs.initsubmodules()
1225 # Check that a subdir (if we're using one) exists. This has to happen
1226 # after the checkout, since it might not exist elsewhere
1227 if not os.path.exists(root_dir):
1228 raise BuildException('Missing subdir ' + root_dir)
1230 # Run an init command if one is required
1232 cmd = replace_config_vars(build['init'], build)
1233 logging.info("Running 'init' commands in %s" % root_dir)
1235 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1236 if p.returncode != 0:
1237 raise BuildException("Error running init command for %s:%s" %
1238 (app['id'], build['version']), p.output)
1240 # Apply patches if any
1242 logging.info("Applying patches")
1243 for patch in build['patch']:
1244 patch = patch.strip()
1245 logging.info("Applying " + patch)
1246 patch_path = os.path.join('metadata', app['id'], patch)
1247 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1248 if p.returncode != 0:
1249 raise BuildException("Failed to apply patch %s" % patch_path)
1251 # Get required source libraries
1253 if build['srclibs']:
1254 logging.info("Collecting source libraries")
1255 for lib in build['srclibs']:
1256 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
1258 for name, number, libpath in srclibpaths:
1259 place_srclib(root_dir, int(number) if number else None, libpath)
1261 basesrclib = vcs.getsrclib()
1262 # If one was used for the main source, add that too.
1264 srclibpaths.append(basesrclib)
1266 # Update the local.properties file
1267 localprops = [os.path.join(build_dir, 'local.properties')]
1269 localprops += [os.path.join(root_dir, 'local.properties')]
1270 for path in localprops:
1272 if os.path.isfile(path):
1273 logging.info("Updating local.properties file at %s" % path)
1274 with open(path, 'r') as f:
1278 logging.info("Creating local.properties file at %s" % path)
1279 # Fix old-fashioned 'sdk-location' by copying
1280 # from sdk.dir, if necessary
1281 if build['oldsdkloc']:
1282 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1283 re.S | re.M).group(1)
1284 props += "sdk-location=%s\n" % sdkloc
1286 props += "sdk.dir=%s\n" % config['sdk_path']
1287 props += "sdk-location=%s\n" % config['sdk_path']
1288 if build['ndk_path']:
1290 props += "ndk.dir=%s\n" % build['ndk_path']
1291 props += "ndk-location=%s\n" % build['ndk_path']
1292 # Add java.encoding if necessary
1293 if build['encoding']:
1294 props += "java.encoding=%s\n" % build['encoding']
1295 with open(path, 'w') as f:
1299 if build['type'] == 'gradle':
1300 flavours = build['gradle']
1302 version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1303 gradlepluginver = None
1305 gradle_dirs = [root_dir]
1307 # Parent dir build.gradle
1308 parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
1309 if parent_dir.startswith(build_dir):
1310 gradle_dirs.append(parent_dir)
1312 for dir_path in gradle_dirs:
1315 if not os.path.isdir(dir_path):
1317 for filename in os.listdir(dir_path):
1318 if not filename.endswith('.gradle'):
1320 path = os.path.join(dir_path, filename)
1321 if not os.path.isfile(path):
1323 for line in file(path):
1324 match = version_regex.match(line)
1326 gradlepluginver = match.group(1)
1330 build['gradlepluginver'] = LooseVersion(gradlepluginver)
1332 logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
1333 build['gradlepluginver'] = LooseVersion('0.11')
1336 n = build["target"].split('-')[1]
1337 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1338 r'compileSdkVersion %s' % n,
1339 os.path.join(root_dir, 'build.gradle'))
1341 # Remove forced debuggable flags
1342 remove_debuggable_flags(root_dir)
1344 # Insert version code and number into the manifest if necessary
1345 if build['forceversion']:
1346 logging.info("Changing the version name")
1347 for path in manifest_paths(root_dir, flavours):
1348 if not os.path.isfile(path):
1350 if has_extension(path, 'xml'):
1351 regsub_file(r'android:versionName="[^"]*"',
1352 r'android:versionName="%s"' % build['version'],
1354 elif has_extension(path, 'gradle'):
1355 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1356 r"""\1versionName '%s'""" % build['version'],
1359 if build['forcevercode']:
1360 logging.info("Changing the version code")
1361 for path in manifest_paths(root_dir, flavours):
1362 if not os.path.isfile(path):
1364 if has_extension(path, 'xml'):
1365 regsub_file(r'android:versionCode="[^"]*"',
1366 r'android:versionCode="%s"' % build['vercode'],
1368 elif has_extension(path, 'gradle'):
1369 regsub_file(r'versionCode[ =]+[0-9]+',
1370 r'versionCode %s' % build['vercode'],
1373 # Delete unwanted files
1375 logging.info("Removing specified files")
1376 for part in getpaths(build_dir, build, 'rm'):
1377 dest = os.path.join(build_dir, part)
1378 logging.info("Removing {0}".format(part))
1379 if os.path.lexists(dest):
1380 if os.path.islink(dest):
1381 FDroidPopen(['unlink', dest], output=False)
1383 FDroidPopen(['rm', '-rf', dest], output=False)
1385 logging.info("...but it didn't exist")
1387 remove_signing_keys(build_dir)
1389 # Add required external libraries
1390 if build['extlibs']:
1391 logging.info("Collecting prebuilt libraries")
1392 libsdir = os.path.join(root_dir, 'libs')
1393 if not os.path.exists(libsdir):
1395 for lib in build['extlibs']:
1397 logging.info("...installing extlib {0}".format(lib))
1398 libf = os.path.basename(lib)
1399 libsrc = os.path.join(extlib_dir, lib)
1400 if not os.path.exists(libsrc):
1401 raise BuildException("Missing extlib file {0}".format(libsrc))
1402 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1404 # Run a pre-build command if one is required
1405 if build['prebuild']:
1406 logging.info("Running 'prebuild' commands in %s" % root_dir)
1408 cmd = replace_config_vars(build['prebuild'], build)
1410 # Substitute source library paths into prebuild commands
1411 for name, number, libpath in srclibpaths:
1412 libpath = os.path.relpath(libpath, root_dir)
1413 cmd = cmd.replace('$$' + name + '$$', libpath)
1415 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1416 if p.returncode != 0:
1417 raise BuildException("Error running prebuild command for %s:%s" %
1418 (app['id'], build['version']), p.output)
1420 # Generate (or update) the ant build file, build.xml...
1421 if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
1422 parms = ['android', 'update', 'lib-project']
1423 lparms = ['android', 'update', 'project']
1426 parms += ['-t', build['target']]
1427 lparms += ['-t', build['target']]
1428 if build['update'] == ['auto']:
1429 update_dirs = ant_subprojects(root_dir) + ['.']
1431 update_dirs = build['update']
1433 for d in update_dirs:
1434 subdir = os.path.join(root_dir, d)
1436 logging.debug("Updating main project")
1437 cmd = parms + ['-p', d]
1439 logging.debug("Updating subproject %s" % d)
1440 cmd = lparms + ['-p', d]
1441 p = SdkToolsPopen(cmd, cwd=root_dir)
1442 # Check to see whether an error was returned without a proper exit
1443 # code (this is the case for the 'no target set or target invalid'
1445 if p.returncode != 0 or p.output.startswith("Error: "):
1446 raise BuildException("Failed to update project at %s" % d, p.output)
1447 # Clean update dirs via ant
1449 logging.info("Cleaning subproject %s" % d)
1450 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1452 return (root_dir, srclibpaths)
1455 # Split and extend via globbing the paths from a field
1456 def getpaths(build_dir, build, field):
1458 for p in build[field]:
1460 full_path = os.path.join(build_dir, p)
1461 full_path = os.path.normpath(full_path)
1462 paths += [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1467 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1473 self.path = os.path.join('stats', 'known_apks.txt')
1475 if os.path.isfile(self.path):
1476 for line in file(self.path):
1477 t = line.rstrip().split(' ')
1479 self.apks[t[0]] = (t[1], None)
1481 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1482 self.changed = False
1484 def writeifchanged(self):
1485 if not self.changed:
1488 if not os.path.exists('stats'):
1492 for apk, app in self.apks.iteritems():
1494 line = apk + ' ' + appid
1496 line += ' ' + time.strftime('%Y-%m-%d', added)
1499 with open(self.path, 'w') as f:
1500 for line in sorted(lst, key=natural_key):
1501 f.write(line + '\n')
1503 # Record an apk (if it's new, otherwise does nothing)
1504 # Returns the date it was added.
1505 def recordapk(self, apk, app):
1506 if apk not in self.apks:
1507 self.apks[apk] = (app, time.gmtime(time.time()))
1509 _, added = self.apks[apk]
1512 # Look up information - given the 'apkname', returns (app id, date added/None).
1513 # Or returns None for an unknown apk.
1514 def getapp(self, apkname):
1515 if apkname in self.apks:
1516 return self.apks[apkname]
1519 # Get the most recent 'num' apps added to the repo, as a list of package ids
1520 # with the most recent first.
1521 def getlatest(self, num):
1523 for apk, app in self.apks.iteritems():
1527 if apps[appid] > added:
1531 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1532 lst = [app for app, _ in sortedapps]
1537 def isApkDebuggable(apkfile, config):
1538 """Returns True if the given apk file is debuggable
1540 :param apkfile: full path to the apk to check"""
1542 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1544 if p.returncode != 0:
1545 logging.critical("Failed to get apk manifest information")
1547 for line in p.output.splitlines():
1548 if 'android:debuggable' in line and not line.endswith('0x0'):
1558 def SdkToolsPopen(commands, cwd=None, output=True):
1560 if cmd not in config:
1561 config[cmd] = find_sdk_tools_cmd(commands[0])
1562 return FDroidPopen([config[cmd]] + commands[1:],
1563 cwd=cwd, output=output)
1566 def FDroidPopen(commands, cwd=None, output=True):
1568 Run a command and capture the possibly huge output.
1570 :param commands: command and argument list like in subprocess.Popen
1571 :param cwd: optionally specifies a working directory
1572 :returns: A PopenResult.
1578 cwd = os.path.normpath(cwd)
1579 logging.debug("Directory: %s" % cwd)
1580 logging.debug("> %s" % ' '.join(commands))
1582 result = PopenResult()
1585 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
1586 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1588 raise BuildException("OSError while trying to execute " +
1589 ' '.join(commands) + ': ' + str(e))
1591 stdout_queue = Queue.Queue()
1592 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1594 # Check the queue for output (until there is no more to get)
1595 while not stdout_reader.eof():
1596 while not stdout_queue.empty():
1597 line = stdout_queue.get()
1598 if output and options.verbose:
1599 # Output directly to console
1600 sys.stderr.write(line)
1602 result.output += line
1606 result.returncode = p.wait()
1610 gradle_comment = re.compile(r'[ ]*//')
1613 def remove_signing_keys(build_dir):
1614 signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1616 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1617 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1618 re.compile(r'.*variant\.outputFile = .*'),
1619 re.compile(r'.*output\.outputFile = .*'),
1620 re.compile(r'.*\.readLine\(.*'),
1622 for root, dirs, files in os.walk(build_dir):
1623 if 'build.gradle' in files:
1624 path = os.path.join(root, 'build.gradle')
1626 with open(path, "r") as o:
1627 lines = o.readlines()
1633 with open(path, "w") as o:
1634 while i < len(lines):
1637 while line.endswith('\\\n'):
1638 line = line.rstrip('\\\n') + lines[i]
1641 if gradle_comment.match(line):
1646 opened += line.count('{')
1647 opened -= line.count('}')
1650 if signing_configs.match(line):
1655 if any(s.match(line) for s in line_matches):
1663 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1666 'project.properties',
1668 'default.properties',
1669 'ant.properties', ]:
1670 if propfile in files:
1671 path = os.path.join(root, propfile)
1673 with open(path, "r") as o:
1674 lines = o.readlines()
1678 with open(path, "w") as o:
1680 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1687 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1690 def reset_env_path():
1691 global env, orig_path
1692 env['PATH'] = orig_path
1695 def add_to_env_path(path):
1697 paths = env['PATH'].split(os.pathsep)
1701 env['PATH'] = os.pathsep.join(paths)
1704 def replace_config_vars(cmd, build):
1706 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1707 # env['ANDROID_NDK'] is set in build_local right before prepare_source
1708 cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
1709 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1710 if build is not None:
1711 cmd = cmd.replace('$$COMMIT$$', build['commit'])
1712 cmd = cmd.replace('$$VERSION$$', build['version'])
1713 cmd = cmd.replace('$$VERCODE$$', build['vercode'])
1717 def place_srclib(root_dir, number, libpath):
1720 relpath = os.path.relpath(libpath, root_dir)
1721 proppath = os.path.join(root_dir, 'project.properties')
1724 if os.path.isfile(proppath):
1725 with open(proppath, "r") as o:
1726 lines = o.readlines()
1728 with open(proppath, "w") as o:
1731 if line.startswith('android.library.reference.%d=' % number):
1732 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1737 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1740 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
1741 """Verify that two apks are the same
1743 One of the inputs is signed, the other is unsigned. The signature metadata
1744 is transferred from the signed to the unsigned apk, and then jarsigner is
1745 used to verify that the signature from the signed apk is also varlid for
1747 :param signed_apk: Path to a signed apk file
1748 :param unsigned_apk: Path to an unsigned apk file expected to match it
1749 :param tmp_dir: Path to directory for temporary files
1750 :returns: None if the verification is successful, otherwise a string
1751 describing what went wrong.
1753 sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
1754 with ZipFile(signed_apk) as signed_apk_as_zip:
1755 meta_inf_files = ['META-INF/MANIFEST.MF']
1756 for f in signed_apk_as_zip.namelist():
1757 if sigfile.match(f):
1758 meta_inf_files.append(f)
1759 if len(meta_inf_files) < 3:
1760 return "Signature files missing from {0}".format(signed_apk)
1761 signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
1762 with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
1763 for meta_inf_file in meta_inf_files:
1764 unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
1766 if subprocess.call(['jarsigner', '-verify', unsigned_apk]) != 0:
1767 logging.info("...NOT verified - {0}".format(signed_apk))
1768 return compare_apks(signed_apk, unsigned_apk, tmp_dir)
1769 logging.info("...successfully verified")
1773 def compare_apks(apk1, apk2, tmp_dir):
1776 Returns None if the apk content is the same (apart from the signing key),
1777 otherwise a string describing what's different, or what went wrong when
1778 trying to do the comparison.
1781 badchars = re.compile('''[/ :;'"]''')
1782 apk1dir = os.path.join(tmp_dir, badchars.sub('_', apk1[0:-4])) # trim .apk
1783 apk2dir = os.path.join(tmp_dir, badchars.sub('_', apk2[0:-4])) # trim .apk
1784 for d in [apk1dir, apk2dir]:
1785 if os.path.exists(d):
1788 os.mkdir(os.path.join(d, 'jar-xf'))
1790 if subprocess.call(['jar', 'xf',
1791 os.path.abspath(apk1)],
1792 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
1793 return("Failed to unpack " + apk1)
1794 if subprocess.call(['jar', 'xf',
1795 os.path.abspath(apk2)],
1796 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
1797 return("Failed to unpack " + apk2)
1799 # try to find apktool in the path, if it hasn't been manually configed
1800 if 'apktool' not in config:
1801 tmp = find_command('apktool')
1803 config['apktool'] = tmp
1804 if 'apktool' in config:
1805 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
1807 return("Failed to unpack " + apk1)
1808 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
1810 return("Failed to unpack " + apk2)
1812 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
1813 lines = p.output.splitlines()
1814 if len(lines) != 1 or 'META-INF' not in lines[0]:
1815 meld = find_command('meld')
1816 if meld is not None:
1817 p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
1818 return("Unexpected diff output - " + p.output)
1820 # since everything verifies, delete the comparison to keep cruft down
1821 shutil.rmtree(apk1dir)
1822 shutil.rmtree(apk2dir)
1824 # If we get here, it seems like they're the same!
1828 def find_command(command):
1829 '''find the full path of a command, or None if it can't be found in the PATH'''
1832 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
1834 fpath, fname = os.path.split(command)
1839 for path in os.environ["PATH"].split(os.pathsep):
1840 path = path.strip('"')
1841 exe_file = os.path.join(path, command)
1842 if is_exe(exe_file):
1849 '''generate a random password for when generating keys'''
1850 h = hashlib.sha256()
1851 h.update(os.urandom(16)) # salt
1852 h.update(bytes(socket.getfqdn()))
1853 return h.digest().encode('base64').strip()
1856 def genkeystore(localconfig):
1857 '''Generate a new key with random passwords and add it to new keystore'''
1858 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
1859 keystoredir = os.path.dirname(localconfig['keystore'])
1860 if keystoredir is None or keystoredir == '':
1861 keystoredir = os.path.join(os.getcwd(), keystoredir)
1862 if not os.path.exists(keystoredir):
1863 os.makedirs(keystoredir, mode=0o700)
1865 write_password_file("keystorepass", localconfig['keystorepass'])
1866 write_password_file("keypass", localconfig['keypass'])
1867 p = FDroidPopen(['keytool', '-genkey',
1868 '-keystore', localconfig['keystore'],
1869 '-alias', localconfig['repo_keyalias'],
1870 '-keyalg', 'RSA', '-keysize', '4096',
1871 '-sigalg', 'SHA256withRSA',
1872 '-validity', '10000',
1873 '-storepass:file', config['keystorepassfile'],
1874 '-keypass:file', config['keypassfile'],
1875 '-dname', localconfig['keydname']])
1876 # TODO keypass should be sent via stdin
1877 if p.returncode != 0:
1878 raise BuildException("Failed to generate key", p.output)
1879 os.chmod(localconfig['keystore'], 0o0600)
1880 # now show the lovely key that was just generated
1881 p = FDroidPopen(['keytool', '-list', '-v',
1882 '-keystore', localconfig['keystore'],
1883 '-alias', localconfig['repo_keyalias'],
1884 '-storepass:file', config['keystorepassfile']])
1885 logging.info(p.output.strip() + '\n\n')
1888 def write_to_config(thisconfig, key, value=None):
1889 '''write a key/value to the local config.py'''
1891 origkey = key + '_orig'
1892 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
1893 with open('config.py', 'r') as f:
1895 pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
1896 repl = '\n' + key + ' = "' + value + '"'
1897 data = re.sub(pattern, repl, data)
1898 # if this key is not in the file, append it
1899 if not re.match('\s*' + key + '\s*=\s*"', data):
1901 # make sure the file ends with a carraige return
1902 if not re.match('\n$', data):
1904 with open('config.py', 'w') as f:
1908 def parse_xml(path):
1909 return XMLElementTree.parse(path).getroot()
1912 def string_is_integer(string):
1920 def get_per_app_repos():
1921 '''per-app repos are dirs named with the packageName of a single app'''
1923 # Android packageNames are Java packages, they may contain uppercase or
1924 # lowercase letters ('A' through 'Z'), numbers, and underscores
1925 # ('_'). However, individual package name parts may only start with
1926 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
1927 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
1930 for root, dirs, files in os.walk(os.getcwd()):
1932 print 'checking', root, 'for', d
1933 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
1934 # standard parts of an fdroid repo, so never packageNames
1937 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):