1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
36 import xml.etree.ElementTree as XMLElementTree
38 from distutils.version import LooseVersion
39 from zipfile import ZipFile
43 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
52 'sdk_path': "$ANDROID_HOME",
55 'r10e': "$ANDROID_NDK"
57 'build_tools': "22.0.1",
61 'sync_from_local_copy_dir': False,
62 'make_current_version_link': True,
63 'current_version_name_source': 'Name',
64 'update_stats': False,
68 'stats_to_carbon': False,
70 'build_server_always': False,
71 'keystore': 'keystore.jks',
72 'smartcardoptions': [],
78 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
79 'repo_name': "My First FDroid Repo Demo",
80 'repo_icon': "fdroid-icon.png",
81 'repo_description': '''
82 This is a repository of apps to be used with FDroid. Applications in this
83 repository are either official binaries built by the original application
84 developers, or are binaries built from source by the admin of f-droid.org
85 using the tools on https://gitlab.com/u/fdroid.
91 def fill_config_defaults(thisconfig):
92 for k, v in default_config.items():
93 if k not in thisconfig:
96 # Expand paths (~users and $vars)
97 def expand_path(path):
101 path = os.path.expanduser(path)
102 path = os.path.expandvars(path)
107 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
112 thisconfig[k + '_orig'] = v
114 for k in ['ndk_paths']:
120 thisconfig[k][k2] = exp
121 thisconfig[k][k2 + '_orig'] = v
124 def read_config(opts, config_file='config.py'):
125 """Read the repository config
127 The config is read from config_file, which is in the current directory when
128 any of the repo management commands are used.
130 global config, options, env, orig_path
132 if config is not None:
134 if not os.path.isfile(config_file):
135 logging.critical("Missing config file - is this a repo directory?")
142 logging.debug("Reading %s" % config_file)
143 execfile(config_file, config)
145 # smartcardoptions must be a list since its command line args for Popen
146 if 'smartcardoptions' in config:
147 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
148 elif 'keystore' in config and config['keystore'] == 'NONE':
149 # keystore='NONE' means use smartcard, these are required defaults
150 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
151 'SunPKCS11-OpenSC', '-providerClass',
152 'sun.security.pkcs11.SunPKCS11',
153 '-providerArg', 'opensc-fdroid.cfg']
155 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
156 st = os.stat(config_file)
157 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
158 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
160 fill_config_defaults(config)
162 # There is no standard, so just set up the most common environment
165 orig_path = env['PATH']
166 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
167 env[n] = config['sdk_path']
169 for k in ["keystorepass", "keypass"]:
171 write_password_file(k)
173 for k in ["repo_description", "archive_description"]:
175 config[k] = clean_description(config[k])
177 if 'serverwebroot' in config:
178 if isinstance(config['serverwebroot'], basestring):
179 roots = [config['serverwebroot']]
180 elif all(isinstance(item, basestring) for item in config['serverwebroot']):
181 roots = config['serverwebroot']
183 raise TypeError('only accepts strings, lists, and tuples')
185 for rootstr in roots:
186 # since this is used with rsync, where trailing slashes have
187 # meaning, ensure there is always a trailing slash
188 if rootstr[-1] != '/':
190 rootlist.append(rootstr.replace('//', '/'))
191 config['serverwebroot'] = rootlist
196 def get_ndk_path(version):
198 version = 'r10e' # latest
199 paths = config['ndk_paths']
200 if version not in paths:
202 return paths[version] or ''
205 def find_sdk_tools_cmd(cmd):
206 '''find a working path to a tool from the Android SDK'''
209 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
210 # try to find a working path to this command, in all the recent possible paths
211 if 'build_tools' in config:
212 build_tools = os.path.join(config['sdk_path'], 'build-tools')
213 # if 'build_tools' was manually set and exists, check only that one
214 configed_build_tools = os.path.join(build_tools, config['build_tools'])
215 if os.path.exists(configed_build_tools):
216 tooldirs.append(configed_build_tools)
218 # no configed version, so hunt known paths for it
219 for f in sorted(os.listdir(build_tools), reverse=True):
220 if os.path.isdir(os.path.join(build_tools, f)):
221 tooldirs.append(os.path.join(build_tools, f))
222 tooldirs.append(build_tools)
223 sdk_tools = os.path.join(config['sdk_path'], 'tools')
224 if os.path.exists(sdk_tools):
225 tooldirs.append(sdk_tools)
226 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
227 if os.path.exists(sdk_platform_tools):
228 tooldirs.append(sdk_platform_tools)
229 tooldirs.append('/usr/bin')
231 if os.path.isfile(os.path.join(d, cmd)):
232 return os.path.join(d, cmd)
233 # did not find the command, exit with error message
234 ensure_build_tools_exists(config)
237 def test_sdk_exists(thisconfig):
238 if 'sdk_path' not in thisconfig:
239 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
242 logging.error("'sdk_path' not set in config.py!")
244 if thisconfig['sdk_path'] == default_config['sdk_path']:
245 logging.error('No Android SDK found!')
246 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
247 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
249 if not os.path.exists(thisconfig['sdk_path']):
250 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
252 if not os.path.isdir(thisconfig['sdk_path']):
253 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
255 for d in ['build-tools', 'platform-tools', 'tools']:
256 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
257 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
258 thisconfig['sdk_path'], d))
263 def ensure_build_tools_exists(thisconfig):
264 if not test_sdk_exists(thisconfig):
266 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
267 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
268 if not os.path.isdir(versioned_build_tools):
269 logging.critical('Android Build Tools path "'
270 + versioned_build_tools + '" does not exist!')
274 def write_password_file(pwtype, password=None):
276 writes out passwords to a protected file instead of passing passwords as
277 command line argments
279 filename = '.fdroid.' + pwtype + '.txt'
280 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
282 os.write(fd, config[pwtype])
284 os.write(fd, password)
286 config[pwtype + 'file'] = filename
289 # Given the arguments in the form of multiple appid:[vc] strings, this returns
290 # a dictionary with the set of vercodes specified for each package.
291 def read_pkg_args(args, allow_vercodes=False):
298 if allow_vercodes and ':' in p:
299 package, vercode = p.split(':')
301 package, vercode = p, None
302 if package not in vercodes:
303 vercodes[package] = [vercode] if vercode else []
305 elif vercode and vercode not in vercodes[package]:
306 vercodes[package] += [vercode] if vercode else []
311 # On top of what read_pkg_args does, this returns the whole app metadata, but
312 # limiting the builds list to the builds matching the vercodes specified.
313 def read_app_args(args, allapps, allow_vercodes=False):
315 vercodes = read_pkg_args(args, allow_vercodes)
321 for appid, app in allapps.iteritems():
322 if appid in vercodes:
325 if len(apps) != len(vercodes):
328 logging.critical("No such package: %s" % p)
329 raise FDroidException("Found invalid app ids in arguments")
331 raise FDroidException("No packages specified")
334 for appid, app in apps.iteritems():
338 app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
339 if len(app['builds']) != len(vercodes[appid]):
341 allvcs = [b['vercode'] for b in app['builds']]
342 for v in vercodes[appid]:
344 logging.critical("No such vercode %s for app %s" % (v, appid))
347 raise FDroidException("Found invalid vercodes for some apps")
352 def has_extension(filename, extension):
353 name, ext = os.path.splitext(filename)
354 ext = ext.lower()[1:]
355 return ext == extension
360 def clean_description(description):
361 'Remove unneeded newlines and spaces from a block of description text'
363 # this is split up by paragraph to make removing the newlines easier
364 for paragraph in re.split(r'\n\n', description):
365 paragraph = re.sub('\r', '', paragraph)
366 paragraph = re.sub('\n', ' ', paragraph)
367 paragraph = re.sub(' {2,}', ' ', paragraph)
368 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
369 returnstring += paragraph + '\n\n'
370 return returnstring.rstrip('\n')
373 def apknameinfo(filename):
375 filename = os.path.basename(filename)
376 if apk_regex is None:
377 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
378 m = apk_regex.match(filename)
380 result = (m.group(1), m.group(2))
381 except AttributeError:
382 raise FDroidException("Invalid apk name: %s" % filename)
386 def getapkname(app, build):
387 return "%s_%s.apk" % (app['id'], build['vercode'])
390 def getsrcname(app, build):
391 return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
398 return app['Auto Name']
403 return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
406 def getvcs(vcstype, remote, local):
408 return vcs_git(remote, local)
409 if vcstype == 'git-svn':
410 return vcs_gitsvn(remote, local)
412 return vcs_hg(remote, local)
414 return vcs_bzr(remote, local)
415 if vcstype == 'srclib':
416 if local != os.path.join('build', 'srclib', remote):
417 raise VCSException("Error: srclib paths are hard-coded!")
418 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
420 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
421 raise VCSException("Invalid vcs type " + vcstype)
424 def getsrclibvcs(name):
425 if name not in metadata.srclibs:
426 raise VCSException("Missing srclib " + name)
427 return metadata.srclibs[name]['Repo Type']
432 def __init__(self, remote, local):
434 # svn, git-svn and bzr may require auth
436 if self.repotype() in ('git-svn', 'bzr'):
438 if self.repotype == 'git-svn':
439 raise VCSException("Authentication is not supported for git-svn")
440 self.username, remote = remote.split('@')
441 if ':' not in self.username:
442 raise VCSException("Password required with username")
443 self.username, self.password = self.username.split(':')
447 self.clone_failed = False
448 self.refreshed = False
454 # Take the local repository to a clean version of the given revision, which
455 # is specificed in the VCS's native format. Beforehand, the repository can
456 # be dirty, or even non-existent. If the repository does already exist
457 # locally, it will be updated from the origin, but only once in the
458 # lifetime of the vcs object.
459 # None is acceptable for 'rev' if you know you are cloning a clean copy of
460 # the repo - otherwise it must specify a valid revision.
461 def gotorevision(self, rev, refresh=True):
463 if self.clone_failed:
464 raise VCSException("Downloading the repository already failed once, not trying again.")
466 # The .fdroidvcs-id file for a repo tells us what VCS type
467 # and remote that directory was created from, allowing us to drop it
468 # automatically if either of those things changes.
469 fdpath = os.path.join(self.local, '..',
470 '.fdroidvcs-' + os.path.basename(self.local))
471 cdata = self.repotype() + ' ' + self.remote
474 if os.path.exists(self.local):
475 if os.path.exists(fdpath):
476 with open(fdpath, 'r') as f:
477 fsdata = f.read().strip()
482 logging.info("Repository details for %s changed - deleting" % (
486 logging.info("Repository details for %s missing - deleting" % (
489 shutil.rmtree(self.local)
493 self.refreshed = True
496 self.gotorevisionx(rev)
497 except FDroidException, e:
500 # If necessary, write the .fdroidvcs file.
501 if writeback and not self.clone_failed:
502 with open(fdpath, 'w') as f:
508 # Derived classes need to implement this. It's called once basic checking
509 # has been performend.
510 def gotorevisionx(self, rev):
511 raise VCSException("This VCS type doesn't define gotorevisionx")
513 # Initialise and update submodules
514 def initsubmodules(self):
515 raise VCSException('Submodules not supported for this vcs type')
517 # Get a list of all known tags
519 if not self._gettags:
520 raise VCSException('gettags not supported for this vcs type')
522 for tag in self._gettags():
523 if re.match('[-A-Za-z0-9_. ]+$', tag):
527 def latesttags(self, tags, number):
528 """Get the most recent tags in a given list.
530 :param tags: a list of tags
531 :param number: the number to return
532 :returns: A list containing the most recent tags in the provided
533 list, up to the maximum number given.
535 raise VCSException('latesttags not supported for this vcs type')
537 # Get current commit reference (hash, revision, etc)
539 raise VCSException('getref not supported for this vcs type')
541 # Returns the srclib (name, path) used in setting up the current
552 # If the local directory exists, but is somehow not a git repository, git
553 # will traverse up the directory tree until it finds one that is (i.e.
554 # fdroidserver) and then we'll proceed to destroy it! This is called as
557 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
558 result = p.output.rstrip()
559 if not result.endswith(self.local):
560 raise VCSException('Repository mismatch')
562 def gotorevisionx(self, rev):
563 if not os.path.exists(self.local):
565 p = FDroidPopen(['git', 'clone', self.remote, self.local])
566 if p.returncode != 0:
567 self.clone_failed = True
568 raise VCSException("Git clone failed", p.output)
572 # Discard any working tree changes
573 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
574 'git', 'reset', '--hard'], cwd=self.local, output=False)
575 if p.returncode != 0:
576 raise VCSException("Git reset failed", p.output)
577 # Remove untracked files now, in case they're tracked in the target
578 # revision (it happens!)
579 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
580 'git', 'clean', '-dffx'], cwd=self.local, output=False)
581 if p.returncode != 0:
582 raise VCSException("Git clean failed", p.output)
583 if not self.refreshed:
584 # Get latest commits and tags from remote
585 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
586 if p.returncode != 0:
587 raise VCSException("Git fetch failed", p.output)
588 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
589 if p.returncode != 0:
590 raise VCSException("Git fetch failed", p.output)
591 # Recreate origin/HEAD as git clone would do it, in case it disappeared
592 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
593 if p.returncode != 0:
594 lines = p.output.splitlines()
595 if 'Multiple remote HEAD branches' not in lines[0]:
596 raise VCSException("Git remote set-head failed", p.output)
597 branch = lines[1].split(' ')[-1]
598 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
599 if p2.returncode != 0:
600 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
601 self.refreshed = True
602 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
603 # a github repo. Most of the time this is the same as origin/master.
604 rev = rev or 'origin/HEAD'
605 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
606 if p.returncode != 0:
607 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
608 # Get rid of any uncontrolled files left behind
609 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
610 if p.returncode != 0:
611 raise VCSException("Git clean failed", p.output)
613 def initsubmodules(self):
615 submfile = os.path.join(self.local, '.gitmodules')
616 if not os.path.isfile(submfile):
617 raise VCSException("No git submodules available")
619 # fix submodules not accessible without an account and public key auth
620 with open(submfile, 'r') as f:
621 lines = f.readlines()
622 with open(submfile, 'w') as f:
624 if 'git@github.com' in line:
625 line = line.replace('git@github.com:', 'https://github.com/')
628 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
629 if p.returncode != 0:
630 raise VCSException("Git submodule sync failed", p.output)
631 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
632 if p.returncode != 0:
633 raise VCSException("Git submodule update failed", p.output)
637 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
638 return p.output.splitlines()
640 def latesttags(self, tags, number):
645 ['git', 'show', '--format=format:%ct', '-s', tag],
646 cwd=self.local, output=False)
647 # Timestamp is on the last line. For a normal tag, it's the only
648 # line, but for annotated tags, the rest of the info precedes it.
649 ts = int(p.output.splitlines()[-1])
652 for _, t in sorted(tl)[-number:]:
657 class vcs_gitsvn(vcs):
662 # If the local directory exists, but is somehow not a git repository, git
663 # will traverse up the directory tree until it finds one that is (i.e.
664 # fdroidserver) and then we'll proceed to destory it! This is called as
667 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
668 result = p.output.rstrip()
669 if not result.endswith(self.local):
670 raise VCSException('Repository mismatch')
672 def gotorevisionx(self, rev):
673 if not os.path.exists(self.local):
675 gitsvn_args = ['git', 'svn', 'clone']
676 if ';' in self.remote:
677 remote_split = self.remote.split(';')
678 for i in remote_split[1:]:
679 if i.startswith('trunk='):
680 gitsvn_args.extend(['-T', i[6:]])
681 elif i.startswith('tags='):
682 gitsvn_args.extend(['-t', i[5:]])
683 elif i.startswith('branches='):
684 gitsvn_args.extend(['-b', i[9:]])
685 gitsvn_args.extend([remote_split[0], self.local])
686 p = FDroidPopen(gitsvn_args, output=False)
687 if p.returncode != 0:
688 self.clone_failed = True
689 raise VCSException("Git svn clone failed", p.output)
691 gitsvn_args.extend([self.remote, self.local])
692 p = FDroidPopen(gitsvn_args, output=False)
693 if p.returncode != 0:
694 self.clone_failed = True
695 raise VCSException("Git svn clone failed", p.output)
699 # Discard any working tree changes
700 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
701 if p.returncode != 0:
702 raise VCSException("Git reset failed", p.output)
703 # Remove untracked files now, in case they're tracked in the target
704 # revision (it happens!)
705 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
706 if p.returncode != 0:
707 raise VCSException("Git clean failed", p.output)
708 if not self.refreshed:
709 # Get new commits, branches and tags from repo
710 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
711 if p.returncode != 0:
712 raise VCSException("Git svn fetch failed")
713 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
714 if p.returncode != 0:
715 raise VCSException("Git svn rebase failed", p.output)
716 self.refreshed = True
718 rev = rev or 'master'
720 nospaces_rev = rev.replace(' ', '%20')
721 # Try finding a svn tag
722 for treeish in ['origin/', '']:
723 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
724 if p.returncode == 0:
726 if p.returncode != 0:
727 # No tag found, normal svn rev translation
728 # Translate svn rev into git format
729 rev_split = rev.split('/')
732 for treeish in ['origin/', '']:
733 if len(rev_split) > 1:
734 treeish += rev_split[0]
735 svn_rev = rev_split[1]
738 # if no branch is specified, then assume trunk (i.e. 'master' branch):
742 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
744 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
745 git_rev = p.output.rstrip()
747 if p.returncode == 0 and git_rev:
750 if p.returncode != 0 or not git_rev:
751 # Try a plain git checkout as a last resort
752 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
753 if p.returncode != 0:
754 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
756 # Check out the git rev equivalent to the svn rev
757 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
758 if p.returncode != 0:
759 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
761 # Get rid of any uncontrolled files left behind
762 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
763 if p.returncode != 0:
764 raise VCSException("Git clean failed", p.output)
768 for treeish in ['origin/', '']:
769 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
775 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
776 if p.returncode != 0:
778 return p.output.strip()
786 def gotorevisionx(self, rev):
787 if not os.path.exists(self.local):
788 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
789 if p.returncode != 0:
790 self.clone_failed = True
791 raise VCSException("Hg clone failed", p.output)
793 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
794 if p.returncode != 0:
795 raise VCSException("Hg status failed", p.output)
796 for line in p.output.splitlines():
797 if not line.startswith('? '):
798 raise VCSException("Unexpected output from hg status -uS: " + line)
799 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
800 if not self.refreshed:
801 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
802 if p.returncode != 0:
803 raise VCSException("Hg pull failed", p.output)
804 self.refreshed = True
806 rev = rev or 'default'
809 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
810 if p.returncode != 0:
811 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
812 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
813 # Also delete untracked files, we have to enable purge extension for that:
814 if "'purge' is provided by the following extension" in p.output:
815 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
816 myfile.write("\n[extensions]\nhgext.purge=\n")
817 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
818 if p.returncode != 0:
819 raise VCSException("HG purge failed", p.output)
820 elif p.returncode != 0:
821 raise VCSException("HG purge failed", p.output)
824 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
825 return p.output.splitlines()[1:]
833 def gotorevisionx(self, rev):
834 if not os.path.exists(self.local):
835 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
836 if p.returncode != 0:
837 self.clone_failed = True
838 raise VCSException("Bzr branch failed", p.output)
840 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
841 if p.returncode != 0:
842 raise VCSException("Bzr revert failed", p.output)
843 if not self.refreshed:
844 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
845 if p.returncode != 0:
846 raise VCSException("Bzr update failed", p.output)
847 self.refreshed = True
849 revargs = list(['-r', rev] if rev else [])
850 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
851 if p.returncode != 0:
852 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
855 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
856 return [tag.split(' ')[0].strip() for tag in
857 p.output.splitlines()]
860 def unescape_string(string):
861 if string[0] == '"' and string[-1] == '"':
864 return string.replace("\\'", "'")
867 def retrieve_string(app_dir, string, xmlfiles=None):
872 os.path.join(app_dir, 'res'),
873 os.path.join(app_dir, 'src', 'main', 'res'),
875 for r, d, f in os.walk(res_dir):
876 if os.path.basename(r) == 'values':
877 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
879 if not string.startswith('@string/'):
880 return unescape_string(string)
882 name = string[len('@string/'):]
884 for path in xmlfiles:
885 if not os.path.isfile(path):
887 xml = parse_xml(path)
888 element = xml.find('string[@name="' + name + '"]')
889 if element is not None and element.text is not None:
890 return retrieve_string(app_dir, element.text.encode('utf-8'), xmlfiles)
895 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
896 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
899 # Return list of existing files that will be used to find the highest vercode
900 def manifest_paths(app_dir, flavours):
902 possible_manifests = \
903 [os.path.join(app_dir, 'AndroidManifest.xml'),
904 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
905 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
906 os.path.join(app_dir, 'build.gradle')]
908 for flavour in flavours:
911 possible_manifests.append(
912 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
914 return [path for path in possible_manifests if os.path.isfile(path)]
917 # Retrieve the package name. Returns the name, or None if not found.
918 def fetch_real_name(app_dir, flavours):
919 for path in manifest_paths(app_dir, flavours):
920 if not has_extension(path, 'xml') or not os.path.isfile(path):
922 logging.debug("fetch_real_name: Checking manifest at " + path)
923 xml = parse_xml(path)
924 app = xml.find('application')
925 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
927 label = app.attrib["{http://schemas.android.com/apk/res/android}label"].encode('utf-8')
928 result = retrieve_string_singleline(app_dir, label)
930 result = result.strip()
935 def get_library_references(root_dir):
937 proppath = os.path.join(root_dir, 'project.properties')
938 if not os.path.isfile(proppath):
940 for line in file(proppath):
941 if not line.startswith('android.library.reference.'):
943 path = line.split('=')[1].strip()
944 relpath = os.path.join(root_dir, path)
945 if not os.path.isdir(relpath):
947 logging.debug("Found subproject at %s" % path)
948 libraries.append(path)
952 def ant_subprojects(root_dir):
953 subprojects = get_library_references(root_dir)
954 for subpath in subprojects:
955 subrelpath = os.path.join(root_dir, subpath)
956 for p in get_library_references(subrelpath):
957 relp = os.path.normpath(os.path.join(subpath, p))
958 if relp not in subprojects:
959 subprojects.insert(0, relp)
963 def remove_debuggable_flags(root_dir):
964 # Remove forced debuggable flags
965 logging.debug("Removing debuggable flags from %s" % root_dir)
966 for root, dirs, files in os.walk(root_dir):
967 if 'AndroidManifest.xml' in files:
968 path = os.path.join(root, 'AndroidManifest.xml')
969 p = FDroidPopen(['sed', '-i', 's/android:debuggable="[^"]*"//g', path], output=False)
970 if p.returncode != 0:
971 raise BuildException("Failed to remove debuggable flags of %s" % path)
974 # Extract some information from the AndroidManifest.xml at the given path.
975 # Returns (version, vercode, package), any or all of which might be None.
976 # All values returned are strings.
977 def parse_androidmanifests(paths, ignoreversions=None):
980 return (None, None, None)
982 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
983 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
984 psearch_g = re.compile(r'.*packageName *=* *["\']([^"]+)["\'].*').search
986 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
994 if not os.path.isfile(path):
997 logging.debug("Parsing manifest at {0}".format(path))
998 gradle = has_extension(path, 'gradle')
1001 # Remember package name, may be defined separately from version+vercode
1002 package = max_package
1005 for line in file(path):
1007 matches = psearch_g(line)
1009 package = matches.group(1)
1011 matches = vnsearch_g(line)
1013 version = matches.group(2)
1015 matches = vcsearch_g(line)
1017 vercode = matches.group(1)
1019 xml = parse_xml(path)
1020 if "package" in xml.attrib:
1021 package = xml.attrib["package"].encode('utf-8')
1022 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1023 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"].encode('utf-8')
1024 base_dir = os.path.dirname(path)
1025 version = retrieve_string_singleline(base_dir, version)
1026 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1027 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"].encode('utf-8')
1028 if string_is_integer(a):
1031 logging.debug("..got package={0}, version={1}, vercode={2}"
1032 .format(package, version, vercode))
1034 # Always grab the package name and version name in case they are not
1035 # together with the highest version code
1036 if max_package is None and package is not None:
1037 max_package = package
1038 if max_version is None and version is not None:
1039 max_version = version
1041 if max_vercode is None or (vercode is not None and vercode > max_vercode):
1042 if not ignoresearch or not ignoresearch(version):
1043 if version is not None:
1044 max_version = version
1045 if vercode is not None:
1046 max_vercode = vercode
1047 if package is not None:
1048 max_package = package
1050 max_version = "Ignore"
1052 if max_version is None:
1053 max_version = "Unknown"
1055 if max_package and not is_valid_package_name(max_package):
1056 raise FDroidException("Invalid package name {0}".format(max_package))
1058 return (max_version, max_vercode, max_package)
1061 def is_valid_package_name(name):
1062 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1065 class FDroidException(Exception):
1067 def __init__(self, value, detail=None):
1069 self.detail = detail
1071 def get_wikitext(self):
1072 ret = repr(self.value) + "\n"
1076 txt = self.detail[-8192:] if len(self.detail) > 8192 else self.detail
1084 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1088 class VCSException(FDroidException):
1092 class BuildException(FDroidException):
1096 # Get the specified source library.
1097 # Returns the path to it. Normally this is the path to be used when referencing
1098 # it, which may be a subdirectory of the actual project. If you want the base
1099 # directory of the project, pass 'basepath=True'.
1100 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1101 raw=False, prepare=True, preponly=False, refresh=True):
1109 name, ref = spec.split('@')
1111 number, name = name.split(':', 1)
1113 name, subdir = name.split('/', 1)
1115 if name not in metadata.srclibs:
1116 raise VCSException('srclib ' + name + ' not found.')
1118 srclib = metadata.srclibs[name]
1120 sdir = os.path.join(srclib_dir, name)
1123 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1124 vcs.srclib = (name, number, sdir)
1126 vcs.gotorevision(ref, refresh)
1133 libdir = os.path.join(sdir, subdir)
1134 elif srclib["Subdir"]:
1135 for subdir in srclib["Subdir"]:
1136 libdir_candidate = os.path.join(sdir, subdir)
1137 if os.path.exists(libdir_candidate):
1138 libdir = libdir_candidate
1144 remove_signing_keys(sdir)
1145 remove_debuggable_flags(sdir)
1149 if srclib["Prepare"]:
1150 cmd = replace_config_vars(srclib["Prepare"], None)
1152 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1153 if p.returncode != 0:
1154 raise BuildException("Error running prepare command for srclib %s"
1160 return (name, number, libdir)
1163 # Prepare the source code for a particular build
1164 # 'vcs' - the appropriate vcs object for the application
1165 # 'app' - the application details from the metadata
1166 # 'build' - the build details from the metadata
1167 # 'build_dir' - the path to the build directory, usually
1169 # 'srclib_dir' - the path to the source libraries directory, usually
1171 # 'extlib_dir' - the path to the external libraries directory, usually
1173 # Returns the (root, srclibpaths) where:
1174 # 'root' is the root directory, which may be the same as 'build_dir' or may
1175 # be a subdirectory of it.
1176 # 'srclibpaths' is information on the srclibs being used
1177 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1179 # Optionally, the actual app source can be in a subdirectory
1181 root_dir = os.path.join(build_dir, build['subdir'])
1183 root_dir = build_dir
1185 # Get a working copy of the right revision
1186 logging.info("Getting source for revision " + build['commit'])
1187 vcs.gotorevision(build['commit'], refresh)
1189 # Initialise submodules if required
1190 if build['submodules']:
1191 logging.info("Initialising submodules")
1192 vcs.initsubmodules()
1194 # Check that a subdir (if we're using one) exists. This has to happen
1195 # after the checkout, since it might not exist elsewhere
1196 if not os.path.exists(root_dir):
1197 raise BuildException('Missing subdir ' + root_dir)
1199 # Run an init command if one is required
1201 cmd = replace_config_vars(build['init'], build)
1202 logging.info("Running 'init' commands in %s" % root_dir)
1204 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1205 if p.returncode != 0:
1206 raise BuildException("Error running init command for %s:%s" %
1207 (app['id'], build['version']), p.output)
1209 # Apply patches if any
1211 logging.info("Applying patches")
1212 for patch in build['patch']:
1213 patch = patch.strip()
1214 logging.info("Applying " + patch)
1215 patch_path = os.path.join('metadata', app['id'], patch)
1216 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1217 if p.returncode != 0:
1218 raise BuildException("Failed to apply patch %s" % patch_path)
1220 # Get required source libraries
1222 if build['srclibs']:
1223 logging.info("Collecting source libraries")
1224 for lib in build['srclibs']:
1225 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
1227 for name, number, libpath in srclibpaths:
1228 place_srclib(root_dir, int(number) if number else None, libpath)
1230 basesrclib = vcs.getsrclib()
1231 # If one was used for the main source, add that too.
1233 srclibpaths.append(basesrclib)
1235 # Update the local.properties file
1236 localprops = [os.path.join(build_dir, 'local.properties')]
1238 localprops += [os.path.join(root_dir, 'local.properties')]
1239 for path in localprops:
1241 if os.path.isfile(path):
1242 logging.info("Updating local.properties file at %s" % path)
1248 logging.info("Creating local.properties file at %s" % path)
1249 # Fix old-fashioned 'sdk-location' by copying
1250 # from sdk.dir, if necessary
1251 if build['oldsdkloc']:
1252 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1253 re.S | re.M).group(1)
1254 props += "sdk-location=%s\n" % sdkloc
1256 props += "sdk.dir=%s\n" % config['sdk_path']
1257 props += "sdk-location=%s\n" % config['sdk_path']
1258 if build['ndk_path']:
1260 props += "ndk.dir=%s\n" % build['ndk_path']
1261 props += "ndk-location=%s\n" % build['ndk_path']
1262 # Add java.encoding if necessary
1263 if build['encoding']:
1264 props += "java.encoding=%s\n" % build['encoding']
1270 if build['type'] == 'gradle':
1271 flavours = build['gradle']
1273 version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1274 gradlepluginver = None
1276 gradle_dirs = [root_dir]
1278 # Parent dir build.gradle
1279 parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
1280 if parent_dir.startswith(build_dir):
1281 gradle_dirs.append(parent_dir)
1283 for dir_path in gradle_dirs:
1286 if not os.path.isdir(dir_path):
1288 for filename in os.listdir(dir_path):
1289 if not filename.endswith('.gradle'):
1291 path = os.path.join(dir_path, filename)
1292 if not os.path.isfile(path):
1294 for line in file(path):
1295 match = version_regex.match(line)
1297 gradlepluginver = match.group(1)
1301 build['gradlepluginver'] = LooseVersion(gradlepluginver)
1303 logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
1304 build['gradlepluginver'] = LooseVersion('0.11')
1307 n = build["target"].split('-')[1]
1308 FDroidPopen(['sed', '-i',
1309 's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g',
1310 'build.gradle'], cwd=root_dir, output=False)
1312 # Remove forced debuggable flags
1313 remove_debuggable_flags(root_dir)
1315 # Insert version code and number into the manifest if necessary
1316 if build['forceversion']:
1317 logging.info("Changing the version name")
1318 for path in manifest_paths(root_dir, flavours):
1319 if not os.path.isfile(path):
1321 if has_extension(path, 'xml'):
1322 p = FDroidPopen(['sed', '-i',
1323 's/android:versionName="[^"]*"/android:versionName="' + build['version'] + '"/g',
1324 path], output=False)
1325 if p.returncode != 0:
1326 raise BuildException("Failed to amend manifest")
1327 elif has_extension(path, 'gradle'):
1328 p = FDroidPopen(['sed', '-i',
1329 's/versionName *=* *.*/versionName = "' + build['version'] + '"/g',
1330 path], output=False)
1331 if p.returncode != 0:
1332 raise BuildException("Failed to amend build.gradle")
1333 if build['forcevercode']:
1334 logging.info("Changing the version code")
1335 for path in manifest_paths(root_dir, flavours):
1336 if not os.path.isfile(path):
1338 if has_extension(path, 'xml'):
1339 p = FDroidPopen(['sed', '-i',
1340 's/android:versionCode="[^"]*"/android:versionCode="' + build['vercode'] + '"/g',
1341 path], output=False)
1342 if p.returncode != 0:
1343 raise BuildException("Failed to amend manifest")
1344 elif has_extension(path, 'gradle'):
1345 p = FDroidPopen(['sed', '-i',
1346 's/versionCode *=* *[0-9]*/versionCode = ' + build['vercode'] + '/g',
1347 path], output=False)
1348 if p.returncode != 0:
1349 raise BuildException("Failed to amend build.gradle")
1351 # Delete unwanted files
1353 logging.info("Removing specified files")
1354 for part in getpaths(build_dir, build, 'rm'):
1355 dest = os.path.join(build_dir, part)
1356 logging.info("Removing {0}".format(part))
1357 if os.path.lexists(dest):
1358 if os.path.islink(dest):
1359 FDroidPopen(['unlink', dest], output=False)
1361 FDroidPopen(['rm', '-rf', dest], output=False)
1363 logging.info("...but it didn't exist")
1365 remove_signing_keys(build_dir)
1367 # Add required external libraries
1368 if build['extlibs']:
1369 logging.info("Collecting prebuilt libraries")
1370 libsdir = os.path.join(root_dir, 'libs')
1371 if not os.path.exists(libsdir):
1373 for lib in build['extlibs']:
1375 logging.info("...installing extlib {0}".format(lib))
1376 libf = os.path.basename(lib)
1377 libsrc = os.path.join(extlib_dir, lib)
1378 if not os.path.exists(libsrc):
1379 raise BuildException("Missing extlib file {0}".format(libsrc))
1380 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1382 # Run a pre-build command if one is required
1383 if build['prebuild']:
1384 logging.info("Running 'prebuild' commands in %s" % root_dir)
1386 cmd = replace_config_vars(build['prebuild'], build)
1388 # Substitute source library paths into prebuild commands
1389 for name, number, libpath in srclibpaths:
1390 libpath = os.path.relpath(libpath, root_dir)
1391 cmd = cmd.replace('$$' + name + '$$', libpath)
1393 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1394 if p.returncode != 0:
1395 raise BuildException("Error running prebuild command for %s:%s" %
1396 (app['id'], build['version']), p.output)
1398 # Generate (or update) the ant build file, build.xml...
1399 if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
1400 parms = ['android', 'update', 'lib-project']
1401 lparms = ['android', 'update', 'project']
1404 parms += ['-t', build['target']]
1405 lparms += ['-t', build['target']]
1406 if build['update'] == ['auto']:
1407 update_dirs = ant_subprojects(root_dir) + ['.']
1409 update_dirs = build['update']
1411 for d in update_dirs:
1412 subdir = os.path.join(root_dir, d)
1414 logging.debug("Updating main project")
1415 cmd = parms + ['-p', d]
1417 logging.debug("Updating subproject %s" % d)
1418 cmd = lparms + ['-p', d]
1419 p = SdkToolsPopen(cmd, cwd=root_dir)
1420 # Check to see whether an error was returned without a proper exit
1421 # code (this is the case for the 'no target set or target invalid'
1423 if p.returncode != 0 or p.output.startswith("Error: "):
1424 raise BuildException("Failed to update project at %s" % d, p.output)
1425 # Clean update dirs via ant
1427 logging.info("Cleaning subproject %s" % d)
1428 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1430 return (root_dir, srclibpaths)
1433 # Split and extend via globbing the paths from a field
1434 def getpaths(build_dir, build, field):
1436 for p in build[field]:
1438 full_path = os.path.join(build_dir, p)
1439 full_path = os.path.normpath(full_path)
1440 paths += [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1444 # Scan the source code in the given directory (and all subdirectories)
1445 # and return the number of fatal problems encountered
1446 def scan_source(build_dir, root_dir, thisbuild):
1450 # Common known non-free blobs (always lower case):
1452 re.compile(r'.*flurryagent', re.IGNORECASE),
1453 re.compile(r'.*paypal.*mpl', re.IGNORECASE),
1454 re.compile(r'.*google.*analytics', re.IGNORECASE),
1455 re.compile(r'.*admob.*sdk.*android', re.IGNORECASE),
1456 re.compile(r'.*google.*ad.*view', re.IGNORECASE),
1457 re.compile(r'.*google.*admob', re.IGNORECASE),
1458 re.compile(r'.*google.*play.*services', re.IGNORECASE),
1459 re.compile(r'.*crittercism', re.IGNORECASE),
1460 re.compile(r'.*heyzap', re.IGNORECASE),
1461 re.compile(r'.*jpct.*ae', re.IGNORECASE),
1462 re.compile(r'.*youtube.*android.*player.*api', re.IGNORECASE),
1463 re.compile(r'.*bugsense', re.IGNORECASE),
1464 re.compile(r'.*crashlytics', re.IGNORECASE),
1465 re.compile(r'.*ouya.*sdk', re.IGNORECASE),
1466 re.compile(r'.*libspen23', re.IGNORECASE),
1469 scanignore = getpaths(build_dir, thisbuild, 'scanignore')
1470 scandelete = getpaths(build_dir, thisbuild, 'scandelete')
1472 scanignore_worked = set()
1473 scandelete_worked = set()
1476 ms = magic.open(magic.MIME_TYPE)
1478 except AttributeError:
1482 for p in scanignore:
1483 if fd.startswith(p):
1484 scanignore_worked.add(p)
1489 for p in scandelete:
1490 if fd.startswith(p):
1491 scandelete_worked.add(p)
1495 def ignoreproblem(what, fd, fp):
1496 logging.info('Ignoring %s at %s' % (what, fd))
1499 def removeproblem(what, fd, fp):
1500 logging.info('Removing %s at %s' % (what, fd))
1504 def warnproblem(what, fd):
1505 logging.warn('Found %s at %s' % (what, fd))
1507 def handleproblem(what, fd, fp):
1509 return ignoreproblem(what, fd, fp)
1511 return removeproblem(what, fd, fp)
1512 logging.error('Found %s at %s' % (what, fd))
1515 # Iterate through all files in the source code
1516 for r, d, f in os.walk(build_dir, topdown=True):
1518 # It's topdown, so checking the basename is enough
1519 for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
1525 # Path (relative) to the file
1526 fp = os.path.join(r, curfile)
1527 fd = fp[len(build_dir) + 1:]
1530 mime = magic.from_file(fp, mime=True) if ms is None else ms.file(fp)
1531 except UnicodeError:
1532 warnproblem('malformed magic number', fd)
1534 if mime == 'application/x-sharedlib':
1535 count += handleproblem('shared library', fd, fp)
1537 elif mime == 'application/x-archive':
1538 count += handleproblem('static library', fd, fp)
1540 elif mime == 'application/x-executable':
1541 count += handleproblem('binary executable', fd, fp)
1543 elif mime == 'application/x-java-applet':
1544 count += handleproblem('Java compiled class', fd, fp)
1549 'application/java-archive',
1550 'application/octet-stream',
1553 if has_extension(fp, 'apk'):
1554 removeproblem('APK file', fd, fp)
1556 elif has_extension(fp, 'jar'):
1558 if any(suspect.match(curfile) for suspect in usual_suspects):
1559 count += handleproblem('usual supect', fd, fp)
1561 warnproblem('JAR file', fd)
1563 elif has_extension(fp, 'zip'):
1564 warnproblem('ZIP file', fd)
1567 warnproblem('unknown compressed or binary file', fd)
1569 elif has_extension(fp, 'java'):
1570 if not os.path.isfile(fp):
1572 for line in file(fp):
1573 if 'DexClassLoader' in line:
1574 count += handleproblem('DexClassLoader', fd, fp)
1577 elif has_extension(fp, 'gradle'):
1578 if not os.path.isfile(fp):
1580 for i, line in enumerate(file(fp)):
1581 if any(suspect.match(line) for suspect in usual_suspects):
1582 count += handleproblem('usual suspect at line %d' % i, fd, fp)
1587 for p in scanignore:
1588 if p not in scanignore_worked:
1589 logging.error('Unused scanignore path: %s' % p)
1592 for p in scandelete:
1593 if p not in scandelete_worked:
1594 logging.error('Unused scandelete path: %s' % p)
1597 # Presence of a jni directory without buildjni=yes might
1598 # indicate a problem (if it's not a problem, explicitly use
1599 # buildjni=no to bypass this check)
1600 if (os.path.exists(os.path.join(root_dir, 'jni')) and
1601 not thisbuild['buildjni']):
1602 logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
1611 self.path = os.path.join('stats', 'known_apks.txt')
1613 if os.path.isfile(self.path):
1614 for line in file(self.path):
1615 t = line.rstrip().split(' ')
1617 self.apks[t[0]] = (t[1], None)
1619 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1620 self.changed = False
1622 def writeifchanged(self):
1624 if not os.path.exists('stats'):
1626 f = open(self.path, 'w')
1628 for apk, app in self.apks.iteritems():
1630 line = apk + ' ' + appid
1632 line += ' ' + time.strftime('%Y-%m-%d', added)
1634 for line in sorted(lst):
1635 f.write(line + '\n')
1638 # Record an apk (if it's new, otherwise does nothing)
1639 # Returns the date it was added.
1640 def recordapk(self, apk, app):
1641 if apk not in self.apks:
1642 self.apks[apk] = (app, time.gmtime(time.time()))
1644 _, added = self.apks[apk]
1647 # Look up information - given the 'apkname', returns (app id, date added/None).
1648 # Or returns None for an unknown apk.
1649 def getapp(self, apkname):
1650 if apkname in self.apks:
1651 return self.apks[apkname]
1654 # Get the most recent 'num' apps added to the repo, as a list of package ids
1655 # with the most recent first.
1656 def getlatest(self, num):
1658 for apk, app in self.apks.iteritems():
1662 if apps[appid] > added:
1666 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1667 lst = [app for app, _ in sortedapps]
1672 def isApkDebuggable(apkfile, config):
1673 """Returns True if the given apk file is debuggable
1675 :param apkfile: full path to the apk to check"""
1677 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1679 if p.returncode != 0:
1680 logging.critical("Failed to get apk manifest information")
1682 for line in p.output.splitlines():
1683 if 'android:debuggable' in line and not line.endswith('0x0'):
1688 class AsynchronousFileReader(threading.Thread):
1691 Helper class to implement asynchronous reading of a file
1692 in a separate thread. Pushes read lines on a queue to
1693 be consumed in another thread.
1696 def __init__(self, fd, queue):
1697 assert isinstance(queue, Queue.Queue)
1698 assert callable(fd.readline)
1699 threading.Thread.__init__(self)
1704 '''The body of the tread: read lines and put them on the queue.'''
1705 for line in iter(self._fd.readline, ''):
1706 self._queue.put(line)
1709 '''Check whether there is no more content to expect.'''
1710 return not self.is_alive() and self._queue.empty()
1718 def SdkToolsPopen(commands, cwd=None, output=True):
1720 if cmd not in config:
1721 config[cmd] = find_sdk_tools_cmd(commands[0])
1722 return FDroidPopen([config[cmd]] + commands[1:],
1723 cwd=cwd, output=output)
1726 def FDroidPopen(commands, cwd=None, output=True):
1728 Run a command and capture the possibly huge output.
1730 :param commands: command and argument list like in subprocess.Popen
1731 :param cwd: optionally specifies a working directory
1732 :returns: A PopenResult.
1738 cwd = os.path.normpath(cwd)
1739 logging.debug("Directory: %s" % cwd)
1740 logging.debug("> %s" % ' '.join(commands))
1742 result = PopenResult()
1745 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
1746 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1748 raise BuildException("OSError while trying to execute " +
1749 ' '.join(commands) + ': ' + str(e))
1751 stdout_queue = Queue.Queue()
1752 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1753 stdout_reader.start()
1755 # Check the queue for output (until there is no more to get)
1756 while not stdout_reader.eof():
1757 while not stdout_queue.empty():
1758 line = stdout_queue.get()
1759 if output and options.verbose:
1760 # Output directly to console
1761 sys.stderr.write(line)
1763 result.output += line
1767 result.returncode = p.wait()
1771 def remove_signing_keys(build_dir):
1772 comment = re.compile(r'[ ]*//')
1773 signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1775 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1776 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1777 re.compile(r'.*variant\.outputFile = .*'),
1778 re.compile(r'.*output\.outputFile = .*'),
1779 re.compile(r'.*\.readLine\(.*'),
1781 for root, dirs, files in os.walk(build_dir):
1782 if 'build.gradle' in files:
1783 path = os.path.join(root, 'build.gradle')
1785 with open(path, "r") as o:
1786 lines = o.readlines()
1792 with open(path, "w") as o:
1793 while i < len(lines):
1796 while line.endswith('\\\n'):
1797 line = line.rstrip('\\\n') + lines[i]
1800 if comment.match(line):
1804 opened += line.count('{')
1805 opened -= line.count('}')
1808 if signing_configs.match(line):
1813 if any(s.match(line) for s in line_matches):
1821 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1824 'project.properties',
1826 'default.properties',
1827 'ant.properties', ]:
1828 if propfile in files:
1829 path = os.path.join(root, propfile)
1831 with open(path, "r") as o:
1832 lines = o.readlines()
1836 with open(path, "w") as o:
1838 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1845 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1848 def reset_env_path():
1849 global env, orig_path
1850 env['PATH'] = orig_path
1853 def add_to_env_path(path):
1855 paths = env['PATH'].split(os.pathsep)
1859 env['PATH'] = os.pathsep.join(paths)
1862 def replace_config_vars(cmd, build):
1864 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1865 # env['ANDROID_NDK'] is set in build_local right before prepare_source
1866 cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
1867 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1868 if build is not None:
1869 cmd = cmd.replace('$$COMMIT$$', build['commit'])
1870 cmd = cmd.replace('$$VERSION$$', build['version'])
1871 cmd = cmd.replace('$$VERCODE$$', build['vercode'])
1875 def place_srclib(root_dir, number, libpath):
1878 relpath = os.path.relpath(libpath, root_dir)
1879 proppath = os.path.join(root_dir, 'project.properties')
1882 if os.path.isfile(proppath):
1883 with open(proppath, "r") as o:
1884 lines = o.readlines()
1886 with open(proppath, "w") as o:
1889 if line.startswith('android.library.reference.%d=' % number):
1890 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1895 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1898 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
1899 """Verify that two apks are the same
1901 One of the inputs is signed, the other is unsigned. The signature metadata
1902 is transferred from the signed to the unsigned apk, and then jarsigner is
1903 used to verify that the signature from the signed apk is also varlid for
1905 :param signed_apk: Path to a signed apk file
1906 :param unsigned_apk: Path to an unsigned apk file expected to match it
1907 :param tmp_dir: Path to directory for temporary files
1908 :returns: None if the verification is successful, otherwise a string
1909 describing what went wrong.
1911 sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
1912 with ZipFile(signed_apk) as signed_apk_as_zip:
1913 meta_inf_files = ['META-INF/MANIFEST.MF']
1914 for f in signed_apk_as_zip.namelist():
1915 if sigfile.match(f):
1916 meta_inf_files.append(f)
1917 if len(meta_inf_files) < 3:
1918 return "Signature files missing from {0}".format(signed_apk)
1919 signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
1920 with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
1921 for meta_inf_file in meta_inf_files:
1922 unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
1924 if subprocess.call(['jarsigner', '-verify', unsigned_apk]) != 0:
1925 logging.info("...NOT verified - {0}".format(signed_apk))
1926 return compare_apks(signed_apk, unsigned_apk, tmp_dir)
1927 logging.info("...successfully verified")
1931 def compare_apks(apk1, apk2, tmp_dir):
1934 Returns None if the apk content is the same (apart from the signing key),
1935 otherwise a string describing what's different, or what went wrong when
1936 trying to do the comparison.
1939 badchars = re.compile('''[/ :;'"]''')
1940 apk1dir = os.path.join(tmp_dir, badchars.sub('_', apk1[0:-4])) # trim .apk
1941 apk2dir = os.path.join(tmp_dir, badchars.sub('_', apk2[0:-4])) # trim .apk
1942 for d in [apk1dir, apk2dir]:
1943 if os.path.exists(d):
1946 os.mkdir(os.path.join(d, 'jar-xf'))
1948 if subprocess.call(['jar', 'xf',
1949 os.path.abspath(apk1)],
1950 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
1951 return("Failed to unpack " + apk1)
1952 if subprocess.call(['jar', 'xf',
1953 os.path.abspath(apk2)],
1954 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
1955 return("Failed to unpack " + apk2)
1957 # try to find apktool in the path, if it hasn't been manually configed
1958 if 'apktool' not in config:
1959 tmp = find_command('apktool')
1961 config['apktool'] = tmp
1962 if 'apktool' in config:
1963 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
1965 return("Failed to unpack " + apk1)
1966 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
1968 return("Failed to unpack " + apk2)
1970 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
1971 lines = p.output.splitlines()
1972 if len(lines) != 1 or 'META-INF' not in lines[0]:
1973 meld = find_command('meld')
1974 if meld is not None:
1975 p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
1976 return("Unexpected diff output - " + p.output)
1978 # since everything verifies, delete the comparison to keep cruft down
1979 shutil.rmtree(apk1dir)
1980 shutil.rmtree(apk2dir)
1982 # If we get here, it seems like they're the same!
1986 def find_command(command):
1987 '''find the full path of a command, or None if it can't be found in the PATH'''
1990 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
1992 fpath, fname = os.path.split(command)
1997 for path in os.environ["PATH"].split(os.pathsep):
1998 path = path.strip('"')
1999 exe_file = os.path.join(path, command)
2000 if is_exe(exe_file):
2007 '''generate a random password for when generating keys'''
2008 h = hashlib.sha256()
2009 h.update(os.urandom(16)) # salt
2010 h.update(bytes(socket.getfqdn()))
2011 return h.digest().encode('base64').strip()
2014 def genkeystore(localconfig):
2015 '''Generate a new key with random passwords and add it to new keystore'''
2016 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2017 keystoredir = os.path.dirname(localconfig['keystore'])
2018 if keystoredir is None or keystoredir == '':
2019 keystoredir = os.path.join(os.getcwd(), keystoredir)
2020 if not os.path.exists(keystoredir):
2021 os.makedirs(keystoredir, mode=0o700)
2023 write_password_file("keystorepass", localconfig['keystorepass'])
2024 write_password_file("keypass", localconfig['keypass'])
2025 p = FDroidPopen(['keytool', '-genkey',
2026 '-keystore', localconfig['keystore'],
2027 '-alias', localconfig['repo_keyalias'],
2028 '-keyalg', 'RSA', '-keysize', '4096',
2029 '-sigalg', 'SHA256withRSA',
2030 '-validity', '10000',
2031 '-storepass:file', config['keystorepassfile'],
2032 '-keypass:file', config['keypassfile'],
2033 '-dname', localconfig['keydname']])
2034 # TODO keypass should be sent via stdin
2035 os.chmod(localconfig['keystore'], 0o0600)
2036 if p.returncode != 0:
2037 raise BuildException("Failed to generate key", p.output)
2038 # now show the lovely key that was just generated
2039 p = FDroidPopen(['keytool', '-list', '-v',
2040 '-keystore', localconfig['keystore'],
2041 '-alias', localconfig['repo_keyalias'],
2042 '-storepass:file', config['keystorepassfile']])
2043 logging.info(p.output.strip() + '\n\n')
2046 def write_to_config(thisconfig, key, value=None):
2047 '''write a key/value to the local config.py'''
2049 origkey = key + '_orig'
2050 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2051 with open('config.py', 'r') as f:
2053 pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
2054 repl = '\n' + key + ' = "' + value + '"'
2055 data = re.sub(pattern, repl, data)
2056 # if this key is not in the file, append it
2057 if not re.match('\s*' + key + '\s*=\s*"', data):
2059 # make sure the file ends with a carraige return
2060 if not re.match('\n$', data):
2062 with open('config.py', 'w') as f:
2066 def parse_xml(path):
2067 return XMLElementTree.parse(path).getroot()
2070 def string_is_integer(string):
2078 def download_file(url, local_filename=None, dldir='tmp'):
2079 filename = url.split('/')[-1]
2080 if local_filename is None:
2081 local_filename = os.path.join(dldir, filename)
2082 # the stream=True parameter keeps memory usage low
2083 r = requests.get(url, stream=True)
2084 with open(local_filename, 'wb') as f:
2085 for chunk in r.iter_content(chunk_size=1024):
2086 if chunk: # filter out keep-alive new chunks
2089 return local_filename