1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
33 from distutils.version import LooseVersion
43 'sdk_path': "$ANDROID_HOME",
44 'ndk_path': "$ANDROID_NDK",
45 'build_tools': "21.1.2",
49 'sync_from_local_copy_dir': False,
50 'make_current_version_link': True,
51 'current_version_name_source': 'Name',
52 'update_stats': False,
56 'stats_to_carbon': False,
58 'build_server_always': False,
59 'keystore': os.path.join("$HOME", '.local', 'share', 'fdroidserver', 'keystore.jks'),
60 'smartcardoptions': [],
66 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
67 'repo_name': "My First FDroid Repo Demo",
68 'repo_icon': "fdroid-icon.png",
69 'repo_description': '''
70 This is a repository of apps to be used with FDroid. Applications in this
71 repository are either official binaries built by the original application
72 developers, or are binaries built from source by the admin of f-droid.org
73 using the tools on https://gitlab.com/u/fdroid.
79 def fill_config_defaults(thisconfig):
80 for k, v in default_config.items():
81 if k not in thisconfig:
84 # Expand paths (~users and $vars)
85 for k in ['sdk_path', 'ndk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
88 v = os.path.expanduser(v)
89 v = os.path.expandvars(v)
92 thisconfig[k + '_orig'] = orig
95 def read_config(opts, config_file='config.py'):
96 """Read the repository config
98 The config is read from config_file, which is in the current directory when
99 any of the repo management commands are used.
101 global config, options, env
103 if config is not None:
105 if not os.path.isfile(config_file):
106 logging.critical("Missing config file - is this a repo directory?")
113 logging.debug("Reading %s" % config_file)
114 execfile(config_file, config)
116 # smartcardoptions must be a list since its command line args for Popen
117 if 'smartcardoptions' in config:
118 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
119 elif 'keystore' in config and config['keystore'] == 'NONE':
120 # keystore='NONE' means use smartcard, these are required defaults
121 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
122 'SunPKCS11-OpenSC', '-providerClass',
123 'sun.security.pkcs11.SunPKCS11',
124 '-providerArg', 'opensc-fdroid.cfg']
126 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
127 st = os.stat(config_file)
128 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
129 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
131 fill_config_defaults(config)
133 # There is no standard, so just set up the most common environment
136 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
137 env[n] = config['sdk_path']
138 for n in ['ANDROID_NDK', 'NDK']:
139 env[n] = config['ndk_path']
141 for k in ["keystorepass", "keypass"]:
143 write_password_file(k)
145 for k in ["repo_description", "archive_description"]:
147 config[k] = clean_description(config[k])
149 if 'serverwebroot' in config:
150 if isinstance(config['serverwebroot'], basestring):
151 roots = [config['serverwebroot']]
152 elif all(isinstance(item, basestring) for item in config['serverwebroot']):
153 roots = config['serverwebroot']
155 raise TypeError('only accepts strings, lists, and tuples')
157 for rootstr in roots:
158 # since this is used with rsync, where trailing slashes have
159 # meaning, ensure there is always a trailing slash
160 if rootstr[-1] != '/':
162 rootlist.append(rootstr.replace('//', '/'))
163 config['serverwebroot'] = rootlist
168 def find_sdk_tools_cmd(cmd):
169 '''find a working path to a tool from the Android SDK'''
172 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
173 # try to find a working path to this command, in all the recent possible paths
174 if 'build_tools' in config:
175 build_tools = os.path.join(config['sdk_path'], 'build-tools')
176 # if 'build_tools' was manually set and exists, check only that one
177 configed_build_tools = os.path.join(build_tools, config['build_tools'])
178 if os.path.exists(configed_build_tools):
179 tooldirs.append(configed_build_tools)
181 # no configed version, so hunt known paths for it
182 for f in sorted(os.listdir(build_tools), reverse=True):
183 if os.path.isdir(os.path.join(build_tools, f)):
184 tooldirs.append(os.path.join(build_tools, f))
185 tooldirs.append(build_tools)
186 sdk_tools = os.path.join(config['sdk_path'], 'tools')
187 if os.path.exists(sdk_tools):
188 tooldirs.append(sdk_tools)
189 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
190 if os.path.exists(sdk_platform_tools):
191 tooldirs.append(sdk_platform_tools)
192 tooldirs.append('/usr/bin')
194 if os.path.isfile(os.path.join(d, cmd)):
195 return os.path.join(d, cmd)
196 # did not find the command, exit with error message
197 ensure_build_tools_exists(config)
200 def test_sdk_exists(thisconfig):
201 if 'sdk_path' not in thisconfig:
202 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
205 logging.error("'sdk_path' not set in config.py!")
207 if thisconfig['sdk_path'] == default_config['sdk_path']:
208 logging.error('No Android SDK found!')
209 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
210 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
212 if not os.path.exists(thisconfig['sdk_path']):
213 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
215 if not os.path.isdir(thisconfig['sdk_path']):
216 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
218 for d in ['build-tools', 'platform-tools', 'tools']:
219 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
220 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
221 thisconfig['sdk_path'], d))
226 def ensure_build_tools_exists(thisconfig):
227 if not test_sdk_exists(thisconfig):
229 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
230 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
231 if not os.path.isdir(versioned_build_tools):
232 logging.critical('Android Build Tools path "'
233 + versioned_build_tools + '" does not exist!')
237 def write_password_file(pwtype, password=None):
239 writes out passwords to a protected file instead of passing passwords as
240 command line argments
242 filename = '.fdroid.' + pwtype + '.txt'
243 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
245 os.write(fd, config[pwtype])
247 os.write(fd, password)
249 config[pwtype + 'file'] = filename
252 # Given the arguments in the form of multiple appid:[vc] strings, this returns
253 # a dictionary with the set of vercodes specified for each package.
254 def read_pkg_args(args, allow_vercodes=False):
261 if allow_vercodes and ':' in p:
262 package, vercode = p.split(':')
264 package, vercode = p, None
265 if package not in vercodes:
266 vercodes[package] = [vercode] if vercode else []
268 elif vercode and vercode not in vercodes[package]:
269 vercodes[package] += [vercode] if vercode else []
274 # On top of what read_pkg_args does, this returns the whole app metadata, but
275 # limiting the builds list to the builds matching the vercodes specified.
276 def read_app_args(args, allapps, allow_vercodes=False):
278 vercodes = read_pkg_args(args, allow_vercodes)
284 for appid, app in allapps.iteritems():
285 if appid in vercodes:
288 if len(apps) != len(vercodes):
291 logging.critical("No such package: %s" % p)
292 raise FDroidException("Found invalid app ids in arguments")
294 raise FDroidException("No packages specified")
297 for appid, app in apps.iteritems():
301 app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
302 if len(app['builds']) != len(vercodes[appid]):
304 allvcs = [b['vercode'] for b in app['builds']]
305 for v in vercodes[appid]:
307 logging.critical("No such vercode %s for app %s" % (v, appid))
310 raise FDroidException("Found invalid vercodes for some apps")
315 def has_extension(filename, extension):
316 name, ext = os.path.splitext(filename)
317 ext = ext.lower()[1:]
318 return ext == extension
323 def clean_description(description):
324 'Remove unneeded newlines and spaces from a block of description text'
326 # this is split up by paragraph to make removing the newlines easier
327 for paragraph in re.split(r'\n\n', description):
328 paragraph = re.sub('\r', '', paragraph)
329 paragraph = re.sub('\n', ' ', paragraph)
330 paragraph = re.sub(' {2,}', ' ', paragraph)
331 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
332 returnstring += paragraph + '\n\n'
333 return returnstring.rstrip('\n')
336 def apknameinfo(filename):
338 filename = os.path.basename(filename)
339 if apk_regex is None:
340 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
341 m = apk_regex.match(filename)
343 result = (m.group(1), m.group(2))
344 except AttributeError:
345 raise FDroidException("Invalid apk name: %s" % filename)
349 def getapkname(app, build):
350 return "%s_%s.apk" % (app['id'], build['vercode'])
353 def getsrcname(app, build):
354 return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
361 return app['Auto Name']
366 return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
369 def getvcs(vcstype, remote, local):
371 return vcs_git(remote, local)
372 if vcstype == 'git-svn':
373 return vcs_gitsvn(remote, local)
375 return vcs_hg(remote, local)
377 return vcs_bzr(remote, local)
378 if vcstype == 'srclib':
379 if local != os.path.join('build', 'srclib', remote):
380 raise VCSException("Error: srclib paths are hard-coded!")
381 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
383 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
384 raise VCSException("Invalid vcs type " + vcstype)
387 def getsrclibvcs(name):
388 if name not in metadata.srclibs:
389 raise VCSException("Missing srclib " + name)
390 return metadata.srclibs[name]['Repo Type']
394 def __init__(self, remote, local):
396 # svn, git-svn and bzr may require auth
398 if self.repotype() in ('git-svn', 'bzr'):
400 self.username, remote = remote.split('@')
401 if ':' not in self.username:
402 raise VCSException("Password required with username")
403 self.username, self.password = self.username.split(':')
407 self.clone_failed = False
408 self.refreshed = False
414 # Take the local repository to a clean version of the given revision, which
415 # is specificed in the VCS's native format. Beforehand, the repository can
416 # be dirty, or even non-existent. If the repository does already exist
417 # locally, it will be updated from the origin, but only once in the
418 # lifetime of the vcs object.
419 # None is acceptable for 'rev' if you know you are cloning a clean copy of
420 # the repo - otherwise it must specify a valid revision.
421 def gotorevision(self, rev):
423 if self.clone_failed:
424 raise VCSException("Downloading the repository already failed once, not trying again.")
426 # The .fdroidvcs-id file for a repo tells us what VCS type
427 # and remote that directory was created from, allowing us to drop it
428 # automatically if either of those things changes.
429 fdpath = os.path.join(self.local, '..',
430 '.fdroidvcs-' + os.path.basename(self.local))
431 cdata = self.repotype() + ' ' + self.remote
434 if os.path.exists(self.local):
435 if os.path.exists(fdpath):
436 with open(fdpath, 'r') as f:
437 fsdata = f.read().strip()
443 "Repository details for %s changed - deleting" % (
447 logging.info("Repository details for %s missing - deleting" % (
450 shutil.rmtree(self.local)
455 self.gotorevisionx(rev)
456 except FDroidException, e:
459 # If necessary, write the .fdroidvcs file.
460 if writeback and not self.clone_failed:
461 with open(fdpath, 'w') as f:
467 # Derived classes need to implement this. It's called once basic checking
468 # has been performend.
469 def gotorevisionx(self, rev):
470 raise VCSException("This VCS type doesn't define gotorevisionx")
472 # Initialise and update submodules
473 def initsubmodules(self):
474 raise VCSException('Submodules not supported for this vcs type')
476 # Get a list of all known tags
478 raise VCSException('gettags not supported for this vcs type')
480 # Get a list of latest number tags
481 def latesttags(self, number):
482 raise VCSException('latesttags not supported for this vcs type')
484 # Get current commit reference (hash, revision, etc)
486 raise VCSException('getref not supported for this vcs type')
488 # Returns the srclib (name, path) used in setting up the current
499 # If the local directory exists, but is somehow not a git repository, git
500 # will traverse up the directory tree until it finds one that is (i.e.
501 # fdroidserver) and then we'll proceed to destroy it! This is called as
504 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
505 result = p.output.rstrip()
506 if not result.endswith(self.local):
507 raise VCSException('Repository mismatch')
509 def gotorevisionx(self, rev):
510 if not os.path.exists(self.local):
512 p = FDroidPopen(['git', 'clone', self.remote, self.local])
513 if p.returncode != 0:
514 self.clone_failed = True
515 raise VCSException("Git clone failed", p.output)
519 # Discard any working tree changes
520 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
521 if p.returncode != 0:
522 raise VCSException("Git reset failed", p.output)
523 # Remove untracked files now, in case they're tracked in the target
524 # revision (it happens!)
525 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
526 if p.returncode != 0:
527 raise VCSException("Git clean failed", p.output)
528 if not self.refreshed:
529 # Get latest commits and tags from remote
530 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
531 if p.returncode != 0:
532 raise VCSException("Git fetch failed", p.output)
533 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
534 if p.returncode != 0:
535 raise VCSException("Git fetch failed", p.output)
536 # Recreate origin/HEAD as git clone would do it, in case it disappeared
537 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
538 if p.returncode != 0:
539 lines = p.output.splitlines()
540 if 'Multiple remote HEAD branches' not in lines[0]:
541 raise VCSException("Git remote set-head failed", p.output)
542 branch = lines[1].split(' ')[-1]
543 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
544 if p2.returncode != 0:
545 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
546 self.refreshed = True
547 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
548 # a github repo. Most of the time this is the same as origin/master.
549 rev = rev or 'origin/HEAD'
550 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
551 if p.returncode != 0:
552 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
553 # Get rid of any uncontrolled files left behind
554 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
555 if p.returncode != 0:
556 raise VCSException("Git clean failed", p.output)
558 def initsubmodules(self):
560 submfile = os.path.join(self.local, '.gitmodules')
561 if not os.path.isfile(submfile):
562 raise VCSException("No git submodules available")
564 # fix submodules not accessible without an account and public key auth
565 with open(submfile, 'r') as f:
566 lines = f.readlines()
567 with open(submfile, 'w') as f:
569 if 'git@github.com' in line:
570 line = line.replace('git@github.com:', 'https://github.com/')
574 ['git', 'reset', '--hard'],
575 ['git', 'clean', '-dffx'],
577 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive'] + cmd, cwd=self.local, output=False)
578 if p.returncode != 0:
579 raise VCSException("Git submodule reset failed", p.output)
580 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
581 if p.returncode != 0:
582 raise VCSException("Git submodule sync failed", p.output)
583 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
584 if p.returncode != 0:
585 raise VCSException("Git submodule update failed", p.output)
589 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
590 return p.output.splitlines()
592 def latesttags(self, alltags, number):
594 p = FDroidPopen(['echo "' + '\n'.join(alltags) + '" | '
595 + 'xargs -I@ git log --format=format:"%at @%n" -1 @ | '
596 + 'sort -n | awk \'{print $2}\''],
597 cwd=self.local, shell=True, output=False)
598 return p.output.splitlines()[-number:]
601 class vcs_gitsvn(vcs):
606 # Damn git-svn tries to use a graphical password prompt, so we have to
607 # trick it into taking the password from stdin
609 if self.username is None:
611 return ('echo "%s" | DISPLAY="" ' % self.password, ' --username "%s"' % self.username)
613 # If the local directory exists, but is somehow not a git repository, git
614 # will traverse up the directory tree until it finds one that is (i.e.
615 # fdroidserver) and then we'll proceed to destory it! This is called as
618 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
619 result = p.output.rstrip()
620 if not result.endswith(self.local):
621 raise VCSException('Repository mismatch')
623 def gotorevisionx(self, rev):
624 if not os.path.exists(self.local):
626 gitsvn_cmd = '%sgit svn clone%s' % self.userargs()
627 if ';' in self.remote:
628 remote_split = self.remote.split(';')
629 for i in remote_split[1:]:
630 if i.startswith('trunk='):
631 gitsvn_cmd += ' -T %s' % i[6:]
632 elif i.startswith('tags='):
633 gitsvn_cmd += ' -t %s' % i[5:]
634 elif i.startswith('branches='):
635 gitsvn_cmd += ' -b %s' % i[9:]
636 p = FDroidPopen([gitsvn_cmd + " %s %s" % (remote_split[0], self.local)], shell=True, output=False)
637 if p.returncode != 0:
638 self.clone_failed = True
639 raise VCSException("Git svn clone failed", p.output)
641 p = FDroidPopen([gitsvn_cmd + " %s %s" % (self.remote, self.local)], shell=True, output=False)
642 if p.returncode != 0:
643 self.clone_failed = True
644 raise VCSException("Git svn clone failed", p.output)
648 # Discard any working tree changes
649 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
650 if p.returncode != 0:
651 raise VCSException("Git reset failed", p.output)
652 # Remove untracked files now, in case they're tracked in the target
653 # revision (it happens!)
654 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
655 if p.returncode != 0:
656 raise VCSException("Git clean failed", p.output)
657 if not self.refreshed:
658 # Get new commits, branches and tags from repo
659 p = FDroidPopen(['%sgit svn fetch %s' % self.userargs()], cwd=self.local, shell=True, output=False)
660 if p.returncode != 0:
661 raise VCSException("Git svn fetch failed")
662 p = FDroidPopen(['%sgit svn rebase %s' % self.userargs()], cwd=self.local, shell=True, output=False)
663 if p.returncode != 0:
664 raise VCSException("Git svn rebase failed", p.output)
665 self.refreshed = True
667 rev = rev or 'master'
669 nospaces_rev = rev.replace(' ', '%20')
670 # Try finding a svn tag
671 for treeish in ['origin/', '']:
672 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
673 if p.returncode == 0:
675 if p.returncode != 0:
676 # No tag found, normal svn rev translation
677 # Translate svn rev into git format
678 rev_split = rev.split('/')
681 for treeish in ['origin/', '']:
682 if len(rev_split) > 1:
683 treeish += rev_split[0]
684 svn_rev = rev_split[1]
687 # if no branch is specified, then assume trunk (i.e. 'master' branch):
691 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
693 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
694 git_rev = p.output.rstrip()
696 if p.returncode == 0 and git_rev:
699 if p.returncode != 0 or not git_rev:
700 # Try a plain git checkout as a last resort
701 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
702 if p.returncode != 0:
703 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
705 # Check out the git rev equivalent to the svn rev
706 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
707 if p.returncode != 0:
708 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
710 # Get rid of any uncontrolled files left behind
711 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
712 if p.returncode != 0:
713 raise VCSException("Git clean failed", p.output)
717 for treeish in ['origin/', '']:
718 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
724 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
725 if p.returncode != 0:
727 return p.output.strip()
735 def gotorevisionx(self, rev):
736 if not os.path.exists(self.local):
737 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
738 if p.returncode != 0:
739 self.clone_failed = True
740 raise VCSException("Hg clone failed", p.output)
742 p = FDroidPopen(['hg status -uS | xargs rm -rf'], cwd=self.local, shell=True, output=False)
743 if p.returncode != 0:
744 raise VCSException("Hg clean failed", p.output)
745 if not self.refreshed:
746 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
747 if p.returncode != 0:
748 raise VCSException("Hg pull failed", p.output)
749 self.refreshed = True
751 rev = rev or 'default'
754 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
755 if p.returncode != 0:
756 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
757 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
758 # Also delete untracked files, we have to enable purge extension for that:
759 if "'purge' is provided by the following extension" in p.output:
760 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
761 myfile.write("\n[extensions]\nhgext.purge=\n")
762 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
763 if p.returncode != 0:
764 raise VCSException("HG purge failed", p.output)
765 elif p.returncode != 0:
766 raise VCSException("HG purge failed", p.output)
769 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
770 return p.output.splitlines()[1:]
778 def gotorevisionx(self, rev):
779 if not os.path.exists(self.local):
780 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
781 if p.returncode != 0:
782 self.clone_failed = True
783 raise VCSException("Bzr branch failed", p.output)
785 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
786 if p.returncode != 0:
787 raise VCSException("Bzr revert failed", p.output)
788 if not self.refreshed:
789 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
790 if p.returncode != 0:
791 raise VCSException("Bzr update failed", p.output)
792 self.refreshed = True
794 revargs = list(['-r', rev] if rev else [])
795 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
796 if p.returncode != 0:
797 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
800 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
801 return [tag.split(' ')[0].strip() for tag in
802 p.output.splitlines()]
805 def retrieve_string(app_dir, string, xmlfiles=None):
808 os.path.join(app_dir, 'res'),
809 os.path.join(app_dir, 'src', 'main'),
814 for res_dir in res_dirs:
815 for r, d, f in os.walk(res_dir):
816 if os.path.basename(r) == 'values':
817 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
820 if string.startswith('@string/'):
821 string_search = re.compile(r'.*name="' + string[8:] + '".*?>"?([^<]+?)"?<.*').search
822 elif string.startswith('&') and string.endswith(';'):
823 string_search = re.compile(r'.*<!ENTITY.*' + string[1:-1] + '.*?"([^"]+?)".*>').search
825 if string_search is not None:
826 for xmlfile in xmlfiles:
827 for line in file(xmlfile):
828 matches = string_search(line)
830 return retrieve_string(app_dir, matches.group(1), xmlfiles)
833 return string.replace("\\'", "'")
836 # Return list of existing files that will be used to find the highest vercode
837 def manifest_paths(app_dir, flavours):
839 possible_manifests = \
840 [os.path.join(app_dir, 'AndroidManifest.xml'),
841 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
842 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
843 os.path.join(app_dir, 'build.gradle')]
845 for flavour in flavours:
848 possible_manifests.append(
849 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
851 return [path for path in possible_manifests if os.path.isfile(path)]
854 # Retrieve the package name. Returns the name, or None if not found.
855 def fetch_real_name(app_dir, flavours):
856 app_search = re.compile(r'.*<application.*').search
857 name_search = re.compile(r'.*android:label="([^"]+)".*').search
859 for f in manifest_paths(app_dir, flavours):
860 if not has_extension(f, 'xml'):
862 logging.debug("fetch_real_name: Checking manifest at " + f)
868 matches = name_search(line)
870 stringname = matches.group(1)
871 logging.debug("fetch_real_name: using string " + stringname)
872 result = retrieve_string(app_dir, stringname)
874 result = result.strip()
879 # Retrieve the version name
880 def version_name(original, app_dir, flavours):
881 for f in manifest_paths(app_dir, flavours):
882 if not has_extension(f, 'xml'):
884 string = retrieve_string(app_dir, original)
890 def get_library_references(root_dir):
892 proppath = os.path.join(root_dir, 'project.properties')
893 if not os.path.isfile(proppath):
895 with open(proppath) as f:
896 for line in f.readlines():
897 if not line.startswith('android.library.reference.'):
899 path = line.split('=')[1].strip()
900 relpath = os.path.join(root_dir, path)
901 if not os.path.isdir(relpath):
903 logging.debug("Found subproject at %s" % path)
904 libraries.append(path)
908 def ant_subprojects(root_dir):
909 subprojects = get_library_references(root_dir)
910 for subpath in subprojects:
911 subrelpath = os.path.join(root_dir, subpath)
912 for p in get_library_references(subrelpath):
913 relp = os.path.normpath(os.path.join(subpath, p))
914 if relp not in subprojects:
915 subprojects.insert(0, relp)
919 def remove_debuggable_flags(root_dir):
920 # Remove forced debuggable flags
921 logging.debug("Removing debuggable flags from %s" % root_dir)
922 for root, dirs, files in os.walk(root_dir):
923 if 'AndroidManifest.xml' in files:
924 path = os.path.join(root, 'AndroidManifest.xml')
925 p = FDroidPopen(['sed', '-i', 's/android:debuggable="[^"]*"//g', path], output=False)
926 if p.returncode != 0:
927 raise BuildException("Failed to remove debuggable flags of %s" % path)
930 # Extract some information from the AndroidManifest.xml at the given path.
931 # Returns (version, vercode, package), any or all of which might be None.
932 # All values returned are strings.
933 def parse_androidmanifests(paths, ignoreversions=None):
936 return (None, None, None)
938 vcsearch = re.compile(r'.*:versionCode="([0-9]+?)".*').search
939 vnsearch = re.compile(r'.*:versionName="([^"]+?)".*').search
940 psearch = re.compile(r'.*package="([^"]+)".*').search
942 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
943 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
944 psearch_g = re.compile(r'.*packageName *=* *["\']([^"]+)["\'].*').search
946 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
954 gradle = has_extension(path, 'gradle')
957 # Remember package name, may be defined separately from version+vercode
958 package = max_package
960 for line in file(path):
963 matches = psearch_g(line)
965 matches = psearch(line)
967 package = matches.group(1)
970 matches = vnsearch_g(line)
972 matches = vnsearch(line)
974 version = matches.group(2 if gradle else 1)
977 matches = vcsearch_g(line)
979 matches = vcsearch(line)
981 vercode = matches.group(1)
983 # Always grab the package name and version name in case they are not
984 # together with the highest version code
985 if max_package is None and package is not None:
986 max_package = package
987 if max_version is None and version is not None:
988 max_version = version
990 if max_vercode is None or (vercode is not None and vercode > max_vercode):
991 if not ignoresearch or not ignoresearch(version):
992 if version is not None:
993 max_version = version
994 if vercode is not None:
995 max_vercode = vercode
996 if package is not None:
997 max_package = package
999 max_version = "Ignore"
1001 if max_version is None:
1002 max_version = "Unknown"
1004 return (max_version, max_vercode, max_package)
1007 class FDroidException(Exception):
1008 def __init__(self, value, detail=None):
1010 self.detail = detail
1012 def get_wikitext(self):
1013 ret = repr(self.value) + "\n"
1017 txt = self.detail[-8192:] if len(self.detail) > 8192 else self.detail
1025 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1029 class VCSException(FDroidException):
1033 class BuildException(FDroidException):
1037 # Get the specified source library.
1038 # Returns the path to it. Normally this is the path to be used when referencing
1039 # it, which may be a subdirectory of the actual project. If you want the base
1040 # directory of the project, pass 'basepath=True'.
1041 def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
1042 basepath=False, raw=False, prepare=True, preponly=False):
1050 name, ref = spec.split('@')
1052 number, name = name.split(':', 1)
1054 name, subdir = name.split('/', 1)
1056 if name not in metadata.srclibs:
1057 raise VCSException('srclib ' + name + ' not found.')
1059 srclib = metadata.srclibs[name]
1061 sdir = os.path.join(srclib_dir, name)
1064 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1065 vcs.srclib = (name, number, sdir)
1067 vcs.gotorevision(ref)
1074 libdir = os.path.join(sdir, subdir)
1075 elif srclib["Subdir"]:
1076 for subdir in srclib["Subdir"]:
1077 libdir_candidate = os.path.join(sdir, subdir)
1078 if os.path.exists(libdir_candidate):
1079 libdir = libdir_candidate
1085 if srclib["Srclibs"]:
1087 for lib in srclib["Srclibs"].replace(';', ',').split(','):
1089 for t in srclibpaths:
1094 raise VCSException('Missing recursive srclib %s for %s' % (
1096 place_srclib(libdir, n, s_tuple[2])
1099 remove_signing_keys(sdir)
1100 remove_debuggable_flags(sdir)
1104 if srclib["Prepare"]:
1105 cmd = replace_config_vars(srclib["Prepare"])
1107 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1108 if p.returncode != 0:
1109 raise BuildException("Error running prepare command for srclib %s"
1115 return (name, number, libdir)
1118 # Prepare the source code for a particular build
1119 # 'vcs' - the appropriate vcs object for the application
1120 # 'app' - the application details from the metadata
1121 # 'build' - the build details from the metadata
1122 # 'build_dir' - the path to the build directory, usually
1124 # 'srclib_dir' - the path to the source libraries directory, usually
1126 # 'extlib_dir' - the path to the external libraries directory, usually
1128 # Returns the (root, srclibpaths) where:
1129 # 'root' is the root directory, which may be the same as 'build_dir' or may
1130 # be a subdirectory of it.
1131 # 'srclibpaths' is information on the srclibs being used
1132 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False):
1134 # Optionally, the actual app source can be in a subdirectory
1136 root_dir = os.path.join(build_dir, build['subdir'])
1138 root_dir = build_dir
1140 # Get a working copy of the right revision
1141 logging.info("Getting source for revision " + build['commit'])
1142 vcs.gotorevision(build['commit'])
1144 # Initialise submodules if requred
1145 if build['submodules']:
1146 logging.info("Initialising submodules")
1147 vcs.initsubmodules()
1149 # Check that a subdir (if we're using one) exists. This has to happen
1150 # after the checkout, since it might not exist elsewhere
1151 if not os.path.exists(root_dir):
1152 raise BuildException('Missing subdir ' + root_dir)
1154 # Run an init command if one is required
1156 cmd = replace_config_vars(build['init'])
1157 logging.info("Running 'init' commands in %s" % root_dir)
1159 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1160 if p.returncode != 0:
1161 raise BuildException("Error running init command for %s:%s" %
1162 (app['id'], build['version']), p.output)
1164 # Apply patches if any
1166 logging.info("Applying patches")
1167 for patch in build['patch']:
1168 patch = patch.strip()
1169 logging.info("Applying " + patch)
1170 patch_path = os.path.join('metadata', app['id'], patch)
1171 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1172 if p.returncode != 0:
1173 raise BuildException("Failed to apply patch %s" % patch_path)
1175 # Get required source libraries
1177 if build['srclibs']:
1178 logging.info("Collecting source libraries")
1179 for lib in build['srclibs']:
1180 srclibpaths.append(getsrclib(lib, srclib_dir, srclibpaths,
1183 for name, number, libpath in srclibpaths:
1184 place_srclib(root_dir, int(number) if number else None, libpath)
1186 basesrclib = vcs.getsrclib()
1187 # If one was used for the main source, add that too.
1189 srclibpaths.append(basesrclib)
1191 # Update the local.properties file
1192 localprops = [os.path.join(build_dir, 'local.properties')]
1194 localprops += [os.path.join(root_dir, 'local.properties')]
1195 for path in localprops:
1197 if os.path.isfile(path):
1198 logging.info("Updating local.properties file at %s" % path)
1204 logging.info("Creating local.properties file at %s" % path)
1205 # Fix old-fashioned 'sdk-location' by copying
1206 # from sdk.dir, if necessary
1207 if build['oldsdkloc']:
1208 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1209 re.S | re.M).group(1)
1210 props += "sdk-location=%s\n" % sdkloc
1212 props += "sdk.dir=%s\n" % config['sdk_path']
1213 props += "sdk-location=%s\n" % config['sdk_path']
1214 if config['ndk_path']:
1216 props += "ndk.dir=%s\n" % config['ndk_path']
1217 props += "ndk-location=%s\n" % config['ndk_path']
1218 # Add java.encoding if necessary
1219 if build['encoding']:
1220 props += "java.encoding=%s\n" % build['encoding']
1226 if build['type'] == 'gradle':
1227 flavours = build['gradle']
1229 version_regex = re.compile(r".*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1230 gradlepluginver = None
1232 gradle_files = [os.path.join(root_dir, 'build.gradle')]
1234 # Parent dir build.gradle
1235 parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
1236 if parent_dir.startswith(build_dir):
1237 gradle_files.append(os.path.join(parent_dir, 'build.gradle'))
1239 for path in gradle_files:
1242 if not os.path.isfile(path):
1244 with open(path) as f:
1246 match = version_regex.match(line)
1248 gradlepluginver = match.group(1)
1252 build['gradlepluginver'] = LooseVersion(gradlepluginver)
1254 logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
1255 build['gradlepluginver'] = LooseVersion('0.11')
1258 n = build["target"].split('-')[1]
1259 FDroidPopen(['sed', '-i',
1260 's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g',
1261 'build.gradle'], cwd=root_dir, output=False)
1263 # Remove forced debuggable flags
1264 remove_debuggable_flags(root_dir)
1266 # Insert version code and number into the manifest if necessary
1267 if build['forceversion']:
1268 logging.info("Changing the version name")
1269 for path in manifest_paths(root_dir, flavours):
1270 if not os.path.isfile(path):
1272 if has_extension(path, 'xml'):
1273 p = FDroidPopen(['sed', '-i',
1274 's/android:versionName="[^"]*"/android:versionName="' + build['version'] + '"/g',
1275 path], output=False)
1276 if p.returncode != 0:
1277 raise BuildException("Failed to amend manifest")
1278 elif has_extension(path, 'gradle'):
1279 p = FDroidPopen(['sed', '-i',
1280 's/versionName *=* *"[^"]*"/versionName = "' + build['version'] + '"/g',
1281 path], output=False)
1282 if p.returncode != 0:
1283 raise BuildException("Failed to amend build.gradle")
1284 if build['forcevercode']:
1285 logging.info("Changing the version code")
1286 for path in manifest_paths(root_dir, flavours):
1287 if not os.path.isfile(path):
1289 if has_extension(path, 'xml'):
1290 p = FDroidPopen(['sed', '-i',
1291 's/android:versionCode="[^"]*"/android:versionCode="' + build['vercode'] + '"/g',
1292 path], output=False)
1293 if p.returncode != 0:
1294 raise BuildException("Failed to amend manifest")
1295 elif has_extension(path, 'gradle'):
1296 p = FDroidPopen(['sed', '-i',
1297 's/versionCode *=* *[0-9]*/versionCode = ' + build['vercode'] + '/g',
1298 path], output=False)
1299 if p.returncode != 0:
1300 raise BuildException("Failed to amend build.gradle")
1302 # Delete unwanted files
1304 logging.info("Removing specified files")
1305 for part in getpaths(build_dir, build, 'rm'):
1306 dest = os.path.join(build_dir, part)
1307 logging.info("Removing {0}".format(part))
1308 if os.path.lexists(dest):
1309 if os.path.islink(dest):
1310 FDroidPopen(['unlink ' + dest], shell=True, output=False)
1312 FDroidPopen(['rm -rf ' + dest], shell=True, output=False)
1314 logging.info("...but it didn't exist")
1316 remove_signing_keys(build_dir)
1318 # Add required external libraries
1319 if build['extlibs']:
1320 logging.info("Collecting prebuilt libraries")
1321 libsdir = os.path.join(root_dir, 'libs')
1322 if not os.path.exists(libsdir):
1324 for lib in build['extlibs']:
1326 logging.info("...installing extlib {0}".format(lib))
1327 libf = os.path.basename(lib)
1328 libsrc = os.path.join(extlib_dir, lib)
1329 if not os.path.exists(libsrc):
1330 raise BuildException("Missing extlib file {0}".format(libsrc))
1331 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1333 # Run a pre-build command if one is required
1334 if build['prebuild']:
1335 logging.info("Running 'prebuild' commands in %s" % root_dir)
1337 cmd = replace_config_vars(build['prebuild'])
1339 # Substitute source library paths into prebuild commands
1340 for name, number, libpath in srclibpaths:
1341 libpath = os.path.relpath(libpath, root_dir)
1342 cmd = cmd.replace('$$' + name + '$$', libpath)
1344 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1345 if p.returncode != 0:
1346 raise BuildException("Error running prebuild command for %s:%s" %
1347 (app['id'], build['version']), p.output)
1349 # Generate (or update) the ant build file, build.xml...
1350 if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
1351 parms = ['android', 'update', 'lib-project']
1352 lparms = ['android', 'update', 'project']
1355 parms += ['-t', build['target']]
1356 lparms += ['-t', build['target']]
1357 if build['update'] == ['auto']:
1358 update_dirs = ant_subprojects(root_dir) + ['.']
1360 update_dirs = build['update']
1362 for d in update_dirs:
1363 subdir = os.path.join(root_dir, d)
1365 logging.debug("Updating main project")
1366 cmd = parms + ['-p', d]
1368 logging.debug("Updating subproject %s" % d)
1369 cmd = lparms + ['-p', d]
1370 p = SdkToolsPopen(cmd, cwd=root_dir)
1371 # Check to see whether an error was returned without a proper exit
1372 # code (this is the case for the 'no target set or target invalid'
1374 if p.returncode != 0 or p.output.startswith("Error: "):
1375 raise BuildException("Failed to update project at %s" % d, p.output)
1376 # Clean update dirs via ant
1378 logging.info("Cleaning subproject %s" % d)
1379 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1381 return (root_dir, srclibpaths)
1384 # Split and extend via globbing the paths from a field
1385 def getpaths(build_dir, build, field):
1387 for p in build[field]:
1389 full_path = os.path.join(build_dir, p)
1390 full_path = os.path.normpath(full_path)
1391 paths += [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1395 # Scan the source code in the given directory (and all subdirectories)
1396 # and return the number of fatal problems encountered
1397 def scan_source(build_dir, root_dir, thisbuild):
1401 # Common known non-free blobs (always lower case):
1403 re.compile(r'flurryagent', re.IGNORECASE),
1404 re.compile(r'paypal.*mpl', re.IGNORECASE),
1405 re.compile(r'google.*analytics', re.IGNORECASE),
1406 re.compile(r'admob.*sdk.*android', re.IGNORECASE),
1407 re.compile(r'google.*ad.*view', re.IGNORECASE),
1408 re.compile(r'google.*admob', re.IGNORECASE),
1409 re.compile(r'google.*play.*services', re.IGNORECASE),
1410 re.compile(r'crittercism', re.IGNORECASE),
1411 re.compile(r'heyzap', re.IGNORECASE),
1412 re.compile(r'jpct.*ae', re.IGNORECASE),
1413 re.compile(r'youtube.*android.*player.*api', re.IGNORECASE),
1414 re.compile(r'bugsense', re.IGNORECASE),
1415 re.compile(r'crashlytics', re.IGNORECASE),
1416 re.compile(r'ouya.*sdk', re.IGNORECASE),
1417 re.compile(r'libspen23', re.IGNORECASE),
1420 scanignore = getpaths(build_dir, thisbuild, 'scanignore')
1421 scandelete = getpaths(build_dir, thisbuild, 'scandelete')
1424 ms = magic.open(magic.MIME_TYPE)
1426 except AttributeError:
1430 for i in scanignore:
1431 if fd.startswith(i):
1436 for i in scandelete:
1437 if fd.startswith(i):
1441 def removeproblem(what, fd, fp):
1442 logging.info('Removing %s at %s' % (what, fd))
1445 def warnproblem(what, fd):
1446 logging.warn('Found %s at %s' % (what, fd))
1448 def handleproblem(what, fd, fp):
1450 logging.info('Ignoring %s at %s' % (what, fd))
1452 removeproblem(what, fd, fp)
1454 logging.error('Found %s at %s' % (what, fd))
1458 # Iterate through all files in the source code
1459 for r, d, f in os.walk(build_dir, topdown=True):
1461 # It's topdown, so checking the basename is enough
1462 for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
1468 # Path (relative) to the file
1469 fp = os.path.join(r, curfile)
1470 fd = fp[len(build_dir) + 1:]
1473 mime = magic.from_file(fp, mime=True) if ms is None else ms.file(fp)
1474 except UnicodeError:
1475 warnproblem('malformed magic number', fd)
1477 if mime == 'application/x-sharedlib':
1478 count += handleproblem('shared library', fd, fp)
1480 elif mime == 'application/x-archive':
1481 count += handleproblem('static library', fd, fp)
1483 elif mime == 'application/x-executable':
1484 count += handleproblem('binary executable', fd, fp)
1486 elif mime == 'application/x-java-applet':
1487 count += handleproblem('Java compiled class', fd, fp)
1492 'application/java-archive',
1493 'application/octet-stream',
1497 if has_extension(fp, 'apk'):
1498 removeproblem('APK file', fd, fp)
1500 elif has_extension(fp, 'jar'):
1502 if any(suspect.match(curfile) for suspect in usual_suspects):
1503 count += handleproblem('usual supect', fd, fp)
1505 warnproblem('JAR file', fd)
1507 elif has_extension(fp, 'zip'):
1508 warnproblem('ZIP file', fd)
1511 warnproblem('unknown compressed or binary file', fd)
1513 elif has_extension(fp, 'java'):
1514 for line in file(fp):
1515 if 'DexClassLoader' in line:
1516 count += handleproblem('DexClassLoader', fd, fp)
1521 # Presence of a jni directory without buildjni=yes might
1522 # indicate a problem (if it's not a problem, explicitly use
1523 # buildjni=no to bypass this check)
1524 if (os.path.exists(os.path.join(root_dir, 'jni')) and
1525 not thisbuild['buildjni']):
1526 logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
1535 self.path = os.path.join('stats', 'known_apks.txt')
1537 if os.path.exists(self.path):
1538 for line in file(self.path):
1539 t = line.rstrip().split(' ')
1541 self.apks[t[0]] = (t[1], None)
1543 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1544 self.changed = False
1546 def writeifchanged(self):
1548 if not os.path.exists('stats'):
1550 f = open(self.path, 'w')
1552 for apk, app in self.apks.iteritems():
1554 line = apk + ' ' + appid
1556 line += ' ' + time.strftime('%Y-%m-%d', added)
1558 for line in sorted(lst):
1559 f.write(line + '\n')
1562 # Record an apk (if it's new, otherwise does nothing)
1563 # Returns the date it was added.
1564 def recordapk(self, apk, app):
1565 if apk not in self.apks:
1566 self.apks[apk] = (app, time.gmtime(time.time()))
1568 _, added = self.apks[apk]
1571 # Look up information - given the 'apkname', returns (app id, date added/None).
1572 # Or returns None for an unknown apk.
1573 def getapp(self, apkname):
1574 if apkname in self.apks:
1575 return self.apks[apkname]
1578 # Get the most recent 'num' apps added to the repo, as a list of package ids
1579 # with the most recent first.
1580 def getlatest(self, num):
1582 for apk, app in self.apks.iteritems():
1586 if apps[appid] > added:
1590 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1591 lst = [app for app, _ in sortedapps]
1596 def isApkDebuggable(apkfile, config):
1597 """Returns True if the given apk file is debuggable
1599 :param apkfile: full path to the apk to check"""
1601 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'])
1602 if p.returncode != 0:
1603 logging.critical("Failed to get apk manifest information")
1605 for line in p.output.splitlines():
1606 if 'android:debuggable' in line and not line.endswith('0x0'):
1611 class AsynchronousFileReader(threading.Thread):
1613 Helper class to implement asynchronous reading of a file
1614 in a separate thread. Pushes read lines on a queue to
1615 be consumed in another thread.
1618 def __init__(self, fd, queue):
1619 assert isinstance(queue, Queue.Queue)
1620 assert callable(fd.readline)
1621 threading.Thread.__init__(self)
1626 '''The body of the tread: read lines and put them on the queue.'''
1627 for line in iter(self._fd.readline, ''):
1628 self._queue.put(line)
1631 '''Check whether there is no more content to expect.'''
1632 return not self.is_alive() and self._queue.empty()
1640 def SdkToolsPopen(commands, cwd=None, shell=False, output=True):
1642 if cmd not in config:
1643 config[cmd] = find_sdk_tools_cmd(commands[0])
1644 return FDroidPopen([config[cmd]] + commands[1:],
1645 cwd=cwd, shell=shell, output=output)
1648 def FDroidPopen(commands, cwd=None, shell=False, output=True):
1650 Run a command and capture the possibly huge output.
1652 :param commands: command and argument list like in subprocess.Popen
1653 :param cwd: optionally specifies a working directory
1654 :returns: A PopenResult.
1660 cwd = os.path.normpath(cwd)
1661 logging.debug("Directory: %s" % cwd)
1662 logging.debug("> %s" % ' '.join(commands))
1664 result = PopenResult()
1667 p = subprocess.Popen(commands, cwd=cwd, shell=shell, env=env,
1668 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1670 raise BuildException("OSError while trying to execute " + ' '.join(commands) + ': ' + str(e))
1672 stdout_queue = Queue.Queue()
1673 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1674 stdout_reader.start()
1676 # Check the queue for output (until there is no more to get)
1677 while not stdout_reader.eof():
1678 while not stdout_queue.empty():
1679 line = stdout_queue.get()
1680 if output and options.verbose:
1681 # Output directly to console
1682 sys.stderr.write(line)
1684 result.output += line
1688 result.returncode = p.wait()
1692 def remove_signing_keys(build_dir):
1693 comment = re.compile(r'[ ]*//')
1694 signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1696 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1697 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1698 re.compile(r'.*variant\.outputFile = .*'),
1699 re.compile(r'.*output\.outputFile = .*'),
1700 re.compile(r'.*\.readLine\(.*'),
1702 for root, dirs, files in os.walk(build_dir):
1703 if 'build.gradle' in files:
1704 path = os.path.join(root, 'build.gradle')
1706 with open(path, "r") as o:
1707 lines = o.readlines()
1712 with open(path, "w") as o:
1714 if comment.match(line):
1718 opened += line.count('{')
1719 opened -= line.count('}')
1722 if signing_configs.match(line):
1727 if any(s.match(line) for s in line_matches):
1735 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1738 'project.properties',
1740 'default.properties',
1743 if propfile in files:
1744 path = os.path.join(root, propfile)
1746 with open(path, "r") as o:
1747 lines = o.readlines()
1751 with open(path, "w") as o:
1753 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1760 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1763 def replace_config_vars(cmd):
1764 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1765 cmd = cmd.replace('$$NDK$$', config['ndk_path'])
1766 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1770 def place_srclib(root_dir, number, libpath):
1773 relpath = os.path.relpath(libpath, root_dir)
1774 proppath = os.path.join(root_dir, 'project.properties')
1777 if os.path.isfile(proppath):
1778 with open(proppath, "r") as o:
1779 lines = o.readlines()
1781 with open(proppath, "w") as o:
1784 if line.startswith('android.library.reference.%d=' % number):
1785 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1790 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1793 def compare_apks(apk1, apk2, tmp_dir):
1796 Returns None if the apk content is the same (apart from the signing key),
1797 otherwise a string describing what's different, or what went wrong when
1798 trying to do the comparison.
1801 thisdir = os.path.join(tmp_dir, 'this_apk')
1802 thatdir = os.path.join(tmp_dir, 'that_apk')
1803 for d in [thisdir, thatdir]:
1804 if os.path.exists(d):
1808 if subprocess.call(['jar', 'xf',
1809 os.path.abspath(apk1)],
1811 return("Failed to unpack " + apk1)
1812 if subprocess.call(['jar', 'xf',
1813 os.path.abspath(apk2)],
1815 return("Failed to unpack " + apk2)
1817 p = FDroidPopen(['diff', '-r', 'this_apk', 'that_apk'], cwd=tmp_dir,
1819 lines = p.output.splitlines()
1820 if len(lines) != 1 or 'META-INF' not in lines[0]:
1821 return("Unexpected diff output - " + p.output)
1823 # If we get here, it seems like they're the same!