1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
33 from distutils.version import LooseVersion
42 def get_default_config():
44 'sdk_path': os.getenv("ANDROID_HOME") or "",
45 'ndk_path': os.getenv("ANDROID_NDK") or "",
46 'build_tools': "20.0.0",
50 'sync_from_local_copy_dir': False,
51 'update_stats': False,
52 'stats_to_carbon': False,
54 'build_server_always': False,
55 'keystore': os.path.join(os.getenv("HOME"), '.local', 'share', 'fdroidserver', 'keystore.jks'),
56 'smartcardoptions': [],
62 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
63 'repo_name': "My First FDroid Repo Demo",
64 'repo_icon': "fdroid-icon.png",
65 'repo_description': '''
66 This is a repository of apps to be used with FDroid. Applications in this
67 repository are either official binaries built by the original application
68 developers, or are binaries built from source by the admin of f-droid.org
69 using the tools on https://gitlab.com/u/fdroid.
75 def read_config(opts, config_file='config.py'):
76 """Read the repository config
78 The config is read from config_file, which is in the current directory when
79 any of the repo management commands are used.
81 global config, options, env
83 if config is not None:
85 if not os.path.isfile(config_file):
86 logging.critical("Missing config file - is this a repo directory?")
93 logging.debug("Reading %s" % config_file)
94 execfile(config_file, config)
96 # smartcardoptions must be a list since its command line args for Popen
97 if 'smartcardoptions' in config:
98 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
99 elif 'keystore' in config and config['keystore'] == 'NONE':
100 # keystore='NONE' means use smartcard, these are required defaults
101 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
102 'SunPKCS11-OpenSC', '-providerClass',
103 'sun.security.pkcs11.SunPKCS11',
104 '-providerArg', 'opensc-fdroid.cfg']
106 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
107 st = os.stat(config_file)
108 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
109 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
111 defconfig = get_default_config()
112 for k, v in defconfig.items():
116 # Expand environment variables
117 for k, v in config.items():
120 v = os.path.expanduser(v)
121 config[k] = os.path.expandvars(v)
123 if not test_sdk_exists(config):
126 if not test_build_tools_exists(config):
131 os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
134 os.path.join(config['sdk_path'], 'tools', 'zipalign'),
135 os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'zipalign'),
138 os.path.join(config['sdk_path'], 'tools', 'android'),
141 os.path.join(config['sdk_path'], 'platform-tools', 'adb'),
145 for b, paths in bin_paths.items():
148 if os.path.isfile(path):
151 if config[b] is None:
152 logging.warn("Could not find %s in any of the following paths:\n%s" % (
153 b, '\n'.join(paths)))
155 # There is no standard, so just set up the most common environment
158 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
159 env[n] = config['sdk_path']
160 for n in ['ANDROID_NDK', 'NDK']:
161 env[n] = config['ndk_path']
163 for k in ["keystorepass", "keypass"]:
165 write_password_file(k)
167 for k in ["repo_description", "archive_description"]:
169 config[k] = clean_description(config[k])
171 if 'serverwebroot' in config:
172 if isinstance(config['serverwebroot'], basestring):
173 roots = [config['serverwebroot']]
174 elif all(isinstance(item, basestring) for item in config['serverwebroot']):
175 roots = config['serverwebroot']
177 raise TypeError('only accepts strings, lists, and tuples')
179 for rootstr in roots:
180 # since this is used with rsync, where trailing slashes have
181 # meaning, ensure there is always a trailing slash
182 if rootstr[-1] != '/':
184 rootlist.append(rootstr.replace('//', '/'))
185 config['serverwebroot'] = rootlist
190 def test_sdk_exists(c):
191 if c['sdk_path'] is None:
192 # c['sdk_path'] is set to the value of ANDROID_HOME by default
193 logging.error('No Android SDK found! ANDROID_HOME is not set and sdk_path is not in config.py!')
194 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
195 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
197 if not os.path.exists(c['sdk_path']):
198 logging.critical('Android SDK path "' + c['sdk_path'] + '" does not exist!')
200 if not os.path.isdir(c['sdk_path']):
201 logging.critical('Android SDK path "' + c['sdk_path'] + '" is not a directory!')
203 for d in ['build-tools', 'platform-tools', 'tools']:
204 if not os.path.isdir(os.path.join(c['sdk_path'], d)):
205 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
211 def test_build_tools_exists(c):
212 if not test_sdk_exists(c):
214 build_tools = os.path.join(c['sdk_path'], 'build-tools')
215 versioned_build_tools = os.path.join(build_tools, c['build_tools'])
216 if not os.path.isdir(versioned_build_tools):
217 logging.critical('Android Build Tools path "'
218 + versioned_build_tools + '" does not exist!')
223 def write_password_file(pwtype, password=None):
225 writes out passwords to a protected file instead of passing passwords as
226 command line argments
228 filename = '.fdroid.' + pwtype + '.txt'
229 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
231 os.write(fd, config[pwtype])
233 os.write(fd, password)
235 config[pwtype + 'file'] = filename
238 # Given the arguments in the form of multiple appid:[vc] strings, this returns
239 # a dictionary with the set of vercodes specified for each package.
240 def read_pkg_args(args, allow_vercodes=False):
247 if allow_vercodes and ':' in p:
248 package, vercode = p.split(':')
250 package, vercode = p, None
251 if package not in vercodes:
252 vercodes[package] = [vercode] if vercode else []
254 elif vercode and vercode not in vercodes[package]:
255 vercodes[package] += [vercode] if vercode else []
260 # On top of what read_pkg_args does, this returns the whole app metadata, but
261 # limiting the builds list to the builds matching the vercodes specified.
262 def read_app_args(args, allapps, allow_vercodes=False):
264 vercodes = read_pkg_args(args, allow_vercodes)
269 apps = [app for app in allapps if app['id'] in vercodes]
271 if len(apps) != len(vercodes):
272 allids = [app["id"] for app in allapps]
275 logging.critical("No such package: %s" % p)
276 raise FDroidException("Found invalid app ids in arguments")
278 raise FDroidException("No packages specified")
282 vc = vercodes[app['id']]
285 app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
286 if len(app['builds']) != len(vercodes[app['id']]):
288 allvcs = [b['vercode'] for b in app['builds']]
289 for v in vercodes[app['id']]:
291 logging.critical("No such vercode %s for app %s" % (v, app['id']))
294 raise FDroidException("Found invalid vercodes for some apps")
299 def has_extension(filename, extension):
300 name, ext = os.path.splitext(filename)
301 ext = ext.lower()[1:]
302 return ext == extension
307 def clean_description(description):
308 'Remove unneeded newlines and spaces from a block of description text'
310 # this is split up by paragraph to make removing the newlines easier
311 for paragraph in re.split(r'\n\n', description):
312 paragraph = re.sub('\r', '', paragraph)
313 paragraph = re.sub('\n', ' ', paragraph)
314 paragraph = re.sub(' {2,}', ' ', paragraph)
315 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
316 returnstring += paragraph + '\n\n'
317 return returnstring.rstrip('\n')
320 def apknameinfo(filename):
322 filename = os.path.basename(filename)
323 if apk_regex is None:
324 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
325 m = apk_regex.match(filename)
327 result = (m.group(1), m.group(2))
328 except AttributeError:
329 raise FDroidException("Invalid apk name: %s" % filename)
333 def getapkname(app, build):
334 return "%s_%s.apk" % (app['id'], build['vercode'])
337 def getsrcname(app, build):
338 return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
345 return app['Auto Name']
350 return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
353 def getvcs(vcstype, remote, local):
355 return vcs_git(remote, local)
357 return vcs_svn(remote, local)
358 if vcstype == 'git-svn':
359 return vcs_gitsvn(remote, local)
361 return vcs_hg(remote, local)
363 return vcs_bzr(remote, local)
364 if vcstype == 'srclib':
365 if local != os.path.join('build', 'srclib', remote):
366 raise VCSException("Error: srclib paths are hard-coded!")
367 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
368 raise VCSException("Invalid vcs type " + vcstype)
371 def getsrclibvcs(name):
372 if name not in metadata.srclibs:
373 raise VCSException("Missing srclib " + name)
374 return metadata.srclibs[name]['Repo Type']
378 def __init__(self, remote, local):
380 # svn, git-svn and bzr may require auth
382 if self.repotype() in ('svn', 'git-svn', 'bzr'):
384 self.username, remote = remote.split('@')
385 if ':' not in self.username:
386 raise VCSException("Password required with username")
387 self.username, self.password = self.username.split(':')
391 self.clone_failed = False
392 self.refreshed = False
398 # Take the local repository to a clean version of the given revision, which
399 # is specificed in the VCS's native format. Beforehand, the repository can
400 # be dirty, or even non-existent. If the repository does already exist
401 # locally, it will be updated from the origin, but only once in the
402 # lifetime of the vcs object.
403 # None is acceptable for 'rev' if you know you are cloning a clean copy of
404 # the repo - otherwise it must specify a valid revision.
405 def gotorevision(self, rev):
407 if self.clone_failed:
408 raise VCSException("Downloading the repository already failed once, not trying again.")
410 # The .fdroidvcs-id file for a repo tells us what VCS type
411 # and remote that directory was created from, allowing us to drop it
412 # automatically if either of those things changes.
413 fdpath = os.path.join(self.local, '..',
414 '.fdroidvcs-' + os.path.basename(self.local))
415 cdata = self.repotype() + ' ' + self.remote
418 if os.path.exists(self.local):
419 if os.path.exists(fdpath):
420 with open(fdpath, 'r') as f:
421 fsdata = f.read().strip()
427 "Repository details for %s changed - deleting" % (
431 logging.info("Repository details for %s missing - deleting" % (
434 shutil.rmtree(self.local)
439 self.gotorevisionx(rev)
440 except FDroidException, e:
443 # If necessary, write the .fdroidvcs file.
444 if writeback and not self.clone_failed:
445 with open(fdpath, 'w') as f:
451 # Derived classes need to implement this. It's called once basic checking
452 # has been performend.
453 def gotorevisionx(self, rev):
454 raise VCSException("This VCS type doesn't define gotorevisionx")
456 # Initialise and update submodules
457 def initsubmodules(self):
458 raise VCSException('Submodules not supported for this vcs type')
460 # Get a list of all known tags
462 raise VCSException('gettags not supported for this vcs type')
464 # Get a list of latest number tags
465 def latesttags(self, number):
466 raise VCSException('latesttags not supported for this vcs type')
468 # Get current commit reference (hash, revision, etc)
470 raise VCSException('getref not supported for this vcs type')
472 # Returns the srclib (name, path) used in setting up the current
483 # If the local directory exists, but is somehow not a git repository, git
484 # will traverse up the directory tree until it finds one that is (i.e.
485 # fdroidserver) and then we'll proceed to destroy it! This is called as
488 p = SilentPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
489 result = p.output.rstrip()
490 if not result.endswith(self.local):
491 raise VCSException('Repository mismatch')
493 def gotorevisionx(self, rev):
494 if not os.path.exists(self.local):
496 p = FDroidPopen(['git', 'clone', self.remote, self.local])
497 if p.returncode != 0:
498 self.clone_failed = True
499 raise VCSException("Git clone failed", p.output)
503 # Discard any working tree changes
504 p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
505 if p.returncode != 0:
506 raise VCSException("Git reset failed", p.output)
507 # Remove untracked files now, in case they're tracked in the target
508 # revision (it happens!)
509 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
510 if p.returncode != 0:
511 raise VCSException("Git clean failed", p.output)
512 if not self.refreshed:
513 # Get latest commits and tags from remote
514 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
515 if p.returncode != 0:
516 raise VCSException("Git fetch failed", p.output)
517 p = SilentPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local)
518 if p.returncode != 0:
519 raise VCSException("Git fetch failed", p.output)
520 # Recreate origin/HEAD as git clone would do it, in case it disappeared
521 p = SilentPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local)
522 if p.returncode != 0:
523 lines = p.output.splitlines()
524 if 'Multiple remote HEAD branches' not in lines[0]:
525 raise VCSException("Git remote set-head failed", p.output)
526 branch = lines[1].split(' ')[-1]
527 p2 = SilentPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local)
528 if p2.returncode != 0:
529 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
530 self.refreshed = True
531 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
532 # a github repo. Most of the time this is the same as origin/master.
533 rev = rev or 'origin/HEAD'
534 p = SilentPopen(['git', 'checkout', '-f', rev], cwd=self.local)
535 if p.returncode != 0:
536 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
537 # Get rid of any uncontrolled files left behind
538 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
539 if p.returncode != 0:
540 raise VCSException("Git clean failed", p.output)
542 def initsubmodules(self):
544 submfile = os.path.join(self.local, '.gitmodules')
545 if not os.path.isfile(submfile):
546 raise VCSException("No git submodules available")
548 # fix submodules not accessible without an account and public key auth
549 with open(submfile, 'r') as f:
550 lines = f.readlines()
551 with open(submfile, 'w') as f:
553 if 'git@github.com' in line:
554 line = line.replace('git@github.com:', 'https://github.com/')
558 ['git', 'reset', '--hard'],
559 ['git', 'clean', '-dffx'],
561 p = SilentPopen(['git', 'submodule', 'foreach', '--recursive'] + cmd, cwd=self.local)
562 if p.returncode != 0:
563 raise VCSException("Git submodule reset failed", p.output)
564 p = SilentPopen(['git', 'submodule', 'sync'], cwd=self.local)
565 if p.returncode != 0:
566 raise VCSException("Git submodule sync failed", p.output)
567 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
568 if p.returncode != 0:
569 raise VCSException("Git submodule update failed", p.output)
573 p = SilentPopen(['git', 'tag'], cwd=self.local)
574 return p.output.splitlines()
576 def latesttags(self, alltags, number):
578 p = SilentPopen(['echo "' + '\n'.join(alltags) + '" | '
579 + 'xargs -I@ git log --format=format:"%at @%n" -1 @ | '
580 + 'sort -n | awk \'{print $2}\''],
581 cwd=self.local, shell=True)
582 return p.output.splitlines()[-number:]
585 class vcs_gitsvn(vcs):
590 # Damn git-svn tries to use a graphical password prompt, so we have to
591 # trick it into taking the password from stdin
593 if self.username is None:
595 return ('echo "%s" | DISPLAY="" ' % self.password, ' --username "%s"' % self.username)
597 # If the local directory exists, but is somehow not a git repository, git
598 # will traverse up the directory tree until it finds one that is (i.e.
599 # fdroidserver) and then we'll proceed to destory it! This is called as
602 p = SilentPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
603 result = p.output.rstrip()
604 if not result.endswith(self.local):
605 raise VCSException('Repository mismatch')
607 def gotorevisionx(self, rev):
608 if not os.path.exists(self.local):
610 gitsvn_cmd = '%sgit svn clone%s' % self.userargs()
611 if ';' in self.remote:
612 remote_split = self.remote.split(';')
613 for i in remote_split[1:]:
614 if i.startswith('trunk='):
615 gitsvn_cmd += ' -T %s' % i[6:]
616 elif i.startswith('tags='):
617 gitsvn_cmd += ' -t %s' % i[5:]
618 elif i.startswith('branches='):
619 gitsvn_cmd += ' -b %s' % i[9:]
620 p = SilentPopen([gitsvn_cmd + " %s %s" % (remote_split[0], self.local)], shell=True)
621 if p.returncode != 0:
622 self.clone_failed = True
623 raise VCSException("Git clone failed", p.output)
625 p = SilentPopen([gitsvn_cmd + " %s %s" % (self.remote, self.local)], shell=True)
626 if p.returncode != 0:
627 self.clone_failed = True
628 raise VCSException("Git clone failed", p.output)
632 # Discard any working tree changes
633 p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
634 if p.returncode != 0:
635 raise VCSException("Git reset failed", p.output)
636 # Remove untracked files now, in case they're tracked in the target
637 # revision (it happens!)
638 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
639 if p.returncode != 0:
640 raise VCSException("Git clean failed", p.output)
641 if not self.refreshed:
642 # Get new commits, branches and tags from repo
643 p = SilentPopen(['%sgit svn fetch %s' % self.userargs()], cwd=self.local, shell=True)
644 if p.returncode != 0:
645 raise VCSException("Git svn fetch failed")
646 p = SilentPopen(['%sgit svn rebase %s' % self.userargs()], cwd=self.local, shell=True)
647 if p.returncode != 0:
648 raise VCSException("Git svn rebase failed", p.output)
649 self.refreshed = True
651 rev = rev or 'master'
653 nospaces_rev = rev.replace(' ', '%20')
654 # Try finding a svn tag
655 for treeish in ['origin/', '']:
656 p = SilentPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev],
658 if p.returncode == 0:
660 if p.returncode != 0:
661 # No tag found, normal svn rev translation
662 # Translate svn rev into git format
663 rev_split = rev.split('/')
666 for treeish in ['origin/', '']:
667 if len(rev_split) > 1:
668 treeish += rev_split[0]
669 svn_rev = rev_split[1]
672 # if no branch is specified, then assume trunk (i.e. 'master' branch):
676 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
678 p = SilentPopen(['git', 'svn', 'find-rev', svn_rev, treeish],
680 git_rev = p.output.rstrip()
682 if p.returncode == 0 and git_rev:
685 if p.returncode != 0 or not git_rev:
686 # Try a plain git checkout as a last resort
687 p = SilentPopen(['git', 'checkout', rev], cwd=self.local)
688 if p.returncode != 0:
689 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
691 # Check out the git rev equivalent to the svn rev
692 p = SilentPopen(['git', 'checkout', git_rev], cwd=self.local)
693 if p.returncode != 0:
694 raise VCSException("Git svn checkout of '%s' failed" % rev, p.output)
696 # Get rid of any uncontrolled files left behind
697 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
698 if p.returncode != 0:
699 raise VCSException("Git clean failed", p.output)
703 for treeish in ['origin/', '']:
704 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
710 p = SilentPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local)
711 if p.returncode != 0:
713 return p.output.strip()
722 if self.username is None:
723 return ['--non-interactive']
724 return ['--username', self.username,
725 '--password', self.password,
728 def gotorevisionx(self, rev):
729 if not os.path.exists(self.local):
730 p = SilentPopen(['svn', 'checkout', self.remote, self.local] + self.userargs())
731 if p.returncode != 0:
732 self.clone_failed = True
733 raise VCSException("Svn checkout of '%s' failed" % rev, p.output)
737 r"svn status | awk '/\?/ {print $2}' | xargs rm -rf"):
738 p = SilentPopen([svncommand], cwd=self.local, shell=True)
739 if p.returncode != 0:
740 raise VCSException("Svn reset ({0}) failed in {1}".format(svncommand, self.local), p.output)
741 if not self.refreshed:
742 p = SilentPopen(['svn', 'update'] + self.userargs(), cwd=self.local)
743 if p.returncode != 0:
744 raise VCSException("Svn update failed", p.output)
745 self.refreshed = True
747 revargs = list(['-r', rev] if rev else [])
748 p = SilentPopen(['svn', 'update', '--force'] + revargs + self.userargs(), cwd=self.local)
749 if p.returncode != 0:
750 raise VCSException("Svn update failed", p.output)
753 p = SilentPopen(['svn', 'info'], cwd=self.local)
754 for line in p.output.splitlines():
755 if line and line.startswith('Last Changed Rev: '):
765 def gotorevisionx(self, rev):
766 if not os.path.exists(self.local):
767 p = SilentPopen(['hg', 'clone', self.remote, self.local])
768 if p.returncode != 0:
769 self.clone_failed = True
770 raise VCSException("Hg clone failed", p.output)
772 p = SilentPopen(['hg status -uS | xargs rm -rf'], cwd=self.local, shell=True)
773 if p.returncode != 0:
774 raise VCSException("Hg clean failed", p.output)
775 if not self.refreshed:
776 p = SilentPopen(['hg', 'pull'], cwd=self.local)
777 if p.returncode != 0:
778 raise VCSException("Hg pull failed", p.output)
779 self.refreshed = True
781 rev = rev or 'default'
784 p = SilentPopen(['hg', 'update', '-C', rev], cwd=self.local)
785 if p.returncode != 0:
786 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
787 p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
788 # Also delete untracked files, we have to enable purge extension for that:
789 if "'purge' is provided by the following extension" in p.output:
790 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
791 myfile.write("\n[extensions]\nhgext.purge=\n")
792 p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
793 if p.returncode != 0:
794 raise VCSException("HG purge failed", p.output)
795 elif p.returncode != 0:
796 raise VCSException("HG purge failed", p.output)
799 p = SilentPopen(['hg', 'tags', '-q'], cwd=self.local)
800 return p.output.splitlines()[1:]
808 def gotorevisionx(self, rev):
809 if not os.path.exists(self.local):
810 p = SilentPopen(['bzr', 'branch', self.remote, self.local])
811 if p.returncode != 0:
812 self.clone_failed = True
813 raise VCSException("Bzr branch failed", p.output)
815 p = SilentPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local)
816 if p.returncode != 0:
817 raise VCSException("Bzr revert failed", p.output)
818 if not self.refreshed:
819 p = SilentPopen(['bzr', 'pull'], cwd=self.local)
820 if p.returncode != 0:
821 raise VCSException("Bzr update failed", p.output)
822 self.refreshed = True
824 revargs = list(['-r', rev] if rev else [])
825 p = SilentPopen(['bzr', 'revert'] + revargs, cwd=self.local)
826 if p.returncode != 0:
827 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
830 p = SilentPopen(['bzr', 'tags'], cwd=self.local)
831 return [tag.split(' ')[0].strip() for tag in
832 p.output.splitlines()]
835 def retrieve_string(app_dir, string, xmlfiles=None):
838 os.path.join(app_dir, 'res'),
839 os.path.join(app_dir, 'src', 'main'),
844 for res_dir in res_dirs:
845 for r, d, f in os.walk(res_dir):
846 if os.path.basename(r) == 'values':
847 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
850 if string.startswith('@string/'):
851 string_search = re.compile(r'.*name="' + string[8:] + '".*?>([^<]+?)<.*').search
852 elif string.startswith('&') and string.endswith(';'):
853 string_search = re.compile(r'.*<!ENTITY.*' + string[1:-1] + '.*?"([^"]+?)".*>').search
855 if string_search is not None:
856 for xmlfile in xmlfiles:
857 for line in file(xmlfile):
858 matches = string_search(line)
860 return retrieve_string(app_dir, matches.group(1), xmlfiles)
863 return string.replace("\\'", "'")
866 # Return list of existing files that will be used to find the highest vercode
867 def manifest_paths(app_dir, flavour):
869 possible_manifests = \
870 [os.path.join(app_dir, 'AndroidManifest.xml'),
871 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
872 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
873 os.path.join(app_dir, 'build.gradle')]
876 possible_manifests.append(
877 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
879 return [path for path in possible_manifests if os.path.isfile(path)]
882 # Retrieve the package name. Returns the name, or None if not found.
883 def fetch_real_name(app_dir, flavour):
884 app_search = re.compile(r'.*<application.*').search
885 name_search = re.compile(r'.*android:label="([^"]+)".*').search
887 for f in manifest_paths(app_dir, flavour):
888 if not has_extension(f, 'xml'):
890 logging.debug("fetch_real_name: Checking manifest at " + f)
896 matches = name_search(line)
898 stringname = matches.group(1)
899 logging.debug("fetch_real_name: using string " + stringname)
900 result = retrieve_string(app_dir, stringname)
902 result = result.strip()
907 # Retrieve the version name
908 def version_name(original, app_dir, flavour):
909 for f in manifest_paths(app_dir, flavour):
910 if not has_extension(f, 'xml'):
912 string = retrieve_string(app_dir, original)
918 def get_library_references(root_dir):
920 proppath = os.path.join(root_dir, 'project.properties')
921 if not os.path.isfile(proppath):
923 with open(proppath) as f:
924 for line in f.readlines():
925 if not line.startswith('android.library.reference.'):
927 path = line.split('=')[1].strip()
928 relpath = os.path.join(root_dir, path)
929 if not os.path.isdir(relpath):
931 logging.debug("Found subproject at %s" % path)
932 libraries.append(path)
936 def ant_subprojects(root_dir):
937 subprojects = get_library_references(root_dir)
938 for subpath in subprojects:
939 subrelpath = os.path.join(root_dir, subpath)
940 for p in get_library_references(subrelpath):
941 relp = os.path.normpath(os.path.join(subpath, p))
942 if relp not in subprojects:
943 subprojects.insert(0, relp)
947 def remove_debuggable_flags(root_dir):
948 # Remove forced debuggable flags
949 logging.debug("Removing debuggable flags from %s" % root_dir)
950 for root, dirs, files in os.walk(root_dir):
951 if 'AndroidManifest.xml' in files:
952 path = os.path.join(root, 'AndroidManifest.xml')
953 p = SilentPopen(['sed', '-i', 's/android:debuggable="[^"]*"//g', path])
954 if p.returncode != 0:
955 raise BuildException("Failed to remove debuggable flags of %s" % path)
958 # Extract some information from the AndroidManifest.xml at the given path.
959 # Returns (version, vercode, package), any or all of which might be None.
960 # All values returned are strings.
961 def parse_androidmanifests(paths, ignoreversions=None):
964 return (None, None, None)
966 vcsearch = re.compile(r'.*:versionCode="([0-9]+?)".*').search
967 vnsearch = re.compile(r'.*:versionName="([^"]+?)".*').search
968 psearch = re.compile(r'.*package="([^"]+)".*').search
970 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
971 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
972 psearch_g = re.compile(r'.*packageName *=* *["\']([^"]+)["\'].*').search
974 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
982 gradle = has_extension(path, 'gradle')
985 # Remember package name, may be defined separately from version+vercode
986 package = max_package
988 for line in file(path):
991 matches = psearch_g(line)
993 matches = psearch(line)
995 package = matches.group(1)
998 matches = vnsearch_g(line)
1000 matches = vnsearch(line)
1002 version = matches.group(2 if gradle else 1)
1005 matches = vcsearch_g(line)
1007 matches = vcsearch(line)
1009 vercode = matches.group(1)
1011 # Always grab the package name and version name in case they are not
1012 # together with the highest version code
1013 if max_package is None and package is not None:
1014 max_package = package
1015 if max_version is None and version is not None:
1016 max_version = version
1018 if max_vercode is None or (vercode is not None and vercode > max_vercode):
1019 if not ignoresearch or not ignoresearch(version):
1020 if version is not None:
1021 max_version = version
1022 if vercode is not None:
1023 max_vercode = vercode
1024 if package is not None:
1025 max_package = package
1027 max_version = "Ignore"
1029 if max_version is None:
1030 max_version = "Unknown"
1032 return (max_version, max_vercode, max_package)
1035 class FDroidException(Exception):
1036 def __init__(self, value, detail=None):
1038 self.detail = detail
1040 def get_wikitext(self):
1041 ret = repr(self.value) + "\n"
1045 txt = self.detail[-8192:] if len(self.detail) > 8192 else self.detail
1053 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1057 class VCSException(FDroidException):
1061 class BuildException(FDroidException):
1065 # Get the specified source library.
1066 # Returns the path to it. Normally this is the path to be used when referencing
1067 # it, which may be a subdirectory of the actual project. If you want the base
1068 # directory of the project, pass 'basepath=True'.
1069 def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
1070 basepath=False, raw=False, prepare=True, preponly=False):
1078 name, ref = spec.split('@')
1080 number, name = name.split(':', 1)
1082 name, subdir = name.split('/', 1)
1084 if name not in metadata.srclibs:
1085 raise VCSException('srclib ' + name + ' not found.')
1087 srclib = metadata.srclibs[name]
1089 sdir = os.path.join(srclib_dir, name)
1092 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1093 vcs.srclib = (name, number, sdir)
1095 vcs.gotorevision(ref)
1102 libdir = os.path.join(sdir, subdir)
1103 elif srclib["Subdir"]:
1104 for subdir in srclib["Subdir"]:
1105 libdir_candidate = os.path.join(sdir, subdir)
1106 if os.path.exists(libdir_candidate):
1107 libdir = libdir_candidate
1113 if srclib["Srclibs"]:
1115 for lib in srclib["Srclibs"].replace(';', ',').split(','):
1117 for t in srclibpaths:
1122 raise VCSException('Missing recursive srclib %s for %s' % (
1124 place_srclib(libdir, n, s_tuple[2])
1127 remove_signing_keys(sdir)
1128 remove_debuggable_flags(sdir)
1132 if srclib["Prepare"]:
1133 cmd = replace_config_vars(srclib["Prepare"])
1135 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1136 if p.returncode != 0:
1137 raise BuildException("Error running prepare command for srclib %s"
1143 return (name, number, libdir)
1146 # Prepare the source code for a particular build
1147 # 'vcs' - the appropriate vcs object for the application
1148 # 'app' - the application details from the metadata
1149 # 'build' - the build details from the metadata
1150 # 'build_dir' - the path to the build directory, usually
1152 # 'srclib_dir' - the path to the source libraries directory, usually
1154 # 'extlib_dir' - the path to the external libraries directory, usually
1156 # Returns the (root, srclibpaths) where:
1157 # 'root' is the root directory, which may be the same as 'build_dir' or may
1158 # be a subdirectory of it.
1159 # 'srclibpaths' is information on the srclibs being used
1160 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False):
1162 # Optionally, the actual app source can be in a subdirectory
1164 root_dir = os.path.join(build_dir, build['subdir'])
1166 root_dir = build_dir
1168 # Get a working copy of the right revision
1169 logging.info("Getting source for revision " + build['commit'])
1170 vcs.gotorevision(build['commit'])
1172 # Initialise submodules if requred
1173 if build['submodules']:
1174 logging.info("Initialising submodules")
1175 vcs.initsubmodules()
1177 # Check that a subdir (if we're using one) exists. This has to happen
1178 # after the checkout, since it might not exist elsewhere
1179 if not os.path.exists(root_dir):
1180 raise BuildException('Missing subdir ' + root_dir)
1182 # Run an init command if one is required
1184 cmd = replace_config_vars(build['init'])
1185 logging.info("Running 'init' commands in %s" % root_dir)
1187 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1188 if p.returncode != 0:
1189 raise BuildException("Error running init command for %s:%s" %
1190 (app['id'], build['version']), p.output)
1192 # Apply patches if any
1194 logging.info("Applying patches")
1195 for patch in build['patch']:
1196 patch = patch.strip()
1197 logging.info("Applying " + patch)
1198 patch_path = os.path.join('metadata', app['id'], patch)
1199 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1200 if p.returncode != 0:
1201 raise BuildException("Failed to apply patch %s" % patch_path)
1203 # Get required source libraries
1205 if build['srclibs']:
1206 logging.info("Collecting source libraries")
1207 for lib in build['srclibs']:
1208 srclibpaths.append(getsrclib(lib, srclib_dir, srclibpaths,
1211 for name, number, libpath in srclibpaths:
1212 place_srclib(root_dir, int(number) if number else None, libpath)
1214 basesrclib = vcs.getsrclib()
1215 # If one was used for the main source, add that too.
1217 srclibpaths.append(basesrclib)
1219 # Update the local.properties file
1220 localprops = [os.path.join(build_dir, 'local.properties')]
1222 localprops += [os.path.join(root_dir, 'local.properties')]
1223 for path in localprops:
1224 if not os.path.isfile(path):
1226 logging.info("Updating properties file at %s" % path)
1231 # Fix old-fashioned 'sdk-location' by copying
1232 # from sdk.dir, if necessary
1233 if build['oldsdkloc']:
1234 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1235 re.S | re.M).group(1)
1236 props += "sdk-location=%s\n" % sdkloc
1238 props += "sdk.dir=%s\n" % config['sdk_path']
1239 props += "sdk-location=%s\n" % config['sdk_path']
1240 if 'ndk_path' in config:
1242 props += "ndk.dir=%s\n" % config['ndk_path']
1243 props += "ndk-location=%s\n" % config['ndk_path']
1244 # Add java.encoding if necessary
1245 if build['encoding']:
1246 props += "java.encoding=%s\n" % build['encoding']
1252 if build['type'] == 'gradle':
1253 flavour = build['gradle']
1254 if flavour in ['main', 'yes', '']:
1257 version_regex = re.compile(r".*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1258 gradlepluginver = None
1260 gradle_files = [os.path.join(root_dir, 'build.gradle')]
1262 # Parent dir build.gradle
1263 parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
1264 if parent_dir.startswith(build_dir):
1265 gradle_files.append(os.path.join(parent_dir, 'build.gradle'))
1267 for path in gradle_files:
1270 if not os.path.isfile(path):
1272 with open(path) as f:
1274 match = version_regex.match(line)
1276 gradlepluginver = match.group(1)
1280 build['gradlepluginver'] = LooseVersion(gradlepluginver)
1282 logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
1283 build['gradlepluginver'] = LooseVersion('0.11')
1286 n = build["target"].split('-')[1]
1287 SilentPopen(['sed', '-i',
1288 's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g',
1292 # Remove forced debuggable flags
1293 remove_debuggable_flags(root_dir)
1295 # Insert version code and number into the manifest if necessary
1296 if build['forceversion']:
1297 logging.info("Changing the version name")
1298 for path in manifest_paths(root_dir, flavour):
1299 if not os.path.isfile(path):
1301 if has_extension(path, 'xml'):
1302 p = SilentPopen(['sed', '-i',
1303 's/android:versionName="[^"]*"/android:versionName="'
1304 + build['version'] + '"/g',
1306 if p.returncode != 0:
1307 raise BuildException("Failed to amend manifest")
1308 elif has_extension(path, 'gradle'):
1309 p = SilentPopen(['sed', '-i',
1310 's/versionName *=* *"[^"]*"/versionName = "'
1311 + build['version'] + '"/g',
1313 if p.returncode != 0:
1314 raise BuildException("Failed to amend build.gradle")
1315 if build['forcevercode']:
1316 logging.info("Changing the version code")
1317 for path in manifest_paths(root_dir, flavour):
1318 if not os.path.isfile(path):
1320 if has_extension(path, 'xml'):
1321 p = SilentPopen(['sed', '-i',
1322 's/android:versionCode="[^"]*"/android:versionCode="'
1323 + build['vercode'] + '"/g',
1325 if p.returncode != 0:
1326 raise BuildException("Failed to amend manifest")
1327 elif has_extension(path, 'gradle'):
1328 p = SilentPopen(['sed', '-i',
1329 's/versionCode *=* *[0-9]*/versionCode = '
1330 + build['vercode'] + '/g',
1332 if p.returncode != 0:
1333 raise BuildException("Failed to amend build.gradle")
1335 # Delete unwanted files
1337 logging.info("Removing specified files")
1338 for part in getpaths(build_dir, build, 'rm'):
1339 dest = os.path.join(build_dir, part)
1340 logging.info("Removing {0}".format(part))
1341 if os.path.lexists(dest):
1342 if os.path.islink(dest):
1343 SilentPopen(['unlink ' + dest], shell=True)
1345 SilentPopen(['rm -rf ' + dest], shell=True)
1347 logging.info("...but it didn't exist")
1349 remove_signing_keys(build_dir)
1351 # Add required external libraries
1352 if build['extlibs']:
1353 logging.info("Collecting prebuilt libraries")
1354 libsdir = os.path.join(root_dir, 'libs')
1355 if not os.path.exists(libsdir):
1357 for lib in build['extlibs']:
1359 logging.info("...installing extlib {0}".format(lib))
1360 libf = os.path.basename(lib)
1361 libsrc = os.path.join(extlib_dir, lib)
1362 if not os.path.exists(libsrc):
1363 raise BuildException("Missing extlib file {0}".format(libsrc))
1364 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1366 # Run a pre-build command if one is required
1367 if build['prebuild']:
1368 logging.info("Running 'prebuild' commands in %s" % root_dir)
1370 cmd = replace_config_vars(build['prebuild'])
1372 # Substitute source library paths into prebuild commands
1373 for name, number, libpath in srclibpaths:
1374 libpath = os.path.relpath(libpath, root_dir)
1375 cmd = cmd.replace('$$' + name + '$$', libpath)
1377 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1378 if p.returncode != 0:
1379 raise BuildException("Error running prebuild command for %s:%s" %
1380 (app['id'], build['version']), p.output)
1382 # Generate (or update) the ant build file, build.xml...
1383 if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
1384 parms = [config['android'], 'update', 'lib-project']
1385 lparms = [config['android'], 'update', 'project']
1388 parms += ['-t', build['target']]
1389 lparms += ['-t', build['target']]
1390 if build['update'] == ['auto']:
1391 update_dirs = ant_subprojects(root_dir) + ['.']
1393 update_dirs = build['update']
1395 for d in update_dirs:
1396 subdir = os.path.join(root_dir, d)
1398 logging.debug("Updating main project")
1399 cmd = parms + ['-p', d]
1401 logging.debug("Updating subproject %s" % d)
1402 cmd = lparms + ['-p', d]
1403 p = FDroidPopen(cmd, cwd=root_dir)
1404 # Check to see whether an error was returned without a proper exit
1405 # code (this is the case for the 'no target set or target invalid'
1407 if p.returncode != 0 or p.output.startswith("Error: "):
1408 raise BuildException("Failed to update project at %s" % d, p.output)
1409 # Clean update dirs via ant
1411 logging.info("Cleaning subproject %s" % d)
1412 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1414 return (root_dir, srclibpaths)
1417 # Split and extend via globbing the paths from a field
1418 def getpaths(build_dir, build, field):
1420 for p in build[field]:
1422 full_path = os.path.join(build_dir, p)
1423 full_path = os.path.normpath(full_path)
1424 paths += [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1428 # Scan the source code in the given directory (and all subdirectories)
1429 # and return the number of fatal problems encountered
1430 def scan_source(build_dir, root_dir, thisbuild):
1434 # Common known non-free blobs (always lower case):
1436 re.compile(r'flurryagent', re.IGNORECASE),
1437 re.compile(r'paypal.*mpl', re.IGNORECASE),
1438 re.compile(r'google.*analytics', re.IGNORECASE),
1439 re.compile(r'admob.*sdk.*android', re.IGNORECASE),
1440 re.compile(r'google.*ad.*view', re.IGNORECASE),
1441 re.compile(r'google.*admob', re.IGNORECASE),
1442 re.compile(r'google.*play.*services', re.IGNORECASE),
1443 re.compile(r'crittercism', re.IGNORECASE),
1444 re.compile(r'heyzap', re.IGNORECASE),
1445 re.compile(r'jpct.*ae', re.IGNORECASE),
1446 re.compile(r'youtube.*android.*player.*api', re.IGNORECASE),
1447 re.compile(r'bugsense', re.IGNORECASE),
1448 re.compile(r'crashlytics', re.IGNORECASE),
1449 re.compile(r'ouya.*sdk', re.IGNORECASE),
1450 re.compile(r'libspen23', re.IGNORECASE),
1453 scanignore = getpaths(build_dir, thisbuild, 'scanignore')
1454 scandelete = getpaths(build_dir, thisbuild, 'scandelete')
1457 ms = magic.open(magic.MIME_TYPE)
1459 except AttributeError:
1463 for i in scanignore:
1464 if fd.startswith(i):
1469 for i in scandelete:
1470 if fd.startswith(i):
1474 def removeproblem(what, fd, fp):
1475 logging.info('Removing %s at %s' % (what, fd))
1478 def warnproblem(what, fd):
1479 logging.warn('Found %s at %s' % (what, fd))
1481 def handleproblem(what, fd, fp):
1483 removeproblem(what, fd, fp)
1485 logging.error('Found %s at %s' % (what, fd))
1489 # Iterate through all files in the source code
1490 for r, d, f in os.walk(build_dir, topdown=True):
1492 # It's topdown, so checking the basename is enough
1493 for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
1499 # Path (relative) to the file
1500 fp = os.path.join(r, curfile)
1501 fd = fp[len(build_dir) + 1:]
1503 # Check if this file has been explicitly excluded from scanning
1507 mime = magic.from_file(fp, mime=True) if ms is None else ms.file(fp)
1509 if mime == 'application/x-sharedlib':
1510 count += handleproblem('shared library', fd, fp)
1512 elif mime == 'application/x-archive':
1513 count += handleproblem('static library', fd, fp)
1515 elif mime == 'application/x-executable':
1516 count += handleproblem('binary executable', fd, fp)
1518 elif mime == 'application/x-java-applet':
1519 count += handleproblem('Java compiled class', fd, fp)
1524 'application/java-archive',
1525 'application/octet-stream',
1529 if has_extension(fp, 'apk'):
1530 removeproblem('APK file', fd, fp)
1532 elif has_extension(fp, 'jar'):
1534 if any(suspect.match(curfile) for suspect in usual_suspects):
1535 count += handleproblem('usual supect', fd, fp)
1537 warnproblem('JAR file', fd)
1539 elif has_extension(fp, 'zip'):
1540 warnproblem('ZIP file', fd)
1543 warnproblem('unknown compressed or binary file', fd)
1545 elif has_extension(fp, 'java'):
1546 for line in file(fp):
1547 if 'DexClassLoader' in line:
1548 count += handleproblem('DexClassLoader', fd, fp)
1553 # Presence of a jni directory without buildjni=yes might
1554 # indicate a problem (if it's not a problem, explicitly use
1555 # buildjni=no to bypass this check)
1556 if (os.path.exists(os.path.join(root_dir, 'jni')) and
1557 not thisbuild['buildjni']):
1558 logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
1567 self.path = os.path.join('stats', 'known_apks.txt')
1569 if os.path.exists(self.path):
1570 for line in file(self.path):
1571 t = line.rstrip().split(' ')
1573 self.apks[t[0]] = (t[1], None)
1575 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1576 self.changed = False
1578 def writeifchanged(self):
1580 if not os.path.exists('stats'):
1582 f = open(self.path, 'w')
1584 for apk, app in self.apks.iteritems():
1586 line = apk + ' ' + appid
1588 line += ' ' + time.strftime('%Y-%m-%d', added)
1590 for line in sorted(lst):
1591 f.write(line + '\n')
1594 # Record an apk (if it's new, otherwise does nothing)
1595 # Returns the date it was added.
1596 def recordapk(self, apk, app):
1597 if apk not in self.apks:
1598 self.apks[apk] = (app, time.gmtime(time.time()))
1600 _, added = self.apks[apk]
1603 # Look up information - given the 'apkname', returns (app id, date added/None).
1604 # Or returns None for an unknown apk.
1605 def getapp(self, apkname):
1606 if apkname in self.apks:
1607 return self.apks[apkname]
1610 # Get the most recent 'num' apps added to the repo, as a list of package ids
1611 # with the most recent first.
1612 def getlatest(self, num):
1614 for apk, app in self.apks.iteritems():
1618 if apps[appid] > added:
1622 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1623 lst = [app for app, _ in sortedapps]
1628 def isApkDebuggable(apkfile, config):
1629 """Returns True if the given apk file is debuggable
1631 :param apkfile: full path to the apk to check"""
1633 p = SilentPopen([os.path.join(config['sdk_path'], 'build-tools',
1634 config['build_tools'], 'aapt'),
1635 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'])
1636 if p.returncode != 0:
1637 logging.critical("Failed to get apk manifest information")
1639 for line in p.output.splitlines():
1640 if 'android:debuggable' in line and not line.endswith('0x0'):
1645 class AsynchronousFileReader(threading.Thread):
1647 Helper class to implement asynchronous reading of a file
1648 in a separate thread. Pushes read lines on a queue to
1649 be consumed in another thread.
1652 def __init__(self, fd, queue):
1653 assert isinstance(queue, Queue.Queue)
1654 assert callable(fd.readline)
1655 threading.Thread.__init__(self)
1660 '''The body of the tread: read lines and put them on the queue.'''
1661 for line in iter(self._fd.readline, ''):
1662 self._queue.put(line)
1665 '''Check whether there is no more content to expect.'''
1666 return not self.is_alive() and self._queue.empty()
1674 def SilentPopen(commands, cwd=None, shell=False):
1675 return FDroidPopen(commands, cwd=cwd, shell=shell, output=False)
1678 def FDroidPopen(commands, cwd=None, shell=False, output=True):
1680 Run a command and capture the possibly huge output.
1682 :param commands: command and argument list like in subprocess.Popen
1683 :param cwd: optionally specifies a working directory
1684 :returns: A PopenResult.
1690 cwd = os.path.normpath(cwd)
1691 logging.debug("Directory: %s" % cwd)
1692 logging.debug("> %s" % ' '.join(commands))
1694 result = PopenResult()
1695 p = subprocess.Popen(commands, cwd=cwd, shell=shell, env=env,
1696 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1698 stdout_queue = Queue.Queue()
1699 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1700 stdout_reader.start()
1702 # Check the queue for output (until there is no more to get)
1703 while not stdout_reader.eof():
1704 while not stdout_queue.empty():
1705 line = stdout_queue.get()
1706 if output and options.verbose:
1707 # Output directly to console
1708 sys.stderr.write(line)
1710 result.output += line
1715 result.returncode = p.returncode
1719 def remove_signing_keys(build_dir):
1720 comment = re.compile(r'[ ]*//')
1721 signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1723 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1724 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1725 re.compile(r'.*variant\.outputFile = .*'),
1726 re.compile(r'.*\.readLine\(.*'),
1728 for root, dirs, files in os.walk(build_dir):
1729 if 'build.gradle' in files:
1730 path = os.path.join(root, 'build.gradle')
1732 with open(path, "r") as o:
1733 lines = o.readlines()
1738 with open(path, "w") as o:
1740 if comment.match(line):
1744 opened += line.count('{')
1745 opened -= line.count('}')
1748 if signing_configs.match(line):
1753 if any(s.match(line) for s in line_matches):
1761 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1764 'project.properties',
1766 'default.properties',
1769 if propfile in files:
1770 path = os.path.join(root, propfile)
1772 with open(path, "r") as o:
1773 lines = o.readlines()
1777 with open(path, "w") as o:
1779 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1786 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1789 def replace_config_vars(cmd):
1790 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1791 cmd = cmd.replace('$$NDK$$', config['ndk_path'])
1792 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1796 def place_srclib(root_dir, number, libpath):
1799 relpath = os.path.relpath(libpath, root_dir)
1800 proppath = os.path.join(root_dir, 'project.properties')
1803 if os.path.isfile(proppath):
1804 with open(proppath, "r") as o:
1805 lines = o.readlines()
1807 with open(proppath, "w") as o:
1810 if line.startswith('android.library.reference.%d=' % number):
1811 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1816 o.write('android.library.reference.%d=%s\n' % (number, relpath))