1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
33 from distutils.version import LooseVersion
43 'sdk_path': "$ANDROID_HOME",
44 'ndk_path': "$ANDROID_NDK",
45 'build_tools': "20.0.0",
49 'sync_from_local_copy_dir': False,
50 'update_stats': False,
54 'stats_to_carbon': False,
56 'build_server_always': False,
57 'keystore': os.path.join("$HOME", '.local', 'share', 'fdroidserver', 'keystore.jks'),
58 'smartcardoptions': [],
64 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
65 'repo_name': "My First FDroid Repo Demo",
66 'repo_icon': "fdroid-icon.png",
67 'repo_description': '''
68 This is a repository of apps to be used with FDroid. Applications in this
69 repository are either official binaries built by the original application
70 developers, or are binaries built from source by the admin of f-droid.org
71 using the tools on https://gitlab.com/u/fdroid.
77 def fill_config_defaults(config):
78 for k, v in default_config.items():
82 # Expand paths (~users and $vars)
83 for k in ['sdk_path', 'ndk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
86 v = os.path.expanduser(v)
87 v = os.path.expandvars(v)
90 config[k + '_orig'] = orig
93 def read_config(opts, config_file='config.py'):
94 """Read the repository config
96 The config is read from config_file, which is in the current directory when
97 any of the repo management commands are used.
99 global config, options, env
101 if config is not None:
103 if not os.path.isfile(config_file):
104 logging.critical("Missing config file - is this a repo directory?")
111 logging.debug("Reading %s" % config_file)
112 execfile(config_file, config)
114 # smartcardoptions must be a list since its command line args for Popen
115 if 'smartcardoptions' in config:
116 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
117 elif 'keystore' in config and config['keystore'] == 'NONE':
118 # keystore='NONE' means use smartcard, these are required defaults
119 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
120 'SunPKCS11-OpenSC', '-providerClass',
121 'sun.security.pkcs11.SunPKCS11',
122 '-providerArg', 'opensc-fdroid.cfg']
124 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
125 st = os.stat(config_file)
126 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
127 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
129 fill_config_defaults(config)
131 if not test_sdk_exists(config):
134 if not test_build_tools_exists(config):
139 os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
142 os.path.join(config['sdk_path'], 'tools', 'zipalign'),
143 os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'zipalign'),
146 os.path.join(config['sdk_path'], 'tools', 'android'),
149 os.path.join(config['sdk_path'], 'platform-tools', 'adb'),
153 for b, paths in bin_paths.items():
156 if os.path.isfile(path):
159 if config[b] is None:
160 logging.warn("Could not find %s in any of the following paths:\n%s" % (
161 b, '\n'.join(paths)))
163 # There is no standard, so just set up the most common environment
166 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
167 env[n] = config['sdk_path']
168 for n in ['ANDROID_NDK', 'NDK']:
169 env[n] = config['ndk_path']
171 for k in ["keystorepass", "keypass"]:
173 write_password_file(k)
175 for k in ["repo_description", "archive_description"]:
177 config[k] = clean_description(config[k])
179 if 'serverwebroot' in config:
180 if isinstance(config['serverwebroot'], basestring):
181 roots = [config['serverwebroot']]
182 elif all(isinstance(item, basestring) for item in config['serverwebroot']):
183 roots = config['serverwebroot']
185 raise TypeError('only accepts strings, lists, and tuples')
187 for rootstr in roots:
188 # since this is used with rsync, where trailing slashes have
189 # meaning, ensure there is always a trailing slash
190 if rootstr[-1] != '/':
192 rootlist.append(rootstr.replace('//', '/'))
193 config['serverwebroot'] = rootlist
198 def test_sdk_exists(config):
199 if config['sdk_path'] == default_config['sdk_path']:
200 logging.error('No Android SDK found!')
201 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
202 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
204 if not os.path.exists(config['sdk_path']):
205 logging.critical('Android SDK path "' + config['sdk_path'] + '" does not exist!')
207 if not os.path.isdir(config['sdk_path']):
208 logging.critical('Android SDK path "' + config['sdk_path'] + '" is not a directory!')
210 for d in ['build-tools', 'platform-tools', 'tools']:
211 if not os.path.isdir(os.path.join(config['sdk_path'], d)):
212 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
213 config['sdk_path'], d))
218 def test_build_tools_exists(config):
219 if not test_sdk_exists(config):
221 build_tools = os.path.join(config['sdk_path'], 'build-tools')
222 versioned_build_tools = os.path.join(build_tools, config['build_tools'])
223 if not os.path.isdir(versioned_build_tools):
224 logging.critical('Android Build Tools path "'
225 + versioned_build_tools + '" does not exist!')
230 def write_password_file(pwtype, password=None):
232 writes out passwords to a protected file instead of passing passwords as
233 command line argments
235 filename = '.fdroid.' + pwtype + '.txt'
236 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
238 os.write(fd, config[pwtype])
240 os.write(fd, password)
242 config[pwtype + 'file'] = filename
245 # Given the arguments in the form of multiple appid:[vc] strings, this returns
246 # a dictionary with the set of vercodes specified for each package.
247 def read_pkg_args(args, allow_vercodes=False):
254 if allow_vercodes and ':' in p:
255 package, vercode = p.split(':')
257 package, vercode = p, None
258 if package not in vercodes:
259 vercodes[package] = [vercode] if vercode else []
261 elif vercode and vercode not in vercodes[package]:
262 vercodes[package] += [vercode] if vercode else []
267 # On top of what read_pkg_args does, this returns the whole app metadata, but
268 # limiting the builds list to the builds matching the vercodes specified.
269 def read_app_args(args, allapps, allow_vercodes=False):
271 vercodes = read_pkg_args(args, allow_vercodes)
277 for appid, app in allapps.iteritems():
278 if appid in vercodes:
281 if len(apps) != len(vercodes):
284 logging.critical("No such package: %s" % p)
285 raise FDroidException("Found invalid app ids in arguments")
287 raise FDroidException("No packages specified")
290 for appid, app in apps.iteritems():
294 app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
295 if len(app['builds']) != len(vercodes[appid]):
297 allvcs = [b['vercode'] for b in app['builds']]
298 for v in vercodes[appid]:
300 logging.critical("No such vercode %s for app %s" % (v, appid))
303 raise FDroidException("Found invalid vercodes for some apps")
308 def has_extension(filename, extension):
309 name, ext = os.path.splitext(filename)
310 ext = ext.lower()[1:]
311 return ext == extension
316 def clean_description(description):
317 'Remove unneeded newlines and spaces from a block of description text'
319 # this is split up by paragraph to make removing the newlines easier
320 for paragraph in re.split(r'\n\n', description):
321 paragraph = re.sub('\r', '', paragraph)
322 paragraph = re.sub('\n', ' ', paragraph)
323 paragraph = re.sub(' {2,}', ' ', paragraph)
324 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
325 returnstring += paragraph + '\n\n'
326 return returnstring.rstrip('\n')
329 def apknameinfo(filename):
331 filename = os.path.basename(filename)
332 if apk_regex is None:
333 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
334 m = apk_regex.match(filename)
336 result = (m.group(1), m.group(2))
337 except AttributeError:
338 raise FDroidException("Invalid apk name: %s" % filename)
342 def getapkname(app, build):
343 return "%s_%s.apk" % (app['id'], build['vercode'])
346 def getsrcname(app, build):
347 return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
354 return app['Auto Name']
359 return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
362 def getvcs(vcstype, remote, local):
364 return vcs_git(remote, local)
365 if vcstype == 'git-svn':
366 return vcs_gitsvn(remote, local)
368 return vcs_hg(remote, local)
370 return vcs_bzr(remote, local)
371 if vcstype == 'srclib':
372 if local != os.path.join('build', 'srclib', remote):
373 raise VCSException("Error: srclib paths are hard-coded!")
374 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
376 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
377 raise VCSException("Invalid vcs type " + vcstype)
380 def getsrclibvcs(name):
381 if name not in metadata.srclibs:
382 raise VCSException("Missing srclib " + name)
383 return metadata.srclibs[name]['Repo Type']
387 def __init__(self, remote, local):
389 # svn, git-svn and bzr may require auth
391 if self.repotype() in ('git-svn', 'bzr'):
393 self.username, remote = remote.split('@')
394 if ':' not in self.username:
395 raise VCSException("Password required with username")
396 self.username, self.password = self.username.split(':')
400 self.clone_failed = False
401 self.refreshed = False
407 # Take the local repository to a clean version of the given revision, which
408 # is specificed in the VCS's native format. Beforehand, the repository can
409 # be dirty, or even non-existent. If the repository does already exist
410 # locally, it will be updated from the origin, but only once in the
411 # lifetime of the vcs object.
412 # None is acceptable for 'rev' if you know you are cloning a clean copy of
413 # the repo - otherwise it must specify a valid revision.
414 def gotorevision(self, rev):
416 if self.clone_failed:
417 raise VCSException("Downloading the repository already failed once, not trying again.")
419 # The .fdroidvcs-id file for a repo tells us what VCS type
420 # and remote that directory was created from, allowing us to drop it
421 # automatically if either of those things changes.
422 fdpath = os.path.join(self.local, '..',
423 '.fdroidvcs-' + os.path.basename(self.local))
424 cdata = self.repotype() + ' ' + self.remote
427 if os.path.exists(self.local):
428 if os.path.exists(fdpath):
429 with open(fdpath, 'r') as f:
430 fsdata = f.read().strip()
436 "Repository details for %s changed - deleting" % (
440 logging.info("Repository details for %s missing - deleting" % (
443 shutil.rmtree(self.local)
448 self.gotorevisionx(rev)
449 except FDroidException, e:
452 # If necessary, write the .fdroidvcs file.
453 if writeback and not self.clone_failed:
454 with open(fdpath, 'w') as f:
460 # Derived classes need to implement this. It's called once basic checking
461 # has been performend.
462 def gotorevisionx(self, rev):
463 raise VCSException("This VCS type doesn't define gotorevisionx")
465 # Initialise and update submodules
466 def initsubmodules(self):
467 raise VCSException('Submodules not supported for this vcs type')
469 # Get a list of all known tags
471 raise VCSException('gettags not supported for this vcs type')
473 # Get a list of latest number tags
474 def latesttags(self, number):
475 raise VCSException('latesttags not supported for this vcs type')
477 # Get current commit reference (hash, revision, etc)
479 raise VCSException('getref not supported for this vcs type')
481 # Returns the srclib (name, path) used in setting up the current
492 # If the local directory exists, but is somehow not a git repository, git
493 # will traverse up the directory tree until it finds one that is (i.e.
494 # fdroidserver) and then we'll proceed to destroy it! This is called as
497 p = SilentPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
498 result = p.output.rstrip()
499 if not result.endswith(self.local):
500 raise VCSException('Repository mismatch')
502 def gotorevisionx(self, rev):
503 if not os.path.exists(self.local):
505 p = FDroidPopen(['git', 'clone', self.remote, self.local])
506 if p.returncode != 0:
507 self.clone_failed = True
508 raise VCSException("Git clone failed", p.output)
512 # Discard any working tree changes
513 p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
514 if p.returncode != 0:
515 raise VCSException("Git reset failed", p.output)
516 # Remove untracked files now, in case they're tracked in the target
517 # revision (it happens!)
518 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
519 if p.returncode != 0:
520 raise VCSException("Git clean failed", p.output)
521 if not self.refreshed:
522 # Get latest commits and tags from remote
523 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
524 if p.returncode != 0:
525 raise VCSException("Git fetch failed", p.output)
526 p = SilentPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local)
527 if p.returncode != 0:
528 raise VCSException("Git fetch failed", p.output)
529 # Recreate origin/HEAD as git clone would do it, in case it disappeared
530 p = SilentPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local)
531 if p.returncode != 0:
532 lines = p.output.splitlines()
533 if 'Multiple remote HEAD branches' not in lines[0]:
534 raise VCSException("Git remote set-head failed", p.output)
535 branch = lines[1].split(' ')[-1]
536 p2 = SilentPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local)
537 if p2.returncode != 0:
538 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
539 self.refreshed = True
540 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
541 # a github repo. Most of the time this is the same as origin/master.
542 rev = rev or 'origin/HEAD'
543 p = SilentPopen(['git', 'checkout', '-f', rev], cwd=self.local)
544 if p.returncode != 0:
545 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
546 # Get rid of any uncontrolled files left behind
547 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
548 if p.returncode != 0:
549 raise VCSException("Git clean failed", p.output)
551 def initsubmodules(self):
553 submfile = os.path.join(self.local, '.gitmodules')
554 if not os.path.isfile(submfile):
555 raise VCSException("No git submodules available")
557 # fix submodules not accessible without an account and public key auth
558 with open(submfile, 'r') as f:
559 lines = f.readlines()
560 with open(submfile, 'w') as f:
562 if 'git@github.com' in line:
563 line = line.replace('git@github.com:', 'https://github.com/')
567 ['git', 'reset', '--hard'],
568 ['git', 'clean', '-dffx'],
570 p = SilentPopen(['git', 'submodule', 'foreach', '--recursive'] + cmd, cwd=self.local)
571 if p.returncode != 0:
572 raise VCSException("Git submodule reset failed", p.output)
573 p = SilentPopen(['git', 'submodule', 'sync'], cwd=self.local)
574 if p.returncode != 0:
575 raise VCSException("Git submodule sync failed", p.output)
576 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
577 if p.returncode != 0:
578 raise VCSException("Git submodule update failed", p.output)
582 p = SilentPopen(['git', 'tag'], cwd=self.local)
583 return p.output.splitlines()
585 def latesttags(self, alltags, number):
587 p = SilentPopen(['echo "' + '\n'.join(alltags) + '" | '
588 + 'xargs -I@ git log --format=format:"%at @%n" -1 @ | '
589 + 'sort -n | awk \'{print $2}\''],
590 cwd=self.local, shell=True)
591 return p.output.splitlines()[-number:]
594 class vcs_gitsvn(vcs):
599 # Damn git-svn tries to use a graphical password prompt, so we have to
600 # trick it into taking the password from stdin
602 if self.username is None:
604 return ('echo "%s" | DISPLAY="" ' % self.password, ' --username "%s"' % self.username)
606 # If the local directory exists, but is somehow not a git repository, git
607 # will traverse up the directory tree until it finds one that is (i.e.
608 # fdroidserver) and then we'll proceed to destory it! This is called as
611 p = SilentPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
612 result = p.output.rstrip()
613 if not result.endswith(self.local):
614 raise VCSException('Repository mismatch')
616 def gotorevisionx(self, rev):
617 if not os.path.exists(self.local):
619 gitsvn_cmd = '%sgit svn clone%s' % self.userargs()
620 if ';' in self.remote:
621 remote_split = self.remote.split(';')
622 for i in remote_split[1:]:
623 if i.startswith('trunk='):
624 gitsvn_cmd += ' -T %s' % i[6:]
625 elif i.startswith('tags='):
626 gitsvn_cmd += ' -t %s' % i[5:]
627 elif i.startswith('branches='):
628 gitsvn_cmd += ' -b %s' % i[9:]
629 p = SilentPopen([gitsvn_cmd + " %s %s" % (remote_split[0], self.local)], shell=True)
630 if p.returncode != 0:
631 self.clone_failed = True
632 raise VCSException("Git svn clone failed", p.output)
634 p = SilentPopen([gitsvn_cmd + " %s %s" % (self.remote, self.local)], shell=True)
635 if p.returncode != 0:
636 self.clone_failed = True
637 raise VCSException("Git svn clone failed", p.output)
641 # Discard any working tree changes
642 p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
643 if p.returncode != 0:
644 raise VCSException("Git reset failed", p.output)
645 # Remove untracked files now, in case they're tracked in the target
646 # revision (it happens!)
647 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
648 if p.returncode != 0:
649 raise VCSException("Git clean failed", p.output)
650 if not self.refreshed:
651 # Get new commits, branches and tags from repo
652 p = SilentPopen(['%sgit svn fetch %s' % self.userargs()], cwd=self.local, shell=True)
653 if p.returncode != 0:
654 raise VCSException("Git svn fetch failed")
655 p = SilentPopen(['%sgit svn rebase %s' % self.userargs()], cwd=self.local, shell=True)
656 if p.returncode != 0:
657 raise VCSException("Git svn rebase failed", p.output)
658 self.refreshed = True
660 rev = rev or 'master'
662 nospaces_rev = rev.replace(' ', '%20')
663 # Try finding a svn tag
664 for treeish in ['origin/', '']:
665 p = SilentPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev],
667 if p.returncode == 0:
669 if p.returncode != 0:
670 # No tag found, normal svn rev translation
671 # Translate svn rev into git format
672 rev_split = rev.split('/')
675 for treeish in ['origin/', '']:
676 if len(rev_split) > 1:
677 treeish += rev_split[0]
678 svn_rev = rev_split[1]
681 # if no branch is specified, then assume trunk (i.e. 'master' branch):
685 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
687 p = SilentPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish],
689 git_rev = p.output.rstrip()
691 if p.returncode == 0 and git_rev:
694 if p.returncode != 0 or not git_rev:
695 # Try a plain git checkout as a last resort
696 p = SilentPopen(['git', 'checkout', rev], cwd=self.local)
697 if p.returncode != 0:
698 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
700 # Check out the git rev equivalent to the svn rev
701 p = SilentPopen(['git', 'checkout', git_rev], cwd=self.local)
702 if p.returncode != 0:
703 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
705 # Get rid of any uncontrolled files left behind
706 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
707 if p.returncode != 0:
708 raise VCSException("Git clean failed", p.output)
712 for treeish in ['origin/', '']:
713 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
719 p = SilentPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local)
720 if p.returncode != 0:
722 return p.output.strip()
730 def gotorevisionx(self, rev):
731 if not os.path.exists(self.local):
732 p = SilentPopen(['hg', 'clone', self.remote, self.local])
733 if p.returncode != 0:
734 self.clone_failed = True
735 raise VCSException("Hg clone failed", p.output)
737 p = SilentPopen(['hg status -uS | xargs rm -rf'], cwd=self.local, shell=True)
738 if p.returncode != 0:
739 raise VCSException("Hg clean failed", p.output)
740 if not self.refreshed:
741 p = SilentPopen(['hg', 'pull'], cwd=self.local)
742 if p.returncode != 0:
743 raise VCSException("Hg pull failed", p.output)
744 self.refreshed = True
746 rev = rev or 'default'
749 p = SilentPopen(['hg', 'update', '-C', rev], cwd=self.local)
750 if p.returncode != 0:
751 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
752 p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
753 # Also delete untracked files, we have to enable purge extension for that:
754 if "'purge' is provided by the following extension" in p.output:
755 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
756 myfile.write("\n[extensions]\nhgext.purge=\n")
757 p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
758 if p.returncode != 0:
759 raise VCSException("HG purge failed", p.output)
760 elif p.returncode != 0:
761 raise VCSException("HG purge failed", p.output)
764 p = SilentPopen(['hg', 'tags', '-q'], cwd=self.local)
765 return p.output.splitlines()[1:]
773 def gotorevisionx(self, rev):
774 if not os.path.exists(self.local):
775 p = SilentPopen(['bzr', 'branch', self.remote, self.local])
776 if p.returncode != 0:
777 self.clone_failed = True
778 raise VCSException("Bzr branch failed", p.output)
780 p = SilentPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local)
781 if p.returncode != 0:
782 raise VCSException("Bzr revert failed", p.output)
783 if not self.refreshed:
784 p = SilentPopen(['bzr', 'pull'], cwd=self.local)
785 if p.returncode != 0:
786 raise VCSException("Bzr update failed", p.output)
787 self.refreshed = True
789 revargs = list(['-r', rev] if rev else [])
790 p = SilentPopen(['bzr', 'revert'] + revargs, cwd=self.local)
791 if p.returncode != 0:
792 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
795 p = SilentPopen(['bzr', 'tags'], cwd=self.local)
796 return [tag.split(' ')[0].strip() for tag in
797 p.output.splitlines()]
800 def retrieve_string(app_dir, string, xmlfiles=None):
803 os.path.join(app_dir, 'res'),
804 os.path.join(app_dir, 'src', 'main'),
809 for res_dir in res_dirs:
810 for r, d, f in os.walk(res_dir):
811 if os.path.basename(r) == 'values':
812 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
815 if string.startswith('@string/'):
816 string_search = re.compile(r'.*name="' + string[8:] + '".*?>"?([^<]+?)"?<.*').search
817 elif string.startswith('&') and string.endswith(';'):
818 string_search = re.compile(r'.*<!ENTITY.*' + string[1:-1] + '.*?"([^"]+?)".*>').search
820 if string_search is not None:
821 for xmlfile in xmlfiles:
822 for line in file(xmlfile):
823 matches = string_search(line)
825 return retrieve_string(app_dir, matches.group(1), xmlfiles)
828 return string.replace("\\'", "'")
831 # Return list of existing files that will be used to find the highest vercode
832 def manifest_paths(app_dir, flavours):
834 possible_manifests = \
835 [os.path.join(app_dir, 'AndroidManifest.xml'),
836 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
837 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
838 os.path.join(app_dir, 'build.gradle')]
840 for flavour in flavours:
843 possible_manifests.append(
844 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
846 return [path for path in possible_manifests if os.path.isfile(path)]
849 # Retrieve the package name. Returns the name, or None if not found.
850 def fetch_real_name(app_dir, flavours):
851 app_search = re.compile(r'.*<application.*').search
852 name_search = re.compile(r'.*android:label="([^"]+)".*').search
854 for f in manifest_paths(app_dir, flavours):
855 if not has_extension(f, 'xml'):
857 logging.debug("fetch_real_name: Checking manifest at " + f)
863 matches = name_search(line)
865 stringname = matches.group(1)
866 logging.debug("fetch_real_name: using string " + stringname)
867 result = retrieve_string(app_dir, stringname)
869 result = result.strip()
874 # Retrieve the version name
875 def version_name(original, app_dir, flavours):
876 for f in manifest_paths(app_dir, flavours):
877 if not has_extension(f, 'xml'):
879 string = retrieve_string(app_dir, original)
885 def get_library_references(root_dir):
887 proppath = os.path.join(root_dir, 'project.properties')
888 if not os.path.isfile(proppath):
890 with open(proppath) as f:
891 for line in f.readlines():
892 if not line.startswith('android.library.reference.'):
894 path = line.split('=')[1].strip()
895 relpath = os.path.join(root_dir, path)
896 if not os.path.isdir(relpath):
898 logging.debug("Found subproject at %s" % path)
899 libraries.append(path)
903 def ant_subprojects(root_dir):
904 subprojects = get_library_references(root_dir)
905 for subpath in subprojects:
906 subrelpath = os.path.join(root_dir, subpath)
907 for p in get_library_references(subrelpath):
908 relp = os.path.normpath(os.path.join(subpath, p))
909 if relp not in subprojects:
910 subprojects.insert(0, relp)
914 def remove_debuggable_flags(root_dir):
915 # Remove forced debuggable flags
916 logging.debug("Removing debuggable flags from %s" % root_dir)
917 for root, dirs, files in os.walk(root_dir):
918 if 'AndroidManifest.xml' in files:
919 path = os.path.join(root, 'AndroidManifest.xml')
920 p = SilentPopen(['sed', '-i', 's/android:debuggable="[^"]*"//g', path])
921 if p.returncode != 0:
922 raise BuildException("Failed to remove debuggable flags of %s" % path)
925 # Extract some information from the AndroidManifest.xml at the given path.
926 # Returns (version, vercode, package), any or all of which might be None.
927 # All values returned are strings.
928 def parse_androidmanifests(paths, ignoreversions=None):
931 return (None, None, None)
933 vcsearch = re.compile(r'.*:versionCode="([0-9]+?)".*').search
934 vnsearch = re.compile(r'.*:versionName="([^"]+?)".*').search
935 psearch = re.compile(r'.*package="([^"]+)".*').search
937 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
938 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
939 psearch_g = re.compile(r'.*packageName *=* *["\']([^"]+)["\'].*').search
941 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
949 gradle = has_extension(path, 'gradle')
952 # Remember package name, may be defined separately from version+vercode
953 package = max_package
955 for line in file(path):
958 matches = psearch_g(line)
960 matches = psearch(line)
962 package = matches.group(1)
965 matches = vnsearch_g(line)
967 matches = vnsearch(line)
969 version = matches.group(2 if gradle else 1)
972 matches = vcsearch_g(line)
974 matches = vcsearch(line)
976 vercode = matches.group(1)
978 # Always grab the package name and version name in case they are not
979 # together with the highest version code
980 if max_package is None and package is not None:
981 max_package = package
982 if max_version is None and version is not None:
983 max_version = version
985 if max_vercode is None or (vercode is not None and vercode > max_vercode):
986 if not ignoresearch or not ignoresearch(version):
987 if version is not None:
988 max_version = version
989 if vercode is not None:
990 max_vercode = vercode
991 if package is not None:
992 max_package = package
994 max_version = "Ignore"
996 if max_version is None:
997 max_version = "Unknown"
999 return (max_version, max_vercode, max_package)
1002 class FDroidException(Exception):
1003 def __init__(self, value, detail=None):
1005 self.detail = detail
1007 def get_wikitext(self):
1008 ret = repr(self.value) + "\n"
1012 txt = self.detail[-8192:] if len(self.detail) > 8192 else self.detail
1020 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1024 class VCSException(FDroidException):
1028 class BuildException(FDroidException):
1032 # Get the specified source library.
1033 # Returns the path to it. Normally this is the path to be used when referencing
1034 # it, which may be a subdirectory of the actual project. If you want the base
1035 # directory of the project, pass 'basepath=True'.
1036 def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
1037 basepath=False, raw=False, prepare=True, preponly=False):
1045 name, ref = spec.split('@')
1047 number, name = name.split(':', 1)
1049 name, subdir = name.split('/', 1)
1051 if name not in metadata.srclibs:
1052 raise VCSException('srclib ' + name + ' not found.')
1054 srclib = metadata.srclibs[name]
1056 sdir = os.path.join(srclib_dir, name)
1059 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1060 vcs.srclib = (name, number, sdir)
1062 vcs.gotorevision(ref)
1069 libdir = os.path.join(sdir, subdir)
1070 elif srclib["Subdir"]:
1071 for subdir in srclib["Subdir"]:
1072 libdir_candidate = os.path.join(sdir, subdir)
1073 if os.path.exists(libdir_candidate):
1074 libdir = libdir_candidate
1080 if srclib["Srclibs"]:
1082 for lib in srclib["Srclibs"].replace(';', ',').split(','):
1084 for t in srclibpaths:
1089 raise VCSException('Missing recursive srclib %s for %s' % (
1091 place_srclib(libdir, n, s_tuple[2])
1094 remove_signing_keys(sdir)
1095 remove_debuggable_flags(sdir)
1099 if srclib["Prepare"]:
1100 cmd = replace_config_vars(srclib["Prepare"])
1102 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1103 if p.returncode != 0:
1104 raise BuildException("Error running prepare command for srclib %s"
1110 return (name, number, libdir)
1113 # Prepare the source code for a particular build
1114 # 'vcs' - the appropriate vcs object for the application
1115 # 'app' - the application details from the metadata
1116 # 'build' - the build details from the metadata
1117 # 'build_dir' - the path to the build directory, usually
1119 # 'srclib_dir' - the path to the source libraries directory, usually
1121 # 'extlib_dir' - the path to the external libraries directory, usually
1123 # Returns the (root, srclibpaths) where:
1124 # 'root' is the root directory, which may be the same as 'build_dir' or may
1125 # be a subdirectory of it.
1126 # 'srclibpaths' is information on the srclibs being used
1127 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False):
1129 # Optionally, the actual app source can be in a subdirectory
1131 root_dir = os.path.join(build_dir, build['subdir'])
1133 root_dir = build_dir
1135 # Get a working copy of the right revision
1136 logging.info("Getting source for revision " + build['commit'])
1137 vcs.gotorevision(build['commit'])
1139 # Initialise submodules if requred
1140 if build['submodules']:
1141 logging.info("Initialising submodules")
1142 vcs.initsubmodules()
1144 # Check that a subdir (if we're using one) exists. This has to happen
1145 # after the checkout, since it might not exist elsewhere
1146 if not os.path.exists(root_dir):
1147 raise BuildException('Missing subdir ' + root_dir)
1149 # Run an init command if one is required
1151 cmd = replace_config_vars(build['init'])
1152 logging.info("Running 'init' commands in %s" % root_dir)
1154 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1155 if p.returncode != 0:
1156 raise BuildException("Error running init command for %s:%s" %
1157 (app['id'], build['version']), p.output)
1159 # Apply patches if any
1161 logging.info("Applying patches")
1162 for patch in build['patch']:
1163 patch = patch.strip()
1164 logging.info("Applying " + patch)
1165 patch_path = os.path.join('metadata', app['id'], patch)
1166 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1167 if p.returncode != 0:
1168 raise BuildException("Failed to apply patch %s" % patch_path)
1170 # Get required source libraries
1172 if build['srclibs']:
1173 logging.info("Collecting source libraries")
1174 for lib in build['srclibs']:
1175 srclibpaths.append(getsrclib(lib, srclib_dir, srclibpaths,
1178 for name, number, libpath in srclibpaths:
1179 place_srclib(root_dir, int(number) if number else None, libpath)
1181 basesrclib = vcs.getsrclib()
1182 # If one was used for the main source, add that too.
1184 srclibpaths.append(basesrclib)
1186 # Update the local.properties file
1187 localprops = [os.path.join(build_dir, 'local.properties')]
1189 localprops += [os.path.join(root_dir, 'local.properties')]
1190 for path in localprops:
1191 if not os.path.isfile(path):
1193 logging.info("Updating properties file at %s" % path)
1198 # Fix old-fashioned 'sdk-location' by copying
1199 # from sdk.dir, if necessary
1200 if build['oldsdkloc']:
1201 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1202 re.S | re.M).group(1)
1203 props += "sdk-location=%s\n" % sdkloc
1205 props += "sdk.dir=%s\n" % config['sdk_path']
1206 props += "sdk-location=%s\n" % config['sdk_path']
1207 if 'ndk_path' in config:
1209 props += "ndk.dir=%s\n" % config['ndk_path']
1210 props += "ndk-location=%s\n" % config['ndk_path']
1211 # Add java.encoding if necessary
1212 if build['encoding']:
1213 props += "java.encoding=%s\n" % build['encoding']
1219 if build['type'] == 'gradle':
1220 flavours = build['gradle']
1222 version_regex = re.compile(r".*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1223 gradlepluginver = None
1225 gradle_files = [os.path.join(root_dir, 'build.gradle')]
1227 # Parent dir build.gradle
1228 parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
1229 if parent_dir.startswith(build_dir):
1230 gradle_files.append(os.path.join(parent_dir, 'build.gradle'))
1232 for path in gradle_files:
1235 if not os.path.isfile(path):
1237 with open(path) as f:
1239 match = version_regex.match(line)
1241 gradlepluginver = match.group(1)
1245 build['gradlepluginver'] = LooseVersion(gradlepluginver)
1247 logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
1248 build['gradlepluginver'] = LooseVersion('0.11')
1251 n = build["target"].split('-')[1]
1252 SilentPopen(['sed', '-i',
1253 's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g',
1257 # Remove forced debuggable flags
1258 remove_debuggable_flags(root_dir)
1260 # Insert version code and number into the manifest if necessary
1261 if build['forceversion']:
1262 logging.info("Changing the version name")
1263 for path in manifest_paths(root_dir, flavours):
1264 if not os.path.isfile(path):
1266 if has_extension(path, 'xml'):
1267 p = SilentPopen(['sed', '-i',
1268 's/android:versionName="[^"]*"/android:versionName="'
1269 + build['version'] + '"/g',
1271 if p.returncode != 0:
1272 raise BuildException("Failed to amend manifest")
1273 elif has_extension(path, 'gradle'):
1274 p = SilentPopen(['sed', '-i',
1275 's/versionName *=* *"[^"]*"/versionName = "'
1276 + build['version'] + '"/g',
1278 if p.returncode != 0:
1279 raise BuildException("Failed to amend build.gradle")
1280 if build['forcevercode']:
1281 logging.info("Changing the version code")
1282 for path in manifest_paths(root_dir, flavours):
1283 if not os.path.isfile(path):
1285 if has_extension(path, 'xml'):
1286 p = SilentPopen(['sed', '-i',
1287 's/android:versionCode="[^"]*"/android:versionCode="'
1288 + build['vercode'] + '"/g',
1290 if p.returncode != 0:
1291 raise BuildException("Failed to amend manifest")
1292 elif has_extension(path, 'gradle'):
1293 p = SilentPopen(['sed', '-i',
1294 's/versionCode *=* *[0-9]*/versionCode = '
1295 + build['vercode'] + '/g',
1297 if p.returncode != 0:
1298 raise BuildException("Failed to amend build.gradle")
1300 # Delete unwanted files
1302 logging.info("Removing specified files")
1303 for part in getpaths(build_dir, build, 'rm'):
1304 dest = os.path.join(build_dir, part)
1305 logging.info("Removing {0}".format(part))
1306 if os.path.lexists(dest):
1307 if os.path.islink(dest):
1308 SilentPopen(['unlink ' + dest], shell=True)
1310 SilentPopen(['rm -rf ' + dest], shell=True)
1312 logging.info("...but it didn't exist")
1314 remove_signing_keys(build_dir)
1316 # Add required external libraries
1317 if build['extlibs']:
1318 logging.info("Collecting prebuilt libraries")
1319 libsdir = os.path.join(root_dir, 'libs')
1320 if not os.path.exists(libsdir):
1322 for lib in build['extlibs']:
1324 logging.info("...installing extlib {0}".format(lib))
1325 libf = os.path.basename(lib)
1326 libsrc = os.path.join(extlib_dir, lib)
1327 if not os.path.exists(libsrc):
1328 raise BuildException("Missing extlib file {0}".format(libsrc))
1329 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1331 # Run a pre-build command if one is required
1332 if build['prebuild']:
1333 logging.info("Running 'prebuild' commands in %s" % root_dir)
1335 cmd = replace_config_vars(build['prebuild'])
1337 # Substitute source library paths into prebuild commands
1338 for name, number, libpath in srclibpaths:
1339 libpath = os.path.relpath(libpath, root_dir)
1340 cmd = cmd.replace('$$' + name + '$$', libpath)
1342 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1343 if p.returncode != 0:
1344 raise BuildException("Error running prebuild command for %s:%s" %
1345 (app['id'], build['version']), p.output)
1347 # Generate (or update) the ant build file, build.xml...
1348 if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
1349 parms = [config['android'], 'update', 'lib-project']
1350 lparms = [config['android'], 'update', 'project']
1353 parms += ['-t', build['target']]
1354 lparms += ['-t', build['target']]
1355 if build['update'] == ['auto']:
1356 update_dirs = ant_subprojects(root_dir) + ['.']
1358 update_dirs = build['update']
1360 for d in update_dirs:
1361 subdir = os.path.join(root_dir, d)
1363 logging.debug("Updating main project")
1364 cmd = parms + ['-p', d]
1366 logging.debug("Updating subproject %s" % d)
1367 cmd = lparms + ['-p', d]
1368 p = FDroidPopen(cmd, cwd=root_dir)
1369 # Check to see whether an error was returned without a proper exit
1370 # code (this is the case for the 'no target set or target invalid'
1372 if p.returncode != 0 or p.output.startswith("Error: "):
1373 raise BuildException("Failed to update project at %s" % d, p.output)
1374 # Clean update dirs via ant
1376 logging.info("Cleaning subproject %s" % d)
1377 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1379 return (root_dir, srclibpaths)
1382 # Split and extend via globbing the paths from a field
1383 def getpaths(build_dir, build, field):
1385 for p in build[field]:
1387 full_path = os.path.join(build_dir, p)
1388 full_path = os.path.normpath(full_path)
1389 paths += [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1393 # Scan the source code in the given directory (and all subdirectories)
1394 # and return the number of fatal problems encountered
1395 def scan_source(build_dir, root_dir, thisbuild):
1399 # Common known non-free blobs (always lower case):
1401 re.compile(r'flurryagent', re.IGNORECASE),
1402 re.compile(r'paypal.*mpl', re.IGNORECASE),
1403 re.compile(r'google.*analytics', re.IGNORECASE),
1404 re.compile(r'admob.*sdk.*android', re.IGNORECASE),
1405 re.compile(r'google.*ad.*view', re.IGNORECASE),
1406 re.compile(r'google.*admob', re.IGNORECASE),
1407 re.compile(r'google.*play.*services', re.IGNORECASE),
1408 re.compile(r'crittercism', re.IGNORECASE),
1409 re.compile(r'heyzap', re.IGNORECASE),
1410 re.compile(r'jpct.*ae', re.IGNORECASE),
1411 re.compile(r'youtube.*android.*player.*api', re.IGNORECASE),
1412 re.compile(r'bugsense', re.IGNORECASE),
1413 re.compile(r'crashlytics', re.IGNORECASE),
1414 re.compile(r'ouya.*sdk', re.IGNORECASE),
1415 re.compile(r'libspen23', re.IGNORECASE),
1418 scanignore = getpaths(build_dir, thisbuild, 'scanignore')
1419 scandelete = getpaths(build_dir, thisbuild, 'scandelete')
1422 ms = magic.open(magic.MIME_TYPE)
1424 except AttributeError:
1428 for i in scanignore:
1429 if fd.startswith(i):
1434 for i in scandelete:
1435 if fd.startswith(i):
1439 def removeproblem(what, fd, fp):
1440 logging.info('Removing %s at %s' % (what, fd))
1443 def warnproblem(what, fd):
1444 logging.warn('Found %s at %s' % (what, fd))
1446 def handleproblem(what, fd, fp):
1448 logging.info('Ignoring %s at %s' % (what, fd))
1450 removeproblem(what, fd, fp)
1452 logging.error('Found %s at %s' % (what, fd))
1456 # Iterate through all files in the source code
1457 for r, d, f in os.walk(build_dir, topdown=True):
1459 # It's topdown, so checking the basename is enough
1460 for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
1466 # Path (relative) to the file
1467 fp = os.path.join(r, curfile)
1468 fd = fp[len(build_dir) + 1:]
1471 mime = magic.from_file(fp, mime=True) if ms is None else ms.file(fp)
1472 except UnicodeError:
1473 warnproblem('malformed magic number', fd)
1475 if mime == 'application/x-sharedlib':
1476 count += handleproblem('shared library', fd, fp)
1478 elif mime == 'application/x-archive':
1479 count += handleproblem('static library', fd, fp)
1481 elif mime == 'application/x-executable':
1482 count += handleproblem('binary executable', fd, fp)
1484 elif mime == 'application/x-java-applet':
1485 count += handleproblem('Java compiled class', fd, fp)
1490 'application/java-archive',
1491 'application/octet-stream',
1495 if has_extension(fp, 'apk'):
1496 removeproblem('APK file', fd, fp)
1498 elif has_extension(fp, 'jar'):
1500 if any(suspect.match(curfile) for suspect in usual_suspects):
1501 count += handleproblem('usual supect', fd, fp)
1503 warnproblem('JAR file', fd)
1505 elif has_extension(fp, 'zip'):
1506 warnproblem('ZIP file', fd)
1509 warnproblem('unknown compressed or binary file', fd)
1511 elif has_extension(fp, 'java'):
1512 for line in file(fp):
1513 if 'DexClassLoader' in line:
1514 count += handleproblem('DexClassLoader', fd, fp)
1519 # Presence of a jni directory without buildjni=yes might
1520 # indicate a problem (if it's not a problem, explicitly use
1521 # buildjni=no to bypass this check)
1522 if (os.path.exists(os.path.join(root_dir, 'jni')) and
1523 not thisbuild['buildjni']):
1524 logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
1533 self.path = os.path.join('stats', 'known_apks.txt')
1535 if os.path.exists(self.path):
1536 for line in file(self.path):
1537 t = line.rstrip().split(' ')
1539 self.apks[t[0]] = (t[1], None)
1541 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1542 self.changed = False
1544 def writeifchanged(self):
1546 if not os.path.exists('stats'):
1548 f = open(self.path, 'w')
1550 for apk, app in self.apks.iteritems():
1552 line = apk + ' ' + appid
1554 line += ' ' + time.strftime('%Y-%m-%d', added)
1556 for line in sorted(lst):
1557 f.write(line + '\n')
1560 # Record an apk (if it's new, otherwise does nothing)
1561 # Returns the date it was added.
1562 def recordapk(self, apk, app):
1563 if apk not in self.apks:
1564 self.apks[apk] = (app, time.gmtime(time.time()))
1566 _, added = self.apks[apk]
1569 # Look up information - given the 'apkname', returns (app id, date added/None).
1570 # Or returns None for an unknown apk.
1571 def getapp(self, apkname):
1572 if apkname in self.apks:
1573 return self.apks[apkname]
1576 # Get the most recent 'num' apps added to the repo, as a list of package ids
1577 # with the most recent first.
1578 def getlatest(self, num):
1580 for apk, app in self.apks.iteritems():
1584 if apps[appid] > added:
1588 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1589 lst = [app for app, _ in sortedapps]
1594 def isApkDebuggable(apkfile, config):
1595 """Returns True if the given apk file is debuggable
1597 :param apkfile: full path to the apk to check"""
1599 p = SilentPopen([os.path.join(config['sdk_path'], 'build-tools',
1600 config['build_tools'], 'aapt'),
1601 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'])
1602 if p.returncode != 0:
1603 logging.critical("Failed to get apk manifest information")
1605 for line in p.output.splitlines():
1606 if 'android:debuggable' in line and not line.endswith('0x0'):
1611 class AsynchronousFileReader(threading.Thread):
1613 Helper class to implement asynchronous reading of a file
1614 in a separate thread. Pushes read lines on a queue to
1615 be consumed in another thread.
1618 def __init__(self, fd, queue):
1619 assert isinstance(queue, Queue.Queue)
1620 assert callable(fd.readline)
1621 threading.Thread.__init__(self)
1626 '''The body of the tread: read lines and put them on the queue.'''
1627 for line in iter(self._fd.readline, ''):
1628 self._queue.put(line)
1631 '''Check whether there is no more content to expect.'''
1632 return not self.is_alive() and self._queue.empty()
1640 def SilentPopen(commands, cwd=None, shell=False):
1641 return FDroidPopen(commands, cwd=cwd, shell=shell, output=False)
1644 def FDroidPopen(commands, cwd=None, shell=False, output=True):
1646 Run a command and capture the possibly huge output.
1648 :param commands: command and argument list like in subprocess.Popen
1649 :param cwd: optionally specifies a working directory
1650 :returns: A PopenResult.
1656 cwd = os.path.normpath(cwd)
1657 logging.debug("Directory: %s" % cwd)
1658 logging.debug("> %s" % ' '.join(commands))
1660 result = PopenResult()
1663 p = subprocess.Popen(commands, cwd=cwd, shell=shell, env=env,
1664 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1666 raise BuildException("OSError while trying to execute " + ' '.join(commands) + ': ' + str(e))
1668 stdout_queue = Queue.Queue()
1669 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1670 stdout_reader.start()
1672 # Check the queue for output (until there is no more to get)
1673 while not stdout_reader.eof():
1674 while not stdout_queue.empty():
1675 line = stdout_queue.get()
1676 if output and options.verbose:
1677 # Output directly to console
1678 sys.stderr.write(line)
1680 result.output += line
1684 result.returncode = p.wait()
1688 def remove_signing_keys(build_dir):
1689 comment = re.compile(r'[ ]*//')
1690 signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1692 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1693 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1694 re.compile(r'.*variant\.outputFile = .*'),
1695 re.compile(r'.*\.readLine\(.*'),
1697 for root, dirs, files in os.walk(build_dir):
1698 if 'build.gradle' in files:
1699 path = os.path.join(root, 'build.gradle')
1701 with open(path, "r") as o:
1702 lines = o.readlines()
1707 with open(path, "w") as o:
1709 if comment.match(line):
1713 opened += line.count('{')
1714 opened -= line.count('}')
1717 if signing_configs.match(line):
1722 if any(s.match(line) for s in line_matches):
1730 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1733 'project.properties',
1735 'default.properties',
1738 if propfile in files:
1739 path = os.path.join(root, propfile)
1741 with open(path, "r") as o:
1742 lines = o.readlines()
1746 with open(path, "w") as o:
1748 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1755 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1758 def replace_config_vars(cmd):
1759 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1760 cmd = cmd.replace('$$NDK$$', config['ndk_path'])
1761 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1765 def place_srclib(root_dir, number, libpath):
1768 relpath = os.path.relpath(libpath, root_dir)
1769 proppath = os.path.join(root_dir, 'project.properties')
1772 if os.path.isfile(proppath):
1773 with open(proppath, "r") as o:
1774 lines = o.readlines()
1776 with open(proppath, "w") as o:
1779 if line.startswith('android.library.reference.%d=' % number):
1780 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1785 o.write('android.library.reference.%d=%s\n' % (number, relpath))