1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
33 from distutils.version import LooseVersion
42 def get_default_config():
44 'sdk_path': os.getenv("ANDROID_HOME") or "",
45 'ndk_path': os.getenv("ANDROID_NDK") or "",
46 'build_tools': "20.0.0",
50 'sync_from_local_copy_dir': False,
51 'update_stats': False,
52 'stats_to_carbon': False,
54 'build_server_always': False,
55 'keystore': os.path.join(os.getenv("HOME"), '.local', 'share', 'fdroidserver', 'keystore.jks'),
56 'smartcardoptions': [],
62 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
63 'repo_name': "My First FDroid Repo Demo",
64 'repo_icon': "fdroid-icon.png",
65 'repo_description': '''
66 This is a repository of apps to be used with FDroid. Applications in this
67 repository are either official binaries built by the original application
68 developers, or are binaries built from source by the admin of f-droid.org
69 using the tools on https://gitlab.com/u/fdroid.
75 def read_config(opts, config_file='config.py'):
76 """Read the repository config
78 The config is read from config_file, which is in the current directory when
79 any of the repo management commands are used.
81 global config, options, env
83 if config is not None:
85 if not os.path.isfile(config_file):
86 logging.critical("Missing config file - is this a repo directory?")
93 logging.debug("Reading %s" % config_file)
94 execfile(config_file, config)
96 # smartcardoptions must be a list since its command line args for Popen
97 if 'smartcardoptions' in config:
98 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
99 elif 'keystore' in config and config['keystore'] == 'NONE':
100 # keystore='NONE' means use smartcard, these are required defaults
101 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
102 'SunPKCS11-OpenSC', '-providerClass',
103 'sun.security.pkcs11.SunPKCS11',
104 '-providerArg', 'opensc-fdroid.cfg']
106 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
107 st = os.stat(config_file)
108 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
109 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
111 defconfig = get_default_config()
112 for k, v in defconfig.items():
116 # Expand environment variables
117 for k, v in config.items():
120 v = os.path.expanduser(v)
121 config[k] = os.path.expandvars(v)
123 if not test_sdk_exists(config):
126 if not test_build_tools_exists(config):
131 os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
134 os.path.join(config['sdk_path'], 'tools', 'zipalign'),
135 os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'zipalign'),
138 os.path.join(config['sdk_path'], 'tools', 'android'),
141 os.path.join(config['sdk_path'], 'platform-tools', 'adb'),
145 for b, paths in bin_paths.items():
148 if os.path.isfile(path):
151 if config[b] is None:
152 logging.warn("Could not find %s in any of the following paths:\n%s" % (
153 b, '\n'.join(paths)))
155 # There is no standard, so just set up the most common environment
158 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
159 env[n] = config['sdk_path']
160 for n in ['ANDROID_NDK', 'NDK']:
161 env[n] = config['ndk_path']
163 for k in ["keystorepass", "keypass"]:
165 write_password_file(k)
167 for k in ["repo_description", "archive_description"]:
169 config[k] = clean_description(config[k])
171 if 'serverwebroot' in config:
172 if isinstance(config['serverwebroot'], basestring):
173 roots = [config['serverwebroot']]
174 elif all(isinstance(item, basestring) for item in config['serverwebroot']):
175 roots = config['serverwebroot']
177 raise TypeError('only accepts strings, lists, and tuples')
179 for rootstr in roots:
180 # since this is used with rsync, where trailing slashes have
181 # meaning, ensure there is always a trailing slash
182 if rootstr[-1] != '/':
184 rootlist.append(rootstr.replace('//', '/'))
185 config['serverwebroot'] = rootlist
190 def test_sdk_exists(c):
191 if c['sdk_path'] is None:
192 # c['sdk_path'] is set to the value of ANDROID_HOME by default
193 logging.error('No Android SDK found! ANDROID_HOME is not set and sdk_path is not in config.py!')
194 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
195 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
197 if not os.path.exists(c['sdk_path']):
198 logging.critical('Android SDK path "' + c['sdk_path'] + '" does not exist!')
200 if not os.path.isdir(c['sdk_path']):
201 logging.critical('Android SDK path "' + c['sdk_path'] + '" is not a directory!')
203 for d in ['build-tools', 'platform-tools', 'tools']:
204 if not os.path.isdir(os.path.join(c['sdk_path'], d)):
205 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
211 def test_build_tools_exists(c):
212 if not test_sdk_exists(c):
214 build_tools = os.path.join(c['sdk_path'], 'build-tools')
215 versioned_build_tools = os.path.join(build_tools, c['build_tools'])
216 if not os.path.isdir(versioned_build_tools):
217 logging.critical('Android Build Tools path "'
218 + versioned_build_tools + '" does not exist!')
223 def write_password_file(pwtype, password=None):
225 writes out passwords to a protected file instead of passing passwords as
226 command line argments
228 filename = '.fdroid.' + pwtype + '.txt'
229 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
231 os.write(fd, config[pwtype])
233 os.write(fd, password)
235 config[pwtype + 'file'] = filename
238 # Given the arguments in the form of multiple appid:[vc] strings, this returns
239 # a dictionary with the set of vercodes specified for each package.
240 def read_pkg_args(args, allow_vercodes=False):
247 if allow_vercodes and ':' in p:
248 package, vercode = p.split(':')
250 package, vercode = p, None
251 if package not in vercodes:
252 vercodes[package] = [vercode] if vercode else []
254 elif vercode and vercode not in vercodes[package]:
255 vercodes[package] += [vercode] if vercode else []
260 # On top of what read_pkg_args does, this returns the whole app metadata, but
261 # limiting the builds list to the builds matching the vercodes specified.
262 def read_app_args(args, allapps, allow_vercodes=False):
264 vercodes = read_pkg_args(args, allow_vercodes)
269 apps = [app for app in allapps if app['id'] in vercodes]
271 if len(apps) != len(vercodes):
272 allids = [app["id"] for app in allapps]
275 logging.critical("No such package: %s" % p)
276 raise FDroidException("Found invalid app ids in arguments")
278 raise FDroidException("No packages specified")
282 vc = vercodes[app['id']]
285 app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
286 if len(app['builds']) != len(vercodes[app['id']]):
288 allvcs = [b['vercode'] for b in app['builds']]
289 for v in vercodes[app['id']]:
291 logging.critical("No such vercode %s for app %s" % (v, app['id']))
294 raise FDroidException("Found invalid vercodes for some apps")
299 def has_extension(filename, extension):
300 name, ext = os.path.splitext(filename)
301 ext = ext.lower()[1:]
302 return ext == extension
307 def clean_description(description):
308 'Remove unneeded newlines and spaces from a block of description text'
310 # this is split up by paragraph to make removing the newlines easier
311 for paragraph in re.split(r'\n\n', description):
312 paragraph = re.sub('\r', '', paragraph)
313 paragraph = re.sub('\n', ' ', paragraph)
314 paragraph = re.sub(' {2,}', ' ', paragraph)
315 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
316 returnstring += paragraph + '\n\n'
317 return returnstring.rstrip('\n')
320 def apknameinfo(filename):
322 filename = os.path.basename(filename)
323 if apk_regex is None:
324 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
325 m = apk_regex.match(filename)
327 result = (m.group(1), m.group(2))
328 except AttributeError:
329 raise FDroidException("Invalid apk name: %s" % filename)
333 def getapkname(app, build):
334 return "%s_%s.apk" % (app['id'], build['vercode'])
337 def getsrcname(app, build):
338 return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
345 return app['Auto Name']
350 return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
353 def getvcs(vcstype, remote, local):
355 return vcs_git(remote, local)
356 if vcstype == 'git-svn':
357 return vcs_gitsvn(remote, local)
359 return vcs_hg(remote, local)
361 return vcs_bzr(remote, local)
362 if vcstype == 'srclib':
363 if local != os.path.join('build', 'srclib', remote):
364 raise VCSException("Error: srclib paths are hard-coded!")
365 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
367 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
368 raise VCSException("Invalid vcs type " + vcstype)
371 def getsrclibvcs(name):
372 if name not in metadata.srclibs:
373 raise VCSException("Missing srclib " + name)
374 return metadata.srclibs[name]['Repo Type']
378 def __init__(self, remote, local):
380 # svn, git-svn and bzr may require auth
382 if self.repotype() in ('git-svn', 'bzr'):
384 self.username, remote = remote.split('@')
385 if ':' not in self.username:
386 raise VCSException("Password required with username")
387 self.username, self.password = self.username.split(':')
391 self.clone_failed = False
392 self.refreshed = False
398 # Take the local repository to a clean version of the given revision, which
399 # is specificed in the VCS's native format. Beforehand, the repository can
400 # be dirty, or even non-existent. If the repository does already exist
401 # locally, it will be updated from the origin, but only once in the
402 # lifetime of the vcs object.
403 # None is acceptable for 'rev' if you know you are cloning a clean copy of
404 # the repo - otherwise it must specify a valid revision.
405 def gotorevision(self, rev):
407 if self.clone_failed:
408 raise VCSException("Downloading the repository already failed once, not trying again.")
410 # The .fdroidvcs-id file for a repo tells us what VCS type
411 # and remote that directory was created from, allowing us to drop it
412 # automatically if either of those things changes.
413 fdpath = os.path.join(self.local, '..',
414 '.fdroidvcs-' + os.path.basename(self.local))
415 cdata = self.repotype() + ' ' + self.remote
418 if os.path.exists(self.local):
419 if os.path.exists(fdpath):
420 with open(fdpath, 'r') as f:
421 fsdata = f.read().strip()
427 "Repository details for %s changed - deleting" % (
431 logging.info("Repository details for %s missing - deleting" % (
434 shutil.rmtree(self.local)
439 self.gotorevisionx(rev)
440 except FDroidException, e:
443 # If necessary, write the .fdroidvcs file.
444 if writeback and not self.clone_failed:
445 with open(fdpath, 'w') as f:
451 # Derived classes need to implement this. It's called once basic checking
452 # has been performend.
453 def gotorevisionx(self, rev):
454 raise VCSException("This VCS type doesn't define gotorevisionx")
456 # Initialise and update submodules
457 def initsubmodules(self):
458 raise VCSException('Submodules not supported for this vcs type')
460 # Get a list of all known tags
462 raise VCSException('gettags not supported for this vcs type')
464 # Get a list of latest number tags
465 def latesttags(self, number):
466 raise VCSException('latesttags not supported for this vcs type')
468 # Get current commit reference (hash, revision, etc)
470 raise VCSException('getref not supported for this vcs type')
472 # Returns the srclib (name, path) used in setting up the current
483 # If the local directory exists, but is somehow not a git repository, git
484 # will traverse up the directory tree until it finds one that is (i.e.
485 # fdroidserver) and then we'll proceed to destroy it! This is called as
488 p = SilentPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
489 result = p.output.rstrip()
490 if not result.endswith(self.local):
491 raise VCSException('Repository mismatch')
493 def gotorevisionx(self, rev):
494 if not os.path.exists(self.local):
496 p = FDroidPopen(['git', 'clone', self.remote, self.local])
497 if p.returncode != 0:
498 self.clone_failed = True
499 raise VCSException("Git clone failed", p.output)
503 # Discard any working tree changes
504 p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
505 if p.returncode != 0:
506 raise VCSException("Git reset failed", p.output)
507 # Remove untracked files now, in case they're tracked in the target
508 # revision (it happens!)
509 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
510 if p.returncode != 0:
511 raise VCSException("Git clean failed", p.output)
512 if not self.refreshed:
513 # Get latest commits and tags from remote
514 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
515 if p.returncode != 0:
516 raise VCSException("Git fetch failed", p.output)
517 p = SilentPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local)
518 if p.returncode != 0:
519 raise VCSException("Git fetch failed", p.output)
520 # Recreate origin/HEAD as git clone would do it, in case it disappeared
521 p = SilentPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local)
522 if p.returncode != 0:
523 lines = p.output.splitlines()
524 if 'Multiple remote HEAD branches' not in lines[0]:
525 raise VCSException("Git remote set-head failed", p.output)
526 branch = lines[1].split(' ')[-1]
527 p2 = SilentPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local)
528 if p2.returncode != 0:
529 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
530 self.refreshed = True
531 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
532 # a github repo. Most of the time this is the same as origin/master.
533 rev = rev or 'origin/HEAD'
534 p = SilentPopen(['git', 'checkout', '-f', rev], cwd=self.local)
535 if p.returncode != 0:
536 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
537 # Get rid of any uncontrolled files left behind
538 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
539 if p.returncode != 0:
540 raise VCSException("Git clean failed", p.output)
542 def initsubmodules(self):
544 submfile = os.path.join(self.local, '.gitmodules')
545 if not os.path.isfile(submfile):
546 raise VCSException("No git submodules available")
548 # fix submodules not accessible without an account and public key auth
549 with open(submfile, 'r') as f:
550 lines = f.readlines()
551 with open(submfile, 'w') as f:
553 if 'git@github.com' in line:
554 line = line.replace('git@github.com:', 'https://github.com/')
558 ['git', 'reset', '--hard'],
559 ['git', 'clean', '-dffx'],
561 p = SilentPopen(['git', 'submodule', 'foreach', '--recursive'] + cmd, cwd=self.local)
562 if p.returncode != 0:
563 raise VCSException("Git submodule reset failed", p.output)
564 p = SilentPopen(['git', 'submodule', 'sync'], cwd=self.local)
565 if p.returncode != 0:
566 raise VCSException("Git submodule sync failed", p.output)
567 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
568 if p.returncode != 0:
569 raise VCSException("Git submodule update failed", p.output)
573 p = SilentPopen(['git', 'tag'], cwd=self.local)
574 return p.output.splitlines()
576 def latesttags(self, alltags, number):
578 p = SilentPopen(['echo "' + '\n'.join(alltags) + '" | '
579 + 'xargs -I@ git log --format=format:"%at @%n" -1 @ | '
580 + 'sort -n | awk \'{print $2}\''],
581 cwd=self.local, shell=True)
582 return p.output.splitlines()[-number:]
585 class vcs_gitsvn(vcs):
590 # Damn git-svn tries to use a graphical password prompt, so we have to
591 # trick it into taking the password from stdin
593 if self.username is None:
595 return ('echo "%s" | DISPLAY="" ' % self.password, ' --username "%s"' % self.username)
597 # If the local directory exists, but is somehow not a git repository, git
598 # will traverse up the directory tree until it finds one that is (i.e.
599 # fdroidserver) and then we'll proceed to destory it! This is called as
602 p = SilentPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
603 result = p.output.rstrip()
604 if not result.endswith(self.local):
605 raise VCSException('Repository mismatch')
607 def gotorevisionx(self, rev):
608 if not os.path.exists(self.local):
610 gitsvn_cmd = '%sgit svn clone%s' % self.userargs()
611 if ';' in self.remote:
612 remote_split = self.remote.split(';')
613 for i in remote_split[1:]:
614 if i.startswith('trunk='):
615 gitsvn_cmd += ' -T %s' % i[6:]
616 elif i.startswith('tags='):
617 gitsvn_cmd += ' -t %s' % i[5:]
618 elif i.startswith('branches='):
619 gitsvn_cmd += ' -b %s' % i[9:]
620 p = SilentPopen([gitsvn_cmd + " %s %s" % (remote_split[0], self.local)], shell=True)
621 if p.returncode != 0:
622 self.clone_failed = True
623 raise VCSException("Git clone failed", p.output)
625 p = SilentPopen([gitsvn_cmd + " %s %s" % (self.remote, self.local)], shell=True)
626 if p.returncode != 0:
627 self.clone_failed = True
628 raise VCSException("Git clone failed", p.output)
632 # Discard any working tree changes
633 p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
634 if p.returncode != 0:
635 raise VCSException("Git reset failed", p.output)
636 # Remove untracked files now, in case they're tracked in the target
637 # revision (it happens!)
638 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
639 if p.returncode != 0:
640 raise VCSException("Git clean failed", p.output)
641 if not self.refreshed:
642 # Get new commits, branches and tags from repo
643 p = SilentPopen(['%sgit svn fetch %s' % self.userargs()], cwd=self.local, shell=True)
644 if p.returncode != 0:
645 raise VCSException("Git svn fetch failed")
646 p = SilentPopen(['%sgit svn rebase %s' % self.userargs()], cwd=self.local, shell=True)
647 if p.returncode != 0:
648 raise VCSException("Git svn rebase failed", p.output)
649 self.refreshed = True
651 rev = rev or 'master'
653 nospaces_rev = rev.replace(' ', '%20')
654 # Try finding a svn tag
655 for treeish in ['origin/', '']:
656 p = SilentPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev],
658 if p.returncode == 0:
660 if p.returncode != 0:
661 # No tag found, normal svn rev translation
662 # Translate svn rev into git format
663 rev_split = rev.split('/')
666 for treeish in ['origin/', '']:
667 if len(rev_split) > 1:
668 treeish += rev_split[0]
669 svn_rev = rev_split[1]
672 # if no branch is specified, then assume trunk (i.e. 'master' branch):
676 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
678 p = SilentPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish],
680 git_rev = p.output.rstrip()
682 if p.returncode == 0 and git_rev:
685 if p.returncode != 0 or not git_rev:
686 # Try a plain git checkout as a last resort
687 p = SilentPopen(['git', 'checkout', rev], cwd=self.local)
688 if p.returncode != 0:
689 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
691 # Check out the git rev equivalent to the svn rev
692 p = SilentPopen(['git', 'checkout', git_rev], cwd=self.local)
693 if p.returncode != 0:
694 raise VCSException("Git svn checkout of '%s' failed" % rev, p.output)
696 # Get rid of any uncontrolled files left behind
697 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
698 if p.returncode != 0:
699 raise VCSException("Git clean failed", p.output)
703 for treeish in ['origin/', '']:
704 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
710 p = SilentPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local)
711 if p.returncode != 0:
713 return p.output.strip()
721 def gotorevisionx(self, rev):
722 if not os.path.exists(self.local):
723 p = SilentPopen(['hg', 'clone', self.remote, self.local])
724 if p.returncode != 0:
725 self.clone_failed = True
726 raise VCSException("Hg clone failed", p.output)
728 p = SilentPopen(['hg status -uS | xargs rm -rf'], cwd=self.local, shell=True)
729 if p.returncode != 0:
730 raise VCSException("Hg clean failed", p.output)
731 if not self.refreshed:
732 p = SilentPopen(['hg', 'pull'], cwd=self.local)
733 if p.returncode != 0:
734 raise VCSException("Hg pull failed", p.output)
735 self.refreshed = True
737 rev = rev or 'default'
740 p = SilentPopen(['hg', 'update', '-C', rev], cwd=self.local)
741 if p.returncode != 0:
742 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
743 p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
744 # Also delete untracked files, we have to enable purge extension for that:
745 if "'purge' is provided by the following extension" in p.output:
746 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
747 myfile.write("\n[extensions]\nhgext.purge=\n")
748 p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
749 if p.returncode != 0:
750 raise VCSException("HG purge failed", p.output)
751 elif p.returncode != 0:
752 raise VCSException("HG purge failed", p.output)
755 p = SilentPopen(['hg', 'tags', '-q'], cwd=self.local)
756 return p.output.splitlines()[1:]
764 def gotorevisionx(self, rev):
765 if not os.path.exists(self.local):
766 p = SilentPopen(['bzr', 'branch', self.remote, self.local])
767 if p.returncode != 0:
768 self.clone_failed = True
769 raise VCSException("Bzr branch failed", p.output)
771 p = SilentPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local)
772 if p.returncode != 0:
773 raise VCSException("Bzr revert failed", p.output)
774 if not self.refreshed:
775 p = SilentPopen(['bzr', 'pull'], cwd=self.local)
776 if p.returncode != 0:
777 raise VCSException("Bzr update failed", p.output)
778 self.refreshed = True
780 revargs = list(['-r', rev] if rev else [])
781 p = SilentPopen(['bzr', 'revert'] + revargs, cwd=self.local)
782 if p.returncode != 0:
783 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
786 p = SilentPopen(['bzr', 'tags'], cwd=self.local)
787 return [tag.split(' ')[0].strip() for tag in
788 p.output.splitlines()]
791 def retrieve_string(app_dir, string, xmlfiles=None):
794 os.path.join(app_dir, 'res'),
795 os.path.join(app_dir, 'src', 'main'),
800 for res_dir in res_dirs:
801 for r, d, f in os.walk(res_dir):
802 if os.path.basename(r) == 'values':
803 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
806 if string.startswith('@string/'):
807 string_search = re.compile(r'.*name="' + string[8:] + '".*?>([^<]+?)<.*').search
808 elif string.startswith('&') and string.endswith(';'):
809 string_search = re.compile(r'.*<!ENTITY.*' + string[1:-1] + '.*?"([^"]+?)".*>').search
811 if string_search is not None:
812 for xmlfile in xmlfiles:
813 for line in file(xmlfile):
814 matches = string_search(line)
816 return retrieve_string(app_dir, matches.group(1), xmlfiles)
819 return string.replace("\\'", "'")
822 # Return list of existing files that will be used to find the highest vercode
823 def manifest_paths(app_dir, flavour):
825 possible_manifests = \
826 [os.path.join(app_dir, 'AndroidManifest.xml'),
827 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
828 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
829 os.path.join(app_dir, 'build.gradle')]
832 possible_manifests.append(
833 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
835 return [path for path in possible_manifests if os.path.isfile(path)]
838 # Retrieve the package name. Returns the name, or None if not found.
839 def fetch_real_name(app_dir, flavour):
840 app_search = re.compile(r'.*<application.*').search
841 name_search = re.compile(r'.*android:label="([^"]+)".*').search
843 for f in manifest_paths(app_dir, flavour):
844 if not has_extension(f, 'xml'):
846 logging.debug("fetch_real_name: Checking manifest at " + f)
852 matches = name_search(line)
854 stringname = matches.group(1)
855 logging.debug("fetch_real_name: using string " + stringname)
856 result = retrieve_string(app_dir, stringname)
858 result = result.strip()
863 # Retrieve the version name
864 def version_name(original, app_dir, flavour):
865 for f in manifest_paths(app_dir, flavour):
866 if not has_extension(f, 'xml'):
868 string = retrieve_string(app_dir, original)
874 def get_library_references(root_dir):
876 proppath = os.path.join(root_dir, 'project.properties')
877 if not os.path.isfile(proppath):
879 with open(proppath) as f:
880 for line in f.readlines():
881 if not line.startswith('android.library.reference.'):
883 path = line.split('=')[1].strip()
884 relpath = os.path.join(root_dir, path)
885 if not os.path.isdir(relpath):
887 logging.debug("Found subproject at %s" % path)
888 libraries.append(path)
892 def ant_subprojects(root_dir):
893 subprojects = get_library_references(root_dir)
894 for subpath in subprojects:
895 subrelpath = os.path.join(root_dir, subpath)
896 for p in get_library_references(subrelpath):
897 relp = os.path.normpath(os.path.join(subpath, p))
898 if relp not in subprojects:
899 subprojects.insert(0, relp)
903 def remove_debuggable_flags(root_dir):
904 # Remove forced debuggable flags
905 logging.debug("Removing debuggable flags from %s" % root_dir)
906 for root, dirs, files in os.walk(root_dir):
907 if 'AndroidManifest.xml' in files:
908 path = os.path.join(root, 'AndroidManifest.xml')
909 p = SilentPopen(['sed', '-i', 's/android:debuggable="[^"]*"//g', path])
910 if p.returncode != 0:
911 raise BuildException("Failed to remove debuggable flags of %s" % path)
914 # Extract some information from the AndroidManifest.xml at the given path.
915 # Returns (version, vercode, package), any or all of which might be None.
916 # All values returned are strings.
917 def parse_androidmanifests(paths, ignoreversions=None):
920 return (None, None, None)
922 vcsearch = re.compile(r'.*:versionCode="([0-9]+?)".*').search
923 vnsearch = re.compile(r'.*:versionName="([^"]+?)".*').search
924 psearch = re.compile(r'.*package="([^"]+)".*').search
926 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
927 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
928 psearch_g = re.compile(r'.*packageName *=* *["\']([^"]+)["\'].*').search
930 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
938 gradle = has_extension(path, 'gradle')
941 # Remember package name, may be defined separately from version+vercode
942 package = max_package
944 for line in file(path):
947 matches = psearch_g(line)
949 matches = psearch(line)
951 package = matches.group(1)
954 matches = vnsearch_g(line)
956 matches = vnsearch(line)
958 version = matches.group(2 if gradle else 1)
961 matches = vcsearch_g(line)
963 matches = vcsearch(line)
965 vercode = matches.group(1)
967 # Always grab the package name and version name in case they are not
968 # together with the highest version code
969 if max_package is None and package is not None:
970 max_package = package
971 if max_version is None and version is not None:
972 max_version = version
974 if max_vercode is None or (vercode is not None and vercode > max_vercode):
975 if not ignoresearch or not ignoresearch(version):
976 if version is not None:
977 max_version = version
978 if vercode is not None:
979 max_vercode = vercode
980 if package is not None:
981 max_package = package
983 max_version = "Ignore"
985 if max_version is None:
986 max_version = "Unknown"
988 return (max_version, max_vercode, max_package)
991 class FDroidException(Exception):
992 def __init__(self, value, detail=None):
996 def get_wikitext(self):
997 ret = repr(self.value) + "\n"
1001 txt = self.detail[-8192:] if len(self.detail) > 8192 else self.detail
1009 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1013 class VCSException(FDroidException):
1017 class BuildException(FDroidException):
1021 # Get the specified source library.
1022 # Returns the path to it. Normally this is the path to be used when referencing
1023 # it, which may be a subdirectory of the actual project. If you want the base
1024 # directory of the project, pass 'basepath=True'.
1025 def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
1026 basepath=False, raw=False, prepare=True, preponly=False):
1034 name, ref = spec.split('@')
1036 number, name = name.split(':', 1)
1038 name, subdir = name.split('/', 1)
1040 if name not in metadata.srclibs:
1041 raise VCSException('srclib ' + name + ' not found.')
1043 srclib = metadata.srclibs[name]
1045 sdir = os.path.join(srclib_dir, name)
1048 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1049 vcs.srclib = (name, number, sdir)
1051 vcs.gotorevision(ref)
1058 libdir = os.path.join(sdir, subdir)
1059 elif srclib["Subdir"]:
1060 for subdir in srclib["Subdir"]:
1061 libdir_candidate = os.path.join(sdir, subdir)
1062 if os.path.exists(libdir_candidate):
1063 libdir = libdir_candidate
1069 if srclib["Srclibs"]:
1071 for lib in srclib["Srclibs"].replace(';', ',').split(','):
1073 for t in srclibpaths:
1078 raise VCSException('Missing recursive srclib %s for %s' % (
1080 place_srclib(libdir, n, s_tuple[2])
1083 remove_signing_keys(sdir)
1084 remove_debuggable_flags(sdir)
1088 if srclib["Prepare"]:
1089 cmd = replace_config_vars(srclib["Prepare"])
1091 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1092 if p.returncode != 0:
1093 raise BuildException("Error running prepare command for srclib %s"
1099 return (name, number, libdir)
1102 # Prepare the source code for a particular build
1103 # 'vcs' - the appropriate vcs object for the application
1104 # 'app' - the application details from the metadata
1105 # 'build' - the build details from the metadata
1106 # 'build_dir' - the path to the build directory, usually
1108 # 'srclib_dir' - the path to the source libraries directory, usually
1110 # 'extlib_dir' - the path to the external libraries directory, usually
1112 # Returns the (root, srclibpaths) where:
1113 # 'root' is the root directory, which may be the same as 'build_dir' or may
1114 # be a subdirectory of it.
1115 # 'srclibpaths' is information on the srclibs being used
1116 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False):
1118 # Optionally, the actual app source can be in a subdirectory
1120 root_dir = os.path.join(build_dir, build['subdir'])
1122 root_dir = build_dir
1124 # Get a working copy of the right revision
1125 logging.info("Getting source for revision " + build['commit'])
1126 vcs.gotorevision(build['commit'])
1128 # Initialise submodules if requred
1129 if build['submodules']:
1130 logging.info("Initialising submodules")
1131 vcs.initsubmodules()
1133 # Check that a subdir (if we're using one) exists. This has to happen
1134 # after the checkout, since it might not exist elsewhere
1135 if not os.path.exists(root_dir):
1136 raise BuildException('Missing subdir ' + root_dir)
1138 # Run an init command if one is required
1140 cmd = replace_config_vars(build['init'])
1141 logging.info("Running 'init' commands in %s" % root_dir)
1143 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1144 if p.returncode != 0:
1145 raise BuildException("Error running init command for %s:%s" %
1146 (app['id'], build['version']), p.output)
1148 # Apply patches if any
1150 logging.info("Applying patches")
1151 for patch in build['patch']:
1152 patch = patch.strip()
1153 logging.info("Applying " + patch)
1154 patch_path = os.path.join('metadata', app['id'], patch)
1155 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1156 if p.returncode != 0:
1157 raise BuildException("Failed to apply patch %s" % patch_path)
1159 # Get required source libraries
1161 if build['srclibs']:
1162 logging.info("Collecting source libraries")
1163 for lib in build['srclibs']:
1164 srclibpaths.append(getsrclib(lib, srclib_dir, srclibpaths,
1167 for name, number, libpath in srclibpaths:
1168 place_srclib(root_dir, int(number) if number else None, libpath)
1170 basesrclib = vcs.getsrclib()
1171 # If one was used for the main source, add that too.
1173 srclibpaths.append(basesrclib)
1175 # Update the local.properties file
1176 localprops = [os.path.join(build_dir, 'local.properties')]
1178 localprops += [os.path.join(root_dir, 'local.properties')]
1179 for path in localprops:
1180 if not os.path.isfile(path):
1182 logging.info("Updating properties file at %s" % path)
1187 # Fix old-fashioned 'sdk-location' by copying
1188 # from sdk.dir, if necessary
1189 if build['oldsdkloc']:
1190 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1191 re.S | re.M).group(1)
1192 props += "sdk-location=%s\n" % sdkloc
1194 props += "sdk.dir=%s\n" % config['sdk_path']
1195 props += "sdk-location=%s\n" % config['sdk_path']
1196 if 'ndk_path' in config:
1198 props += "ndk.dir=%s\n" % config['ndk_path']
1199 props += "ndk-location=%s\n" % config['ndk_path']
1200 # Add java.encoding if necessary
1201 if build['encoding']:
1202 props += "java.encoding=%s\n" % build['encoding']
1208 if build['type'] == 'gradle':
1209 flavour = build['gradle']
1210 if flavour in ['main', 'yes', '']:
1213 version_regex = re.compile(r".*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1214 gradlepluginver = None
1216 gradle_files = [os.path.join(root_dir, 'build.gradle')]
1218 # Parent dir build.gradle
1219 parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
1220 if parent_dir.startswith(build_dir):
1221 gradle_files.append(os.path.join(parent_dir, 'build.gradle'))
1223 for path in gradle_files:
1226 if not os.path.isfile(path):
1228 with open(path) as f:
1230 match = version_regex.match(line)
1232 gradlepluginver = match.group(1)
1236 build['gradlepluginver'] = LooseVersion(gradlepluginver)
1238 logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
1239 build['gradlepluginver'] = LooseVersion('0.11')
1242 n = build["target"].split('-')[1]
1243 SilentPopen(['sed', '-i',
1244 's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g',
1248 # Remove forced debuggable flags
1249 remove_debuggable_flags(root_dir)
1251 # Insert version code and number into the manifest if necessary
1252 if build['forceversion']:
1253 logging.info("Changing the version name")
1254 for path in manifest_paths(root_dir, flavour):
1255 if not os.path.isfile(path):
1257 if has_extension(path, 'xml'):
1258 p = SilentPopen(['sed', '-i',
1259 's/android:versionName="[^"]*"/android:versionName="'
1260 + build['version'] + '"/g',
1262 if p.returncode != 0:
1263 raise BuildException("Failed to amend manifest")
1264 elif has_extension(path, 'gradle'):
1265 p = SilentPopen(['sed', '-i',
1266 's/versionName *=* *"[^"]*"/versionName = "'
1267 + build['version'] + '"/g',
1269 if p.returncode != 0:
1270 raise BuildException("Failed to amend build.gradle")
1271 if build['forcevercode']:
1272 logging.info("Changing the version code")
1273 for path in manifest_paths(root_dir, flavour):
1274 if not os.path.isfile(path):
1276 if has_extension(path, 'xml'):
1277 p = SilentPopen(['sed', '-i',
1278 's/android:versionCode="[^"]*"/android:versionCode="'
1279 + build['vercode'] + '"/g',
1281 if p.returncode != 0:
1282 raise BuildException("Failed to amend manifest")
1283 elif has_extension(path, 'gradle'):
1284 p = SilentPopen(['sed', '-i',
1285 's/versionCode *=* *[0-9]*/versionCode = '
1286 + build['vercode'] + '/g',
1288 if p.returncode != 0:
1289 raise BuildException("Failed to amend build.gradle")
1291 # Delete unwanted files
1293 logging.info("Removing specified files")
1294 for part in getpaths(build_dir, build, 'rm'):
1295 dest = os.path.join(build_dir, part)
1296 logging.info("Removing {0}".format(part))
1297 if os.path.lexists(dest):
1298 if os.path.islink(dest):
1299 SilentPopen(['unlink ' + dest], shell=True)
1301 SilentPopen(['rm -rf ' + dest], shell=True)
1303 logging.info("...but it didn't exist")
1305 remove_signing_keys(build_dir)
1307 # Add required external libraries
1308 if build['extlibs']:
1309 logging.info("Collecting prebuilt libraries")
1310 libsdir = os.path.join(root_dir, 'libs')
1311 if not os.path.exists(libsdir):
1313 for lib in build['extlibs']:
1315 logging.info("...installing extlib {0}".format(lib))
1316 libf = os.path.basename(lib)
1317 libsrc = os.path.join(extlib_dir, lib)
1318 if not os.path.exists(libsrc):
1319 raise BuildException("Missing extlib file {0}".format(libsrc))
1320 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1322 # Run a pre-build command if one is required
1323 if build['prebuild']:
1324 logging.info("Running 'prebuild' commands in %s" % root_dir)
1326 cmd = replace_config_vars(build['prebuild'])
1328 # Substitute source library paths into prebuild commands
1329 for name, number, libpath in srclibpaths:
1330 libpath = os.path.relpath(libpath, root_dir)
1331 cmd = cmd.replace('$$' + name + '$$', libpath)
1333 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1334 if p.returncode != 0:
1335 raise BuildException("Error running prebuild command for %s:%s" %
1336 (app['id'], build['version']), p.output)
1338 # Generate (or update) the ant build file, build.xml...
1339 if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
1340 parms = [config['android'], 'update', 'lib-project']
1341 lparms = [config['android'], 'update', 'project']
1344 parms += ['-t', build['target']]
1345 lparms += ['-t', build['target']]
1346 if build['update'] == ['auto']:
1347 update_dirs = ant_subprojects(root_dir) + ['.']
1349 update_dirs = build['update']
1351 for d in update_dirs:
1352 subdir = os.path.join(root_dir, d)
1354 logging.debug("Updating main project")
1355 cmd = parms + ['-p', d]
1357 logging.debug("Updating subproject %s" % d)
1358 cmd = lparms + ['-p', d]
1359 p = FDroidPopen(cmd, cwd=root_dir)
1360 # Check to see whether an error was returned without a proper exit
1361 # code (this is the case for the 'no target set or target invalid'
1363 if p.returncode != 0 or p.output.startswith("Error: "):
1364 raise BuildException("Failed to update project at %s" % d, p.output)
1365 # Clean update dirs via ant
1367 logging.info("Cleaning subproject %s" % d)
1368 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1370 return (root_dir, srclibpaths)
1373 # Split and extend via globbing the paths from a field
1374 def getpaths(build_dir, build, field):
1376 for p in build[field]:
1378 full_path = os.path.join(build_dir, p)
1379 full_path = os.path.normpath(full_path)
1380 paths += [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1384 # Scan the source code in the given directory (and all subdirectories)
1385 # and return the number of fatal problems encountered
1386 def scan_source(build_dir, root_dir, thisbuild):
1390 # Common known non-free blobs (always lower case):
1392 re.compile(r'flurryagent', re.IGNORECASE),
1393 re.compile(r'paypal.*mpl', re.IGNORECASE),
1394 re.compile(r'google.*analytics', re.IGNORECASE),
1395 re.compile(r'admob.*sdk.*android', re.IGNORECASE),
1396 re.compile(r'google.*ad.*view', re.IGNORECASE),
1397 re.compile(r'google.*admob', re.IGNORECASE),
1398 re.compile(r'google.*play.*services', re.IGNORECASE),
1399 re.compile(r'crittercism', re.IGNORECASE),
1400 re.compile(r'heyzap', re.IGNORECASE),
1401 re.compile(r'jpct.*ae', re.IGNORECASE),
1402 re.compile(r'youtube.*android.*player.*api', re.IGNORECASE),
1403 re.compile(r'bugsense', re.IGNORECASE),
1404 re.compile(r'crashlytics', re.IGNORECASE),
1405 re.compile(r'ouya.*sdk', re.IGNORECASE),
1406 re.compile(r'libspen23', re.IGNORECASE),
1409 scanignore = getpaths(build_dir, thisbuild, 'scanignore')
1410 scandelete = getpaths(build_dir, thisbuild, 'scandelete')
1413 ms = magic.open(magic.MIME_TYPE)
1415 except AttributeError:
1419 for i in scanignore:
1420 if fd.startswith(i):
1425 for i in scandelete:
1426 if fd.startswith(i):
1430 def removeproblem(what, fd, fp):
1431 logging.info('Removing %s at %s' % (what, fd))
1434 def warnproblem(what, fd):
1435 logging.warn('Found %s at %s' % (what, fd))
1437 def handleproblem(what, fd, fp):
1439 removeproblem(what, fd, fp)
1441 logging.error('Found %s at %s' % (what, fd))
1445 # Iterate through all files in the source code
1446 for r, d, f in os.walk(build_dir, topdown=True):
1448 # It's topdown, so checking the basename is enough
1449 for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
1455 # Path (relative) to the file
1456 fp = os.path.join(r, curfile)
1457 fd = fp[len(build_dir) + 1:]
1459 # Check if this file has been explicitly excluded from scanning
1464 mime = magic.from_file(fp, mime=True) if ms is None else ms.file(fp)
1465 except UnicodeError:
1466 warnproblem('malformed magic number', fd, fp)
1468 if mime == 'application/x-sharedlib':
1469 count += handleproblem('shared library', fd, fp)
1471 elif mime == 'application/x-archive':
1472 count += handleproblem('static library', fd, fp)
1474 elif mime == 'application/x-executable':
1475 count += handleproblem('binary executable', fd, fp)
1477 elif mime == 'application/x-java-applet':
1478 count += handleproblem('Java compiled class', fd, fp)
1483 'application/java-archive',
1484 'application/octet-stream',
1488 if has_extension(fp, 'apk'):
1489 removeproblem('APK file', fd, fp)
1491 elif has_extension(fp, 'jar'):
1493 if any(suspect.match(curfile) for suspect in usual_suspects):
1494 count += handleproblem('usual supect', fd, fp)
1496 warnproblem('JAR file', fd)
1498 elif has_extension(fp, 'zip'):
1499 warnproblem('ZIP file', fd)
1502 warnproblem('unknown compressed or binary file', fd)
1504 elif has_extension(fp, 'java'):
1505 for line in file(fp):
1506 if 'DexClassLoader' in line:
1507 count += handleproblem('DexClassLoader', fd, fp)
1512 # Presence of a jni directory without buildjni=yes might
1513 # indicate a problem (if it's not a problem, explicitly use
1514 # buildjni=no to bypass this check)
1515 if (os.path.exists(os.path.join(root_dir, 'jni')) and
1516 not thisbuild['buildjni']):
1517 logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
1526 self.path = os.path.join('stats', 'known_apks.txt')
1528 if os.path.exists(self.path):
1529 for line in file(self.path):
1530 t = line.rstrip().split(' ')
1532 self.apks[t[0]] = (t[1], None)
1534 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1535 self.changed = False
1537 def writeifchanged(self):
1539 if not os.path.exists('stats'):
1541 f = open(self.path, 'w')
1543 for apk, app in self.apks.iteritems():
1545 line = apk + ' ' + appid
1547 line += ' ' + time.strftime('%Y-%m-%d', added)
1549 for line in sorted(lst):
1550 f.write(line + '\n')
1553 # Record an apk (if it's new, otherwise does nothing)
1554 # Returns the date it was added.
1555 def recordapk(self, apk, app):
1556 if apk not in self.apks:
1557 self.apks[apk] = (app, time.gmtime(time.time()))
1559 _, added = self.apks[apk]
1562 # Look up information - given the 'apkname', returns (app id, date added/None).
1563 # Or returns None for an unknown apk.
1564 def getapp(self, apkname):
1565 if apkname in self.apks:
1566 return self.apks[apkname]
1569 # Get the most recent 'num' apps added to the repo, as a list of package ids
1570 # with the most recent first.
1571 def getlatest(self, num):
1573 for apk, app in self.apks.iteritems():
1577 if apps[appid] > added:
1581 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1582 lst = [app for app, _ in sortedapps]
1587 def isApkDebuggable(apkfile, config):
1588 """Returns True if the given apk file is debuggable
1590 :param apkfile: full path to the apk to check"""
1592 p = SilentPopen([os.path.join(config['sdk_path'], 'build-tools',
1593 config['build_tools'], 'aapt'),
1594 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'])
1595 if p.returncode != 0:
1596 logging.critical("Failed to get apk manifest information")
1598 for line in p.output.splitlines():
1599 if 'android:debuggable' in line and not line.endswith('0x0'):
1604 class AsynchronousFileReader(threading.Thread):
1606 Helper class to implement asynchronous reading of a file
1607 in a separate thread. Pushes read lines on a queue to
1608 be consumed in another thread.
1611 def __init__(self, fd, queue):
1612 assert isinstance(queue, Queue.Queue)
1613 assert callable(fd.readline)
1614 threading.Thread.__init__(self)
1619 '''The body of the tread: read lines and put them on the queue.'''
1620 for line in iter(self._fd.readline, ''):
1621 self._queue.put(line)
1624 '''Check whether there is no more content to expect.'''
1625 return not self.is_alive() and self._queue.empty()
1633 def SilentPopen(commands, cwd=None, shell=False):
1634 return FDroidPopen(commands, cwd=cwd, shell=shell, output=False)
1637 def FDroidPopen(commands, cwd=None, shell=False, output=True):
1639 Run a command and capture the possibly huge output.
1641 :param commands: command and argument list like in subprocess.Popen
1642 :param cwd: optionally specifies a working directory
1643 :returns: A PopenResult.
1649 cwd = os.path.normpath(cwd)
1650 logging.debug("Directory: %s" % cwd)
1651 logging.debug("> %s" % ' '.join(commands))
1653 result = PopenResult()
1654 p = subprocess.Popen(commands, cwd=cwd, shell=shell, env=env,
1655 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1657 stdout_queue = Queue.Queue()
1658 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1659 stdout_reader.start()
1661 # Check the queue for output (until there is no more to get)
1662 while not stdout_reader.eof():
1663 while not stdout_queue.empty():
1664 line = stdout_queue.get()
1665 if output and options.verbose:
1666 # Output directly to console
1667 sys.stderr.write(line)
1669 result.output += line
1674 result.returncode = p.returncode
1678 def remove_signing_keys(build_dir):
1679 comment = re.compile(r'[ ]*//')
1680 signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1682 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1683 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1684 re.compile(r'.*variant\.outputFile = .*'),
1685 re.compile(r'.*\.readLine\(.*'),
1687 for root, dirs, files in os.walk(build_dir):
1688 if 'build.gradle' in files:
1689 path = os.path.join(root, 'build.gradle')
1691 with open(path, "r") as o:
1692 lines = o.readlines()
1697 with open(path, "w") as o:
1699 if comment.match(line):
1703 opened += line.count('{')
1704 opened -= line.count('}')
1707 if signing_configs.match(line):
1712 if any(s.match(line) for s in line_matches):
1720 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1723 'project.properties',
1725 'default.properties',
1728 if propfile in files:
1729 path = os.path.join(root, propfile)
1731 with open(path, "r") as o:
1732 lines = o.readlines()
1736 with open(path, "w") as o:
1738 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1745 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1748 def replace_config_vars(cmd):
1749 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1750 cmd = cmd.replace('$$NDK$$', config['ndk_path'])
1751 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1755 def place_srclib(root_dir, number, libpath):
1758 relpath = os.path.relpath(libpath, root_dir)
1759 proppath = os.path.join(root_dir, 'project.properties')
1762 if os.path.isfile(proppath):
1763 with open(proppath, "r") as o:
1764 lines = o.readlines()
1766 with open(proppath, "w") as o:
1769 if line.startswith('android.library.reference.%d=' % number):
1770 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1775 o.write('android.library.reference.%d=%s\n' % (number, relpath))