1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
33 from distutils.version import LooseVersion
42 def get_default_config():
44 'sdk_path': os.getenv("ANDROID_HOME") or "",
45 'ndk_path': os.getenv("ANDROID_NDK") or "",
46 'build_tools': "20.0.0",
50 'sync_from_local_copy_dir': False,
51 'update_stats': False,
52 'stats_to_carbon': False,
54 'build_server_always': False,
55 'keystore': os.path.join(os.getenv("HOME"), '.local', 'share', 'fdroidserver', 'keystore.jks'),
56 'smartcardoptions': [],
62 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
63 'repo_name': "My First FDroid Repo Demo",
64 'repo_icon': "fdroid-icon.png",
65 'repo_description': '''
66 This is a repository of apps to be used with FDroid. Applications in this
67 repository are either official binaries built by the original application
68 developers, or are binaries built from source by the admin of f-droid.org
69 using the tools on https://gitlab.com/u/fdroid.
75 def read_config(opts, config_file='config.py'):
76 """Read the repository config
78 The config is read from config_file, which is in the current directory when
79 any of the repo management commands are used.
81 global config, options, env
83 if config is not None:
85 if not os.path.isfile(config_file):
86 logging.critical("Missing config file - is this a repo directory?")
93 logging.debug("Reading %s" % config_file)
94 execfile(config_file, config)
96 # smartcardoptions must be a list since its command line args for Popen
97 if 'smartcardoptions' in config:
98 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
99 elif 'keystore' in config and config['keystore'] == 'NONE':
100 # keystore='NONE' means use smartcard, these are required defaults
101 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
102 'SunPKCS11-OpenSC', '-providerClass',
103 'sun.security.pkcs11.SunPKCS11',
104 '-providerArg', 'opensc-fdroid.cfg']
106 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
107 st = os.stat(config_file)
108 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
109 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
111 defconfig = get_default_config()
112 for k, v in defconfig.items():
116 # Expand environment variables
117 for k, v in config.items():
120 v = os.path.expanduser(v)
121 config[k] = os.path.expandvars(v)
123 if not test_sdk_exists(config):
126 if not test_build_tools_exists(config):
131 os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
134 os.path.join(config['sdk_path'], 'tools', 'zipalign'),
135 os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'zipalign'),
138 os.path.join(config['sdk_path'], 'tools', 'android'),
141 os.path.join(config['sdk_path'], 'platform-tools', 'adb'),
145 for b, paths in bin_paths.items():
148 if os.path.isfile(path):
151 if config[b] is None:
152 logging.warn("Could not find %s in any of the following paths:\n%s" % (
153 b, '\n'.join(paths)))
155 # There is no standard, so just set up the most common environment
158 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
159 env[n] = config['sdk_path']
160 for n in ['ANDROID_NDK', 'NDK']:
161 env[n] = config['ndk_path']
163 for k in ["keystorepass", "keypass"]:
165 write_password_file(k)
167 for k in ["repo_description", "archive_description"]:
169 config[k] = clean_description(config[k])
171 # since this is used with rsync, where trailing slashes have meaning,
172 # ensure there is always a trailing slash
173 if 'serverwebroot' in config:
174 if config['serverwebroot'][-1] != '/':
175 config['serverwebroot'] += '/'
176 config['serverwebroot'] = config['serverwebroot'].replace('//', '/')
181 def test_sdk_exists(c):
182 if c['sdk_path'] is None:
183 # c['sdk_path'] is set to the value of ANDROID_HOME by default
184 logging.error('No Android SDK found! ANDROID_HOME is not set and sdk_path is not in config.py!')
185 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
186 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
188 if not os.path.exists(c['sdk_path']):
189 logging.critical('Android SDK path "' + c['sdk_path'] + '" does not exist!')
191 if not os.path.isdir(c['sdk_path']):
192 logging.critical('Android SDK path "' + c['sdk_path'] + '" is not a directory!')
194 for d in ['build-tools', 'platform-tools', 'tools']:
195 if not os.path.isdir(os.path.join(c['sdk_path'], d)):
196 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
202 def test_build_tools_exists(c):
203 if not test_sdk_exists(c):
205 build_tools = os.path.join(c['sdk_path'], 'build-tools')
206 versioned_build_tools = os.path.join(build_tools, c['build_tools'])
207 if not os.path.isdir(versioned_build_tools):
208 logging.critical('Android Build Tools path "'
209 + versioned_build_tools + '" does not exist!')
214 def write_password_file(pwtype, password=None):
216 writes out passwords to a protected file instead of passing passwords as
217 command line argments
219 filename = '.fdroid.' + pwtype + '.txt'
220 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
222 os.write(fd, config[pwtype])
224 os.write(fd, password)
226 config[pwtype + 'file'] = filename
229 # Given the arguments in the form of multiple appid:[vc] strings, this returns
230 # a dictionary with the set of vercodes specified for each package.
231 def read_pkg_args(args, allow_vercodes=False):
238 if allow_vercodes and ':' in p:
239 package, vercode = p.split(':')
241 package, vercode = p, None
242 if package not in vercodes:
243 vercodes[package] = [vercode] if vercode else []
245 elif vercode and vercode not in vercodes[package]:
246 vercodes[package] += [vercode] if vercode else []
251 # On top of what read_pkg_args does, this returns the whole app metadata, but
252 # limiting the builds list to the builds matching the vercodes specified.
253 def read_app_args(args, allapps, allow_vercodes=False):
255 vercodes = read_pkg_args(args, allow_vercodes)
260 apps = [app for app in allapps if app['id'] in vercodes]
262 if len(apps) != len(vercodes):
263 allids = [app["id"] for app in allapps]
266 logging.critical("No such package: %s" % p)
267 raise FDroidException("Found invalid app ids in arguments")
269 raise FDroidException("No packages specified")
273 vc = vercodes[app['id']]
276 app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
277 if len(app['builds']) != len(vercodes[app['id']]):
279 allvcs = [b['vercode'] for b in app['builds']]
280 for v in vercodes[app['id']]:
282 logging.critical("No such vercode %s for app %s" % (v, app['id']))
285 raise FDroidException("Found invalid vercodes for some apps")
290 def has_extension(filename, extension):
291 name, ext = os.path.splitext(filename)
292 ext = ext.lower()[1:]
293 return ext == extension
298 def clean_description(description):
299 'Remove unneeded newlines and spaces from a block of description text'
301 # this is split up by paragraph to make removing the newlines easier
302 for paragraph in re.split(r'\n\n', description):
303 paragraph = re.sub('\r', '', paragraph)
304 paragraph = re.sub('\n', ' ', paragraph)
305 paragraph = re.sub(' {2,}', ' ', paragraph)
306 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
307 returnstring += paragraph + '\n\n'
308 return returnstring.rstrip('\n')
311 def apknameinfo(filename):
313 filename = os.path.basename(filename)
314 if apk_regex is None:
315 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
316 m = apk_regex.match(filename)
318 result = (m.group(1), m.group(2))
319 except AttributeError:
320 raise FDroidException("Invalid apk name: %s" % filename)
324 def getapkname(app, build):
325 return "%s_%s.apk" % (app['id'], build['vercode'])
328 def getsrcname(app, build):
329 return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
336 return app['Auto Name']
341 return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
344 def getvcs(vcstype, remote, local):
346 return vcs_git(remote, local)
348 return vcs_svn(remote, local)
349 if vcstype == 'git-svn':
350 return vcs_gitsvn(remote, local)
352 return vcs_hg(remote, local)
354 return vcs_bzr(remote, local)
355 if vcstype == 'srclib':
356 if local != 'build/srclib/' + remote:
357 raise VCSException("Error: srclib paths are hard-coded!")
358 return getsrclib(remote, 'build/srclib', raw=True)
359 raise VCSException("Invalid vcs type " + vcstype)
362 def getsrclibvcs(name):
363 if name not in metadata.srclibs:
364 raise VCSException("Missing srclib " + name)
365 return metadata.srclibs[name]['Repo Type']
369 def __init__(self, remote, local):
371 # svn, git-svn and bzr may require auth
373 if self.repotype() in ('svn', 'git-svn', 'bzr'):
375 self.username, remote = remote.split('@')
376 if ':' not in self.username:
377 raise VCSException("Password required with username")
378 self.username, self.password = self.username.split(':')
382 self.clone_failed = False
383 self.refreshed = False
389 # Take the local repository to a clean version of the given revision, which
390 # is specificed in the VCS's native format. Beforehand, the repository can
391 # be dirty, or even non-existent. If the repository does already exist
392 # locally, it will be updated from the origin, but only once in the
393 # lifetime of the vcs object.
394 # None is acceptable for 'rev' if you know you are cloning a clean copy of
395 # the repo - otherwise it must specify a valid revision.
396 def gotorevision(self, rev):
398 if self.clone_failed:
399 raise VCSException("Downloading the repository already failed once, not trying again.")
401 # The .fdroidvcs-id file for a repo tells us what VCS type
402 # and remote that directory was created from, allowing us to drop it
403 # automatically if either of those things changes.
404 fdpath = os.path.join(self.local, '..',
405 '.fdroidvcs-' + os.path.basename(self.local))
406 cdata = self.repotype() + ' ' + self.remote
409 if os.path.exists(self.local):
410 if os.path.exists(fdpath):
411 with open(fdpath, 'r') as f:
412 fsdata = f.read().strip()
418 "Repository details for %s changed - deleting" % (
422 logging.info("Repository details for %s missing - deleting" % (
425 shutil.rmtree(self.local)
430 self.gotorevisionx(rev)
431 except FDroidException, e:
434 # If necessary, write the .fdroidvcs file.
435 if writeback and not self.clone_failed:
436 with open(fdpath, 'w') as f:
442 # Derived classes need to implement this. It's called once basic checking
443 # has been performend.
444 def gotorevisionx(self, rev):
445 raise VCSException("This VCS type doesn't define gotorevisionx")
447 # Initialise and update submodules
448 def initsubmodules(self):
449 raise VCSException('Submodules not supported for this vcs type')
451 # Get a list of all known tags
453 raise VCSException('gettags not supported for this vcs type')
455 # Get a list of latest number tags
456 def latesttags(self, number):
457 raise VCSException('latesttags not supported for this vcs type')
459 # Get current commit reference (hash, revision, etc)
461 raise VCSException('getref not supported for this vcs type')
463 # Returns the srclib (name, path) used in setting up the current
474 # If the local directory exists, but is somehow not a git repository, git
475 # will traverse up the directory tree until it finds one that is (i.e.
476 # fdroidserver) and then we'll proceed to destroy it! This is called as
479 p = SilentPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
480 result = p.output.rstrip()
481 if not result.endswith(self.local):
482 raise VCSException('Repository mismatch')
484 def gotorevisionx(self, rev):
485 if not os.path.exists(self.local):
487 p = FDroidPopen(['git', 'clone', self.remote, self.local])
488 if p.returncode != 0:
489 self.clone_failed = True
490 raise VCSException("Git clone failed", p.output)
494 # Discard any working tree changes
495 p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
496 if p.returncode != 0:
497 raise VCSException("Git reset failed", p.output)
498 # Remove untracked files now, in case they're tracked in the target
499 # revision (it happens!)
500 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
501 if p.returncode != 0:
502 raise VCSException("Git clean failed", p.output)
503 if not self.refreshed:
504 # Get latest commits and tags from remote
505 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
506 if p.returncode != 0:
507 raise VCSException("Git fetch failed", p.output)
508 p = SilentPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local)
509 if p.returncode != 0:
510 raise VCSException("Git fetch failed", p.output)
511 # Recreate origin/HEAD as git clone would do it, in case it disappeared
512 p = SilentPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local)
513 if p.returncode != 0:
514 lines = p.output.splitlines()
515 if 'Multiple remote HEAD branches' not in lines[0]:
516 raise VCSException("Git remote set-head failed", p.output)
517 branch = lines[1].split(' ')[-1]
518 p2 = SilentPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local)
519 if p2.returncode != 0:
520 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
521 self.refreshed = True
522 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
523 # a github repo. Most of the time this is the same as origin/master.
524 rev = rev or 'origin/HEAD'
525 p = SilentPopen(['git', 'checkout', '-f', rev], cwd=self.local)
526 if p.returncode != 0:
527 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
528 # Get rid of any uncontrolled files left behind
529 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
530 if p.returncode != 0:
531 raise VCSException("Git clean failed", p.output)
533 def initsubmodules(self):
535 submfile = os.path.join(self.local, '.gitmodules')
536 if not os.path.isfile(submfile):
537 raise VCSException("No git submodules available")
539 # fix submodules not accessible without an account and public key auth
540 with open(submfile, 'r') as f:
541 lines = f.readlines()
542 with open(submfile, 'w') as f:
544 if 'git@github.com' in line:
545 line = line.replace('git@github.com:', 'https://github.com/')
549 ['git', 'reset', '--hard'],
550 ['git', 'clean', '-dffx'],
552 p = SilentPopen(['git', 'submodule', 'foreach', '--recursive'] + cmd, cwd=self.local)
553 if p.returncode != 0:
554 raise VCSException("Git submodule reset failed", p.output)
555 p = SilentPopen(['git', 'submodule', 'sync'], cwd=self.local)
556 if p.returncode != 0:
557 raise VCSException("Git submodule sync failed", p.output)
558 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
559 if p.returncode != 0:
560 raise VCSException("Git submodule update failed", p.output)
564 p = SilentPopen(['git', 'tag'], cwd=self.local)
565 return p.output.splitlines()
567 def latesttags(self, alltags, number):
569 p = SilentPopen(['echo "' + '\n'.join(alltags) + '" | '
570 + 'xargs -I@ git log --format=format:"%at @%n" -1 @ | '
571 + 'sort -n | awk \'{print $2}\''],
572 cwd=self.local, shell=True)
573 return p.output.splitlines()[-number:]
576 class vcs_gitsvn(vcs):
581 # Damn git-svn tries to use a graphical password prompt, so we have to
582 # trick it into taking the password from stdin
584 if self.username is None:
586 return ('echo "%s" | DISPLAY="" ' % self.password, ' --username "%s"' % self.username)
588 # If the local directory exists, but is somehow not a git repository, git
589 # will traverse up the directory tree until it finds one that is (i.e.
590 # fdroidserver) and then we'll proceed to destory it! This is called as
593 p = SilentPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
594 result = p.output.rstrip()
595 if not result.endswith(self.local):
596 raise VCSException('Repository mismatch')
598 def gotorevisionx(self, rev):
599 if not os.path.exists(self.local):
601 gitsvn_cmd = '%sgit svn clone%s' % self.userargs()
602 if ';' in self.remote:
603 remote_split = self.remote.split(';')
604 for i in remote_split[1:]:
605 if i.startswith('trunk='):
606 gitsvn_cmd += ' -T %s' % i[6:]
607 elif i.startswith('tags='):
608 gitsvn_cmd += ' -t %s' % i[5:]
609 elif i.startswith('branches='):
610 gitsvn_cmd += ' -b %s' % i[9:]
611 p = SilentPopen([gitsvn_cmd + " %s %s" % (remote_split[0], self.local)], shell=True)
612 if p.returncode != 0:
613 self.clone_failed = True
614 raise VCSException("Git clone failed", p.output)
616 p = SilentPopen([gitsvn_cmd + " %s %s" % (self.remote, self.local)], shell=True)
617 if p.returncode != 0:
618 self.clone_failed = True
619 raise VCSException("Git clone failed", p.output)
623 # Discard any working tree changes
624 p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
625 if p.returncode != 0:
626 raise VCSException("Git reset failed", p.output)
627 # Remove untracked files now, in case they're tracked in the target
628 # revision (it happens!)
629 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
630 if p.returncode != 0:
631 raise VCSException("Git clean failed", p.output)
632 if not self.refreshed:
633 # Get new commits, branches and tags from repo
634 p = SilentPopen(['%sgit svn fetch %s' % self.userargs()], cwd=self.local, shell=True)
635 if p.returncode != 0:
636 raise VCSException("Git svn fetch failed")
637 p = SilentPopen(['%sgit svn rebase %s' % self.userargs()], cwd=self.local, shell=True)
638 if p.returncode != 0:
639 raise VCSException("Git svn rebase failed", p.output)
640 self.refreshed = True
642 rev = rev or 'master'
644 nospaces_rev = rev.replace(' ', '%20')
645 # Try finding a svn tag
646 for treeish in ['origin/', '']:
647 p = SilentPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev],
649 if p.returncode == 0:
651 if p.returncode != 0:
652 # No tag found, normal svn rev translation
653 # Translate svn rev into git format
654 rev_split = rev.split('/')
657 for treeish in ['origin/', '']:
658 if len(rev_split) > 1:
659 treeish += rev_split[0]
660 svn_rev = rev_split[1]
663 # if no branch is specified, then assume trunk (i.e. 'master' branch):
667 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
669 p = SilentPopen(['git', 'svn', 'find-rev', svn_rev, treeish],
671 git_rev = p.output.rstrip()
673 if p.returncode == 0 and git_rev:
676 if p.returncode != 0 or not git_rev:
677 # Try a plain git checkout as a last resort
678 p = SilentPopen(['git', 'checkout', rev], cwd=self.local)
679 if p.returncode != 0:
680 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
682 # Check out the git rev equivalent to the svn rev
683 p = SilentPopen(['git', 'checkout', git_rev], cwd=self.local)
684 if p.returncode != 0:
685 raise VCSException("Git svn checkout of '%s' failed" % rev, p.output)
687 # Get rid of any uncontrolled files left behind
688 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
689 if p.returncode != 0:
690 raise VCSException("Git clean failed", p.output)
694 return os.listdir(os.path.join(self.local, '.git/svn/refs/remotes/tags'))
698 p = SilentPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local)
699 if p.returncode != 0:
701 return p.output.strip()
710 if self.username is None:
711 return ['--non-interactive']
712 return ['--username', self.username,
713 '--password', self.password,
716 def gotorevisionx(self, rev):
717 if not os.path.exists(self.local):
718 p = SilentPopen(['svn', 'checkout', self.remote, self.local] + self.userargs())
719 if p.returncode != 0:
720 self.clone_failed = True
721 raise VCSException("Svn checkout of '%s' failed" % rev, p.output)
725 r"svn status | awk '/\?/ {print $2}' | xargs rm -rf"):
726 p = SilentPopen([svncommand], cwd=self.local, shell=True)
727 if p.returncode != 0:
728 raise VCSException("Svn reset ({0}) failed in {1}".format(svncommand, self.local), p.output)
729 if not self.refreshed:
730 p = SilentPopen(['svn', 'update'] + self.userargs(), cwd=self.local)
731 if p.returncode != 0:
732 raise VCSException("Svn update failed", p.output)
733 self.refreshed = True
735 revargs = list(['-r', rev] if rev else [])
736 p = SilentPopen(['svn', 'update', '--force'] + revargs + self.userargs(), cwd=self.local)
737 if p.returncode != 0:
738 raise VCSException("Svn update failed", p.output)
741 p = SilentPopen(['svn', 'info'], cwd=self.local)
742 for line in p.output.splitlines():
743 if line and line.startswith('Last Changed Rev: '):
753 def gotorevisionx(self, rev):
754 if not os.path.exists(self.local):
755 p = SilentPopen(['hg', 'clone', self.remote, self.local])
756 if p.returncode != 0:
757 self.clone_failed = True
758 raise VCSException("Hg clone failed", p.output)
760 p = SilentPopen(['hg status -uS | xargs rm -rf'], cwd=self.local, shell=True)
761 if p.returncode != 0:
762 raise VCSException("Hg clean failed", p.output)
763 if not self.refreshed:
764 p = SilentPopen(['hg', 'pull'], cwd=self.local)
765 if p.returncode != 0:
766 raise VCSException("Hg pull failed", p.output)
767 self.refreshed = True
769 rev = rev or 'default'
772 p = SilentPopen(['hg', 'update', '-C', rev], cwd=self.local)
773 if p.returncode != 0:
774 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
775 p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
776 # Also delete untracked files, we have to enable purge extension for that:
777 if "'purge' is provided by the following extension" in p.output:
778 with open(self.local + "/.hg/hgrc", "a") as myfile:
779 myfile.write("\n[extensions]\nhgext.purge=\n")
780 p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
781 if p.returncode != 0:
782 raise VCSException("HG purge failed", p.output)
783 elif p.returncode != 0:
784 raise VCSException("HG purge failed", p.output)
787 p = SilentPopen(['hg', 'tags', '-q'], cwd=self.local)
788 return p.output.splitlines()[1:]
796 def gotorevisionx(self, rev):
797 if not os.path.exists(self.local):
798 p = SilentPopen(['bzr', 'branch', self.remote, self.local])
799 if p.returncode != 0:
800 self.clone_failed = True
801 raise VCSException("Bzr branch failed", p.output)
803 p = SilentPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local)
804 if p.returncode != 0:
805 raise VCSException("Bzr revert failed", p.output)
806 if not self.refreshed:
807 p = SilentPopen(['bzr', 'pull'], cwd=self.local)
808 if p.returncode != 0:
809 raise VCSException("Bzr update failed", p.output)
810 self.refreshed = True
812 revargs = list(['-r', rev] if rev else [])
813 p = SilentPopen(['bzr', 'revert'] + revargs, cwd=self.local)
814 if p.returncode != 0:
815 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
818 p = SilentPopen(['bzr', 'tags'], cwd=self.local)
819 return [tag.split(' ')[0].strip() for tag in
820 p.output.splitlines()]
823 def retrieve_string(app_dir, string, xmlfiles=None):
826 os.path.join(app_dir, 'res'),
827 os.path.join(app_dir, 'src/main'),
832 for res_dir in res_dirs:
833 for r, d, f in os.walk(res_dir):
834 if r.endswith('/values'):
835 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
838 if string.startswith('@string/'):
839 string_search = re.compile(r'.*name="' + string[8:] + '".*?>([^<]+?)<.*').search
840 elif string.startswith('&') and string.endswith(';'):
841 string_search = re.compile(r'.*<!ENTITY.*' + string[1:-1] + '.*?"([^"]+?)".*>').search
843 if string_search is not None:
844 for xmlfile in xmlfiles:
845 for line in file(xmlfile):
846 matches = string_search(line)
848 return retrieve_string(app_dir, matches.group(1), xmlfiles)
851 return string.replace("\\'", "'")
854 # Return list of existing files that will be used to find the highest vercode
855 def manifest_paths(app_dir, flavour):
857 possible_manifests = \
858 [os.path.join(app_dir, 'AndroidManifest.xml'),
859 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
860 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
861 os.path.join(app_dir, 'build.gradle')]
864 possible_manifests.append(
865 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
867 return [path for path in possible_manifests if os.path.isfile(path)]
870 # Retrieve the package name. Returns the name, or None if not found.
871 def fetch_real_name(app_dir, flavour):
872 app_search = re.compile(r'.*<application.*').search
873 name_search = re.compile(r'.*android:label="([^"]+)".*').search
875 for f in manifest_paths(app_dir, flavour):
876 if not has_extension(f, 'xml'):
878 logging.debug("fetch_real_name: Checking manifest at " + f)
884 matches = name_search(line)
886 stringname = matches.group(1)
887 logging.debug("fetch_real_name: using string " + stringname)
888 result = retrieve_string(app_dir, stringname)
890 result = result.strip()
895 # Retrieve the version name
896 def version_name(original, app_dir, flavour):
897 for f in manifest_paths(app_dir, flavour):
898 if not has_extension(f, 'xml'):
900 string = retrieve_string(app_dir, original)
906 def get_library_references(root_dir):
908 proppath = os.path.join(root_dir, 'project.properties')
909 if not os.path.isfile(proppath):
911 with open(proppath) as f:
912 for line in f.readlines():
913 if not line.startswith('android.library.reference.'):
915 path = line.split('=')[1].strip()
916 relpath = os.path.join(root_dir, path)
917 if not os.path.isdir(relpath):
919 logging.debug("Found subproject at %s" % path)
920 libraries.append(path)
924 def ant_subprojects(root_dir):
925 subprojects = get_library_references(root_dir)
926 for subpath in subprojects:
927 subrelpath = os.path.join(root_dir, subpath)
928 for p in get_library_references(subrelpath):
929 relp = os.path.normpath(os.path.join(subpath, p))
930 if relp not in subprojects:
931 subprojects.insert(0, relp)
935 def remove_debuggable_flags(root_dir):
936 # Remove forced debuggable flags
937 logging.debug("Removing debuggable flags from %s" % root_dir)
938 for root, dirs, files in os.walk(root_dir):
939 if 'AndroidManifest.xml' in files:
940 path = os.path.join(root, 'AndroidManifest.xml')
941 p = SilentPopen(['sed', '-i', 's/android:debuggable="[^"]*"//g', path])
942 if p.returncode != 0:
943 raise BuildException("Failed to remove debuggable flags of %s" % path)
946 # Extract some information from the AndroidManifest.xml at the given path.
947 # Returns (version, vercode, package), any or all of which might be None.
948 # All values returned are strings.
949 def parse_androidmanifests(paths, ignoreversions=None):
952 return (None, None, None)
954 vcsearch = re.compile(r'.*:versionCode="([0-9]+?)".*').search
955 vnsearch = re.compile(r'.*:versionName="([^"]+?)".*').search
956 psearch = re.compile(r'.*package="([^"]+)".*').search
958 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
959 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
960 psearch_g = re.compile(r'.*packageName *=* *["\']([^"]+)["\'].*').search
962 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
970 gradle = has_extension(path, 'gradle')
973 # Remember package name, may be defined separately from version+vercode
974 package = max_package
976 for line in file(path):
979 matches = psearch_g(line)
981 matches = psearch(line)
983 package = matches.group(1)
986 matches = vnsearch_g(line)
988 matches = vnsearch(line)
990 version = matches.group(2 if gradle else 1)
993 matches = vcsearch_g(line)
995 matches = vcsearch(line)
997 vercode = matches.group(1)
999 # Always grab the package name and version name in case they are not
1000 # together with the highest version code
1001 if max_package is None and package is not None:
1002 max_package = package
1003 if max_version is None and version is not None:
1004 max_version = version
1006 if max_vercode is None or (vercode is not None and vercode > max_vercode):
1007 if not ignoresearch or not ignoresearch(version):
1008 if version is not None:
1009 max_version = version
1010 if vercode is not None:
1011 max_vercode = vercode
1012 if package is not None:
1013 max_package = package
1015 max_version = "Ignore"
1017 if max_version is None:
1018 max_version = "Unknown"
1020 return (max_version, max_vercode, max_package)
1023 class FDroidException(Exception):
1024 def __init__(self, value, detail=None):
1026 self.detail = detail
1028 def get_wikitext(self):
1029 ret = repr(self.value) + "\n"
1033 txt = self.detail[-8192:] if len(self.detail) > 8192 else self.detail
1041 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1045 class VCSException(FDroidException):
1049 class BuildException(FDroidException):
1053 # Get the specified source library.
1054 # Returns the path to it. Normally this is the path to be used when referencing
1055 # it, which may be a subdirectory of the actual project. If you want the base
1056 # directory of the project, pass 'basepath=True'.
1057 def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
1058 basepath=False, raw=False, prepare=True, preponly=False):
1066 name, ref = spec.split('@')
1068 number, name = name.split(':', 1)
1070 name, subdir = name.split('/', 1)
1072 if name not in metadata.srclibs:
1073 raise VCSException('srclib ' + name + ' not found.')
1075 srclib = metadata.srclibs[name]
1077 sdir = os.path.join(srclib_dir, name)
1080 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1081 vcs.srclib = (name, number, sdir)
1083 vcs.gotorevision(ref)
1090 libdir = os.path.join(sdir, subdir)
1091 elif srclib["Subdir"]:
1092 for subdir in srclib["Subdir"]:
1093 libdir_candidate = os.path.join(sdir, subdir)
1094 if os.path.exists(libdir_candidate):
1095 libdir = libdir_candidate
1101 if srclib["Srclibs"]:
1103 for lib in srclib["Srclibs"].replace(';', ',').split(','):
1105 for t in srclibpaths:
1110 raise VCSException('Missing recursive srclib %s for %s' % (
1112 place_srclib(libdir, n, s_tuple[2])
1115 remove_signing_keys(sdir)
1116 remove_debuggable_flags(sdir)
1120 if srclib["Prepare"]:
1121 cmd = replace_config_vars(srclib["Prepare"])
1123 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1124 if p.returncode != 0:
1125 raise BuildException("Error running prepare command for srclib %s"
1131 return (name, number, libdir)
1134 # Prepare the source code for a particular build
1135 # 'vcs' - the appropriate vcs object for the application
1136 # 'app' - the application details from the metadata
1137 # 'build' - the build details from the metadata
1138 # 'build_dir' - the path to the build directory, usually
1140 # 'srclib_dir' - the path to the source libraries directory, usually
1142 # 'extlib_dir' - the path to the external libraries directory, usually
1144 # Returns the (root, srclibpaths) where:
1145 # 'root' is the root directory, which may be the same as 'build_dir' or may
1146 # be a subdirectory of it.
1147 # 'srclibpaths' is information on the srclibs being used
1148 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False):
1150 # Optionally, the actual app source can be in a subdirectory
1152 root_dir = os.path.join(build_dir, build['subdir'])
1154 root_dir = build_dir
1156 # Get a working copy of the right revision
1157 logging.info("Getting source for revision " + build['commit'])
1158 vcs.gotorevision(build['commit'])
1160 # Initialise submodules if requred
1161 if build['submodules']:
1162 logging.info("Initialising submodules")
1163 vcs.initsubmodules()
1165 # Check that a subdir (if we're using one) exists. This has to happen
1166 # after the checkout, since it might not exist elsewhere
1167 if not os.path.exists(root_dir):
1168 raise BuildException('Missing subdir ' + root_dir)
1170 # Run an init command if one is required
1172 cmd = replace_config_vars(build['init'])
1173 logging.info("Running 'init' commands in %s" % root_dir)
1175 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1176 if p.returncode != 0:
1177 raise BuildException("Error running init command for %s:%s" %
1178 (app['id'], build['version']), p.output)
1180 # Apply patches if any
1182 logging.info("Applying patches")
1183 for patch in build['patch']:
1184 patch = patch.strip()
1185 logging.info("Applying " + patch)
1186 patch_path = os.path.join('metadata', app['id'], patch)
1187 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1188 if p.returncode != 0:
1189 raise BuildException("Failed to apply patch %s" % patch_path)
1191 # Get required source libraries
1193 if build['srclibs']:
1194 logging.info("Collecting source libraries")
1195 for lib in build['srclibs']:
1196 srclibpaths.append(getsrclib(lib, srclib_dir, srclibpaths,
1199 for name, number, libpath in srclibpaths:
1200 place_srclib(root_dir, int(number) if number else None, libpath)
1202 basesrclib = vcs.getsrclib()
1203 # If one was used for the main source, add that too.
1205 srclibpaths.append(basesrclib)
1207 # Update the local.properties file
1208 localprops = [os.path.join(build_dir, 'local.properties')]
1210 localprops += [os.path.join(root_dir, 'local.properties')]
1211 for path in localprops:
1212 if not os.path.isfile(path):
1214 logging.info("Updating properties file at %s" % path)
1219 # Fix old-fashioned 'sdk-location' by copying
1220 # from sdk.dir, if necessary
1221 if build['oldsdkloc']:
1222 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1223 re.S | re.M).group(1)
1224 props += "sdk-location=%s\n" % sdkloc
1226 props += "sdk.dir=%s\n" % config['sdk_path']
1227 props += "sdk-location=%s\n" % config['sdk_path']
1228 if 'ndk_path' in config:
1230 props += "ndk.dir=%s\n" % config['ndk_path']
1231 props += "ndk-location=%s\n" % config['ndk_path']
1232 # Add java.encoding if necessary
1233 if build['encoding']:
1234 props += "java.encoding=%s\n" % build['encoding']
1240 if build['type'] == 'gradle':
1241 flavour = build['gradle']
1242 if flavour in ['main', 'yes', '']:
1245 version_regex = re.compile(r".*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1246 gradlepluginver = None
1248 gradle_files = [os.path.join(root_dir, 'build.gradle')]
1250 # Parent dir build.gradle
1251 parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
1252 if parent_dir.startswith(build_dir):
1253 gradle_files.append(os.path.join(parent_dir, 'build.gradle'))
1255 for path in gradle_files:
1258 if not os.path.isfile(path):
1260 with open(path) as f:
1262 match = version_regex.match(line)
1264 gradlepluginver = match.group(1)
1268 build['gradlepluginver'] = LooseVersion(gradlepluginver)
1270 logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
1271 build['gradlepluginver'] = LooseVersion('0.11')
1274 n = build["target"].split('-')[1]
1275 SilentPopen(['sed', '-i',
1276 's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g',
1280 # Remove forced debuggable flags
1281 remove_debuggable_flags(root_dir)
1283 # Insert version code and number into the manifest if necessary
1284 if build['forceversion']:
1285 logging.info("Changing the version name")
1286 for path in manifest_paths(root_dir, flavour):
1287 if not os.path.isfile(path):
1289 if has_extension(path, 'xml'):
1290 p = SilentPopen(['sed', '-i',
1291 's/android:versionName="[^"]*"/android:versionName="'
1292 + build['version'] + '"/g',
1294 if p.returncode != 0:
1295 raise BuildException("Failed to amend manifest")
1296 elif has_extension(path, 'gradle'):
1297 p = SilentPopen(['sed', '-i',
1298 's/versionName *=* *"[^"]*"/versionName = "'
1299 + build['version'] + '"/g',
1301 if p.returncode != 0:
1302 raise BuildException("Failed to amend build.gradle")
1303 if build['forcevercode']:
1304 logging.info("Changing the version code")
1305 for path in manifest_paths(root_dir, flavour):
1306 if not os.path.isfile(path):
1308 if has_extension(path, 'xml'):
1309 p = SilentPopen(['sed', '-i',
1310 's/android:versionCode="[^"]*"/android:versionCode="'
1311 + build['vercode'] + '"/g',
1313 if p.returncode != 0:
1314 raise BuildException("Failed to amend manifest")
1315 elif has_extension(path, 'gradle'):
1316 p = SilentPopen(['sed', '-i',
1317 's/versionCode *=* *[0-9]*/versionCode = '
1318 + build['vercode'] + '/g',
1320 if p.returncode != 0:
1321 raise BuildException("Failed to amend build.gradle")
1323 # Delete unwanted files
1325 logging.info("Removing specified files")
1326 for part in getpaths(build_dir, build, 'rm'):
1327 dest = os.path.join(build_dir, part)
1328 logging.info("Removing {0}".format(part))
1329 if os.path.lexists(dest):
1330 if os.path.islink(dest):
1331 SilentPopen(['unlink ' + dest], shell=True)
1333 SilentPopen(['rm -rf ' + dest], shell=True)
1335 logging.info("...but it didn't exist")
1337 remove_signing_keys(build_dir)
1339 # Add required external libraries
1340 if build['extlibs']:
1341 logging.info("Collecting prebuilt libraries")
1342 libsdir = os.path.join(root_dir, 'libs')
1343 if not os.path.exists(libsdir):
1345 for lib in build['extlibs']:
1347 logging.info("...installing extlib {0}".format(lib))
1348 libf = os.path.basename(lib)
1349 libsrc = os.path.join(extlib_dir, lib)
1350 if not os.path.exists(libsrc):
1351 raise BuildException("Missing extlib file {0}".format(libsrc))
1352 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1354 # Run a pre-build command if one is required
1355 if build['prebuild']:
1356 logging.info("Running 'prebuild' commands in %s" % root_dir)
1358 cmd = replace_config_vars(build['prebuild'])
1360 # Substitute source library paths into prebuild commands
1361 for name, number, libpath in srclibpaths:
1362 libpath = os.path.relpath(libpath, root_dir)
1363 cmd = cmd.replace('$$' + name + '$$', libpath)
1365 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1366 if p.returncode != 0:
1367 raise BuildException("Error running prebuild command for %s:%s" %
1368 (app['id'], build['version']), p.output)
1370 # Generate (or update) the ant build file, build.xml...
1371 if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
1372 parms = [config['android'], 'update', 'lib-project']
1373 lparms = [config['android'], 'update', 'project']
1376 parms += ['-t', build['target']]
1377 lparms += ['-t', build['target']]
1378 if build['update'] == ['auto']:
1379 update_dirs = ant_subprojects(root_dir) + ['.']
1381 update_dirs = build['update']
1383 for d in update_dirs:
1384 subdir = os.path.join(root_dir, d)
1386 logging.debug("Updating main project")
1387 cmd = parms + ['-p', d]
1389 logging.debug("Updating subproject %s" % d)
1390 cmd = lparms + ['-p', d]
1391 p = FDroidPopen(cmd, cwd=root_dir)
1392 # Check to see whether an error was returned without a proper exit
1393 # code (this is the case for the 'no target set or target invalid'
1395 if p.returncode != 0 or p.output.startswith("Error: "):
1396 raise BuildException("Failed to update project at %s" % d, p.output)
1397 # Clean update dirs via ant
1399 logging.info("Cleaning subproject %s" % d)
1400 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1402 return (root_dir, srclibpaths)
1405 # Split and extend via globbing the paths from a field
1406 def getpaths(build_dir, build, field):
1408 for p in build[field]:
1410 full_path = os.path.join(build_dir, p)
1411 full_path = os.path.normpath(full_path)
1412 paths += [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1416 # Scan the source code in the given directory (and all subdirectories)
1417 # and return the number of fatal problems encountered
1418 def scan_source(build_dir, root_dir, thisbuild):
1422 # Common known non-free blobs (always lower case):
1424 re.compile(r'flurryagent', re.IGNORECASE),
1425 re.compile(r'paypal.*mpl', re.IGNORECASE),
1426 re.compile(r'google.*analytics', re.IGNORECASE),
1427 re.compile(r'admob.*sdk.*android', re.IGNORECASE),
1428 re.compile(r'google.*ad.*view', re.IGNORECASE),
1429 re.compile(r'google.*admob', re.IGNORECASE),
1430 re.compile(r'google.*play.*services', re.IGNORECASE),
1431 re.compile(r'crittercism', re.IGNORECASE),
1432 re.compile(r'heyzap', re.IGNORECASE),
1433 re.compile(r'jpct.*ae', re.IGNORECASE),
1434 re.compile(r'youtube.*android.*player.*api', re.IGNORECASE),
1435 re.compile(r'bugsense', re.IGNORECASE),
1436 re.compile(r'crashlytics', re.IGNORECASE),
1437 re.compile(r'ouya.*sdk', re.IGNORECASE),
1438 re.compile(r'libspen23', re.IGNORECASE),
1441 scanignore = getpaths(build_dir, thisbuild, 'scanignore')
1442 scandelete = getpaths(build_dir, thisbuild, 'scandelete')
1445 ms = magic.open(magic.MIME_TYPE)
1447 except AttributeError:
1451 for i in scanignore:
1452 if fd.startswith(i):
1457 for i in scandelete:
1458 if fd.startswith(i):
1462 def removeproblem(what, fd, fp):
1463 logging.info('Removing %s at %s' % (what, fd))
1466 def warnproblem(what, fd):
1467 logging.warn('Found %s at %s' % (what, fd))
1469 def handleproblem(what, fd, fp):
1471 removeproblem(what, fd, fp)
1473 logging.error('Found %s at %s' % (what, fd))
1477 # Iterate through all files in the source code
1478 for r, d, f in os.walk(build_dir, topdown=True):
1480 # It's topdown, so checking the basename is enough
1481 for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
1487 # Path (relative) to the file
1488 fp = os.path.join(r, curfile)
1489 fd = fp[len(build_dir) + 1:]
1491 # Check if this file has been explicitly excluded from scanning
1495 mime = magic.from_file(fp, mime=True) if ms is None else ms.file(fp)
1497 if mime == 'application/x-sharedlib':
1498 count += handleproblem('shared library', fd, fp)
1500 elif mime == 'application/x-archive':
1501 count += handleproblem('static library', fd, fp)
1503 elif mime == 'application/x-executable':
1504 count += handleproblem('binary executable', fd, fp)
1506 elif mime == 'application/x-java-applet':
1507 count += handleproblem('Java compiled class', fd, fp)
1512 'application/java-archive',
1513 'application/octet-stream',
1517 if has_extension(fp, 'apk'):
1518 removeproblem('APK file', fd, fp)
1520 elif has_extension(fp, 'jar'):
1522 if any(suspect.match(curfile) for suspect in usual_suspects):
1523 count += handleproblem('usual supect', fd, fp)
1525 warnproblem('JAR file', fd)
1527 elif has_extension(fp, 'zip'):
1528 warnproblem('ZIP file', fd)
1531 warnproblem('unknown compressed or binary file', fd)
1533 elif has_extension(fp, 'java'):
1534 for line in file(fp):
1535 if 'DexClassLoader' in line:
1536 count += handleproblem('DexClassLoader', fd, fp)
1541 # Presence of a jni directory without buildjni=yes might
1542 # indicate a problem (if it's not a problem, explicitly use
1543 # buildjni=no to bypass this check)
1544 if (os.path.exists(os.path.join(root_dir, 'jni')) and
1545 not thisbuild['buildjni']):
1546 logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
1555 self.path = os.path.join('stats', 'known_apks.txt')
1557 if os.path.exists(self.path):
1558 for line in file(self.path):
1559 t = line.rstrip().split(' ')
1561 self.apks[t[0]] = (t[1], None)
1563 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1564 self.changed = False
1566 def writeifchanged(self):
1568 if not os.path.exists('stats'):
1570 f = open(self.path, 'w')
1572 for apk, app in self.apks.iteritems():
1574 line = apk + ' ' + appid
1576 line += ' ' + time.strftime('%Y-%m-%d', added)
1578 for line in sorted(lst):
1579 f.write(line + '\n')
1582 # Record an apk (if it's new, otherwise does nothing)
1583 # Returns the date it was added.
1584 def recordapk(self, apk, app):
1585 if apk not in self.apks:
1586 self.apks[apk] = (app, time.gmtime(time.time()))
1588 _, added = self.apks[apk]
1591 # Look up information - given the 'apkname', returns (app id, date added/None).
1592 # Or returns None for an unknown apk.
1593 def getapp(self, apkname):
1594 if apkname in self.apks:
1595 return self.apks[apkname]
1598 # Get the most recent 'num' apps added to the repo, as a list of package ids
1599 # with the most recent first.
1600 def getlatest(self, num):
1602 for apk, app in self.apks.iteritems():
1606 if apps[appid] > added:
1610 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1611 lst = [app for app, _ in sortedapps]
1616 def isApkDebuggable(apkfile, config):
1617 """Returns True if the given apk file is debuggable
1619 :param apkfile: full path to the apk to check"""
1621 p = SilentPopen([os.path.join(config['sdk_path'], 'build-tools',
1622 config['build_tools'], 'aapt'),
1623 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'])
1624 if p.returncode != 0:
1625 logging.critical("Failed to get apk manifest information")
1627 for line in p.output.splitlines():
1628 if 'android:debuggable' in line and not line.endswith('0x0'):
1633 class AsynchronousFileReader(threading.Thread):
1635 Helper class to implement asynchronous reading of a file
1636 in a separate thread. Pushes read lines on a queue to
1637 be consumed in another thread.
1640 def __init__(self, fd, queue):
1641 assert isinstance(queue, Queue.Queue)
1642 assert callable(fd.readline)
1643 threading.Thread.__init__(self)
1648 '''The body of the tread: read lines and put them on the queue.'''
1649 for line in iter(self._fd.readline, ''):
1650 self._queue.put(line)
1653 '''Check whether there is no more content to expect.'''
1654 return not self.is_alive() and self._queue.empty()
1662 def SilentPopen(commands, cwd=None, shell=False):
1663 return FDroidPopen(commands, cwd=cwd, shell=shell, output=False)
1666 def FDroidPopen(commands, cwd=None, shell=False, output=True):
1668 Run a command and capture the possibly huge output.
1670 :param commands: command and argument list like in subprocess.Popen
1671 :param cwd: optionally specifies a working directory
1672 :returns: A PopenResult.
1678 cwd = os.path.normpath(cwd)
1679 logging.debug("Directory: %s" % cwd)
1680 logging.debug("> %s" % ' '.join(commands))
1682 result = PopenResult()
1683 p = subprocess.Popen(commands, cwd=cwd, shell=shell, env=env,
1684 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1686 stdout_queue = Queue.Queue()
1687 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1688 stdout_reader.start()
1690 # Check the queue for output (until there is no more to get)
1691 while not stdout_reader.eof():
1692 while not stdout_queue.empty():
1693 line = stdout_queue.get()
1694 if output and options.verbose:
1695 # Output directly to console
1696 sys.stderr.write(line)
1698 result.output += line
1703 result.returncode = p.returncode
1707 def remove_signing_keys(build_dir):
1708 comment = re.compile(r'[ ]*//')
1709 signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1711 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1712 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1713 re.compile(r'.*variant\.outputFile = .*'),
1714 re.compile(r'.*\.readLine\(.*'),
1716 for root, dirs, files in os.walk(build_dir):
1717 if 'build.gradle' in files:
1718 path = os.path.join(root, 'build.gradle')
1720 with open(path, "r") as o:
1721 lines = o.readlines()
1726 with open(path, "w") as o:
1728 if comment.match(line):
1732 opened += line.count('{')
1733 opened -= line.count('}')
1736 if signing_configs.match(line):
1741 if any(s.match(line) for s in line_matches):
1749 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1752 'project.properties',
1754 'default.properties',
1757 if propfile in files:
1758 path = os.path.join(root, propfile)
1760 with open(path, "r") as o:
1761 lines = o.readlines()
1765 with open(path, "w") as o:
1767 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1774 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1777 def replace_config_vars(cmd):
1778 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1779 cmd = cmd.replace('$$NDK$$', config['ndk_path'])
1780 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1784 def place_srclib(root_dir, number, libpath):
1787 relpath = os.path.relpath(libpath, root_dir)
1788 proppath = os.path.join(root_dir, 'project.properties')
1791 if os.path.isfile(proppath):
1792 with open(proppath, "r") as o:
1793 lines = o.readlines()
1795 with open(proppath, "w") as o:
1798 if line.startswith('android.library.reference.%d=' % number):
1799 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1804 o.write('android.library.reference.%d=%s\n' % (number, relpath))