1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
33 from distutils.version import LooseVersion
43 'sdk_path': "$ANDROID_HOME",
44 'ndk_path': "$ANDROID_NDK",
45 'build_tools': "21.0.2",
49 'sync_from_local_copy_dir': False,
50 'update_stats': False,
54 'stats_to_carbon': False,
56 'build_server_always': False,
57 'keystore': os.path.join("$HOME", '.local', 'share', 'fdroidserver', 'keystore.jks'),
58 'smartcardoptions': [],
64 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
65 'repo_name': "My First FDroid Repo Demo",
66 'repo_icon': "fdroid-icon.png",
67 'repo_description': '''
68 This is a repository of apps to be used with FDroid. Applications in this
69 repository are either official binaries built by the original application
70 developers, or are binaries built from source by the admin of f-droid.org
71 using the tools on https://gitlab.com/u/fdroid.
77 def fill_config_defaults(thisconfig):
78 for k, v in default_config.items():
79 if k not in thisconfig:
82 # Expand paths (~users and $vars)
83 for k in ['sdk_path', 'ndk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
86 v = os.path.expanduser(v)
87 v = os.path.expandvars(v)
90 thisconfig[k + '_orig'] = orig
93 def read_config(opts, config_file='config.py'):
94 """Read the repository config
96 The config is read from config_file, which is in the current directory when
97 any of the repo management commands are used.
99 global config, options, env
101 if config is not None:
103 if not os.path.isfile(config_file):
104 logging.critical("Missing config file - is this a repo directory?")
111 logging.debug("Reading %s" % config_file)
112 execfile(config_file, config)
114 # smartcardoptions must be a list since its command line args for Popen
115 if 'smartcardoptions' in config:
116 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
117 elif 'keystore' in config and config['keystore'] == 'NONE':
118 # keystore='NONE' means use smartcard, these are required defaults
119 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
120 'SunPKCS11-OpenSC', '-providerClass',
121 'sun.security.pkcs11.SunPKCS11',
122 '-providerArg', 'opensc-fdroid.cfg']
124 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
125 st = os.stat(config_file)
126 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
127 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
129 fill_config_defaults(config)
131 if not test_sdk_exists(config):
134 if not test_build_tools_exists(config):
139 os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
142 os.path.join(config['sdk_path'], 'tools', 'zipalign'),
143 os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'zipalign'),
146 os.path.join(config['sdk_path'], 'tools', 'android'),
149 os.path.join(config['sdk_path'], 'platform-tools', 'adb'),
153 for b, paths in bin_paths.items():
156 if os.path.isfile(path):
159 if config[b] is None:
160 logging.warn("Could not find %s in any of the following paths:\n%s" % (
161 b, '\n'.join(paths)))
163 # There is no standard, so just set up the most common environment
166 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
167 env[n] = config['sdk_path']
168 for n in ['ANDROID_NDK', 'NDK']:
169 env[n] = config['ndk_path']
171 for k in ["keystorepass", "keypass"]:
173 write_password_file(k)
175 for k in ["repo_description", "archive_description"]:
177 config[k] = clean_description(config[k])
179 if 'serverwebroot' in config:
180 if isinstance(config['serverwebroot'], basestring):
181 roots = [config['serverwebroot']]
182 elif all(isinstance(item, basestring) for item in config['serverwebroot']):
183 roots = config['serverwebroot']
185 raise TypeError('only accepts strings, lists, and tuples')
187 for rootstr in roots:
188 # since this is used with rsync, where trailing slashes have
189 # meaning, ensure there is always a trailing slash
190 if rootstr[-1] != '/':
192 rootlist.append(rootstr.replace('//', '/'))
193 config['serverwebroot'] = rootlist
198 def test_sdk_exists(thisconfig):
199 if thisconfig['sdk_path'] == default_config['sdk_path']:
200 logging.error('No Android SDK found!')
201 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
202 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
204 if not os.path.exists(thisconfig['sdk_path']):
205 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
207 if not os.path.isdir(thisconfig['sdk_path']):
208 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
210 for d in ['build-tools', 'platform-tools', 'tools']:
211 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
212 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
213 thisconfig['sdk_path'], d))
218 def test_build_tools_exists(thisconfig):
219 if not test_sdk_exists(thisconfig):
221 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
222 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
223 if not os.path.isdir(versioned_build_tools):
224 logging.critical('Android Build Tools path "'
225 + versioned_build_tools + '" does not exist!')
230 def write_password_file(pwtype, password=None):
232 writes out passwords to a protected file instead of passing passwords as
233 command line argments
235 filename = '.fdroid.' + pwtype + '.txt'
236 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
238 os.write(fd, config[pwtype])
240 os.write(fd, password)
242 config[pwtype + 'file'] = filename
245 # Given the arguments in the form of multiple appid:[vc] strings, this returns
246 # a dictionary with the set of vercodes specified for each package.
247 def read_pkg_args(args, allow_vercodes=False):
254 if allow_vercodes and ':' in p:
255 package, vercode = p.split(':')
257 package, vercode = p, None
258 if package not in vercodes:
259 vercodes[package] = [vercode] if vercode else []
261 elif vercode and vercode not in vercodes[package]:
262 vercodes[package] += [vercode] if vercode else []
267 # On top of what read_pkg_args does, this returns the whole app metadata, but
268 # limiting the builds list to the builds matching the vercodes specified.
269 def read_app_args(args, allapps, allow_vercodes=False):
271 vercodes = read_pkg_args(args, allow_vercodes)
277 for appid, app in allapps.iteritems():
278 if appid in vercodes:
281 if len(apps) != len(vercodes):
284 logging.critical("No such package: %s" % p)
285 raise FDroidException("Found invalid app ids in arguments")
287 raise FDroidException("No packages specified")
290 for appid, app in apps.iteritems():
294 app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
295 if len(app['builds']) != len(vercodes[appid]):
297 allvcs = [b['vercode'] for b in app['builds']]
298 for v in vercodes[appid]:
300 logging.critical("No such vercode %s for app %s" % (v, appid))
303 raise FDroidException("Found invalid vercodes for some apps")
308 def has_extension(filename, extension):
309 name, ext = os.path.splitext(filename)
310 ext = ext.lower()[1:]
311 return ext == extension
316 def clean_description(description):
317 'Remove unneeded newlines and spaces from a block of description text'
319 # this is split up by paragraph to make removing the newlines easier
320 for paragraph in re.split(r'\n\n', description):
321 paragraph = re.sub('\r', '', paragraph)
322 paragraph = re.sub('\n', ' ', paragraph)
323 paragraph = re.sub(' {2,}', ' ', paragraph)
324 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
325 returnstring += paragraph + '\n\n'
326 return returnstring.rstrip('\n')
329 def apknameinfo(filename):
331 filename = os.path.basename(filename)
332 if apk_regex is None:
333 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
334 m = apk_regex.match(filename)
336 result = (m.group(1), m.group(2))
337 except AttributeError:
338 raise FDroidException("Invalid apk name: %s" % filename)
342 def getapkname(app, build):
343 return "%s_%s.apk" % (app['id'], build['vercode'])
346 def getsrcname(app, build):
347 return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
354 return app['Auto Name']
359 return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
362 def getvcs(vcstype, remote, local):
364 return vcs_git(remote, local)
365 if vcstype == 'git-svn':
366 return vcs_gitsvn(remote, local)
368 return vcs_hg(remote, local)
370 return vcs_bzr(remote, local)
371 if vcstype == 'srclib':
372 if local != os.path.join('build', 'srclib', remote):
373 raise VCSException("Error: srclib paths are hard-coded!")
374 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
376 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
377 raise VCSException("Invalid vcs type " + vcstype)
380 def getsrclibvcs(name):
381 if name not in metadata.srclibs:
382 raise VCSException("Missing srclib " + name)
383 return metadata.srclibs[name]['Repo Type']
387 def __init__(self, remote, local):
389 # svn, git-svn and bzr may require auth
391 if self.repotype() in ('git-svn', 'bzr'):
393 self.username, remote = remote.split('@')
394 if ':' not in self.username:
395 raise VCSException("Password required with username")
396 self.username, self.password = self.username.split(':')
400 self.clone_failed = False
401 self.refreshed = False
407 # Take the local repository to a clean version of the given revision, which
408 # is specificed in the VCS's native format. Beforehand, the repository can
409 # be dirty, or even non-existent. If the repository does already exist
410 # locally, it will be updated from the origin, but only once in the
411 # lifetime of the vcs object.
412 # None is acceptable for 'rev' if you know you are cloning a clean copy of
413 # the repo - otherwise it must specify a valid revision.
414 def gotorevision(self, rev):
416 if self.clone_failed:
417 raise VCSException("Downloading the repository already failed once, not trying again.")
419 # The .fdroidvcs-id file for a repo tells us what VCS type
420 # and remote that directory was created from, allowing us to drop it
421 # automatically if either of those things changes.
422 fdpath = os.path.join(self.local, '..',
423 '.fdroidvcs-' + os.path.basename(self.local))
424 cdata = self.repotype() + ' ' + self.remote
427 if os.path.exists(self.local):
428 if os.path.exists(fdpath):
429 with open(fdpath, 'r') as f:
430 fsdata = f.read().strip()
436 "Repository details for %s changed - deleting" % (
440 logging.info("Repository details for %s missing - deleting" % (
443 shutil.rmtree(self.local)
448 self.gotorevisionx(rev)
449 except FDroidException, e:
452 # If necessary, write the .fdroidvcs file.
453 if writeback and not self.clone_failed:
454 with open(fdpath, 'w') as f:
460 # Derived classes need to implement this. It's called once basic checking
461 # has been performend.
462 def gotorevisionx(self, rev):
463 raise VCSException("This VCS type doesn't define gotorevisionx")
465 # Initialise and update submodules
466 def initsubmodules(self):
467 raise VCSException('Submodules not supported for this vcs type')
469 # Get a list of all known tags
471 raise VCSException('gettags not supported for this vcs type')
473 # Get a list of latest number tags
474 def latesttags(self, number):
475 raise VCSException('latesttags not supported for this vcs type')
477 # Get current commit reference (hash, revision, etc)
479 raise VCSException('getref not supported for this vcs type')
481 # Returns the srclib (name, path) used in setting up the current
492 # If the local directory exists, but is somehow not a git repository, git
493 # will traverse up the directory tree until it finds one that is (i.e.
494 # fdroidserver) and then we'll proceed to destroy it! This is called as
497 p = SilentPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
498 result = p.output.rstrip()
499 if not result.endswith(self.local):
500 raise VCSException('Repository mismatch')
502 def gotorevisionx(self, rev):
503 if not os.path.exists(self.local):
505 p = FDroidPopen(['git', 'clone', self.remote, self.local])
506 if p.returncode != 0:
507 self.clone_failed = True
508 raise VCSException("Git clone failed", p.output)
512 # Discard any working tree changes
513 p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
514 if p.returncode != 0:
515 raise VCSException("Git reset failed", p.output)
516 # Remove untracked files now, in case they're tracked in the target
517 # revision (it happens!)
518 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
519 if p.returncode != 0:
520 raise VCSException("Git clean failed", p.output)
521 if not self.refreshed:
522 # Get latest commits and tags from remote
523 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
524 if p.returncode != 0:
525 raise VCSException("Git fetch failed", p.output)
526 p = SilentPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local)
527 if p.returncode != 0:
528 raise VCSException("Git fetch failed", p.output)
529 # Recreate origin/HEAD as git clone would do it, in case it disappeared
530 p = SilentPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local)
531 if p.returncode != 0:
532 lines = p.output.splitlines()
533 if 'Multiple remote HEAD branches' not in lines[0]:
534 raise VCSException("Git remote set-head failed", p.output)
535 branch = lines[1].split(' ')[-1]
536 p2 = SilentPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local)
537 if p2.returncode != 0:
538 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
539 self.refreshed = True
540 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
541 # a github repo. Most of the time this is the same as origin/master.
542 rev = rev or 'origin/HEAD'
543 p = SilentPopen(['git', 'checkout', '-f', rev], cwd=self.local)
544 if p.returncode != 0:
545 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
546 # Get rid of any uncontrolled files left behind
547 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
548 if p.returncode != 0:
549 raise VCSException("Git clean failed", p.output)
551 def initsubmodules(self):
553 submfile = os.path.join(self.local, '.gitmodules')
554 if not os.path.isfile(submfile):
555 raise VCSException("No git submodules available")
557 # fix submodules not accessible without an account and public key auth
558 with open(submfile, 'r') as f:
559 lines = f.readlines()
560 with open(submfile, 'w') as f:
562 if 'git@github.com' in line:
563 line = line.replace('git@github.com:', 'https://github.com/')
567 ['git', 'reset', '--hard'],
568 ['git', 'clean', '-dffx'],
570 p = SilentPopen(['git', 'submodule', 'foreach', '--recursive'] + cmd, cwd=self.local)
571 if p.returncode != 0:
572 raise VCSException("Git submodule reset failed", p.output)
573 p = SilentPopen(['git', 'submodule', 'sync'], cwd=self.local)
574 if p.returncode != 0:
575 raise VCSException("Git submodule sync failed", p.output)
576 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
577 if p.returncode != 0:
578 raise VCSException("Git submodule update failed", p.output)
582 p = SilentPopen(['git', 'tag'], cwd=self.local)
583 return p.output.splitlines()
585 def latesttags(self, alltags, number):
587 p = SilentPopen(['echo "' + '\n'.join(alltags) + '" | '
588 + 'xargs -I@ git log --format=format:"%at @%n" -1 @ | '
589 + 'sort -n | awk \'{print $2}\''],
590 cwd=self.local, shell=True)
591 return p.output.splitlines()[-number:]
594 class vcs_gitsvn(vcs):
599 # Damn git-svn tries to use a graphical password prompt, so we have to
600 # trick it into taking the password from stdin
602 if self.username is None:
604 return ('echo "%s" | DISPLAY="" ' % self.password, ' --username "%s"' % self.username)
606 # If the local directory exists, but is somehow not a git repository, git
607 # will traverse up the directory tree until it finds one that is (i.e.
608 # fdroidserver) and then we'll proceed to destory it! This is called as
611 p = SilentPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
612 result = p.output.rstrip()
613 if not result.endswith(self.local):
614 raise VCSException('Repository mismatch')
616 def gotorevisionx(self, rev):
617 if not os.path.exists(self.local):
619 gitsvn_cmd = '%sgit svn clone%s' % self.userargs()
620 if ';' in self.remote:
621 remote_split = self.remote.split(';')
622 for i in remote_split[1:]:
623 if i.startswith('trunk='):
624 gitsvn_cmd += ' -T %s' % i[6:]
625 elif i.startswith('tags='):
626 gitsvn_cmd += ' -t %s' % i[5:]
627 elif i.startswith('branches='):
628 gitsvn_cmd += ' -b %s' % i[9:]
629 p = SilentPopen([gitsvn_cmd + " %s %s" % (remote_split[0], self.local)], shell=True)
630 if p.returncode != 0:
631 self.clone_failed = True
632 raise VCSException("Git svn clone failed", p.output)
634 p = SilentPopen([gitsvn_cmd + " %s %s" % (self.remote, self.local)], shell=True)
635 if p.returncode != 0:
636 self.clone_failed = True
637 raise VCSException("Git svn clone failed", p.output)
641 # Discard any working tree changes
642 p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
643 if p.returncode != 0:
644 raise VCSException("Git reset failed", p.output)
645 # Remove untracked files now, in case they're tracked in the target
646 # revision (it happens!)
647 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
648 if p.returncode != 0:
649 raise VCSException("Git clean failed", p.output)
650 if not self.refreshed:
651 # Get new commits, branches and tags from repo
652 p = SilentPopen(['%sgit svn fetch %s' % self.userargs()], cwd=self.local, shell=True)
653 if p.returncode != 0:
654 raise VCSException("Git svn fetch failed")
655 p = SilentPopen(['%sgit svn rebase %s' % self.userargs()], cwd=self.local, shell=True)
656 if p.returncode != 0:
657 raise VCSException("Git svn rebase failed", p.output)
658 self.refreshed = True
660 rev = rev or 'master'
662 nospaces_rev = rev.replace(' ', '%20')
663 # Try finding a svn tag
664 for treeish in ['origin/', '']:
665 p = SilentPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev],
667 if p.returncode == 0:
669 if p.returncode != 0:
670 # No tag found, normal svn rev translation
671 # Translate svn rev into git format
672 rev_split = rev.split('/')
675 for treeish in ['origin/', '']:
676 if len(rev_split) > 1:
677 treeish += rev_split[0]
678 svn_rev = rev_split[1]
681 # if no branch is specified, then assume trunk (i.e. 'master' branch):
685 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
687 p = SilentPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish],
689 git_rev = p.output.rstrip()
691 if p.returncode == 0 and git_rev:
694 if p.returncode != 0 or not git_rev:
695 # Try a plain git checkout as a last resort
696 p = SilentPopen(['git', 'checkout', rev], cwd=self.local)
697 if p.returncode != 0:
698 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
700 # Check out the git rev equivalent to the svn rev
701 p = SilentPopen(['git', 'checkout', git_rev], cwd=self.local)
702 if p.returncode != 0:
703 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
705 # Get rid of any uncontrolled files left behind
706 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
707 if p.returncode != 0:
708 raise VCSException("Git clean failed", p.output)
712 for treeish in ['origin/', '']:
713 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
719 p = SilentPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local)
720 if p.returncode != 0:
722 return p.output.strip()
730 def gotorevisionx(self, rev):
731 if not os.path.exists(self.local):
732 p = SilentPopen(['hg', 'clone', self.remote, self.local])
733 if p.returncode != 0:
734 self.clone_failed = True
735 raise VCSException("Hg clone failed", p.output)
737 p = SilentPopen(['hg status -uS | xargs rm -rf'], cwd=self.local, shell=True)
738 if p.returncode != 0:
739 raise VCSException("Hg clean failed", p.output)
740 if not self.refreshed:
741 p = SilentPopen(['hg', 'pull'], cwd=self.local)
742 if p.returncode != 0:
743 raise VCSException("Hg pull failed", p.output)
744 self.refreshed = True
746 rev = rev or 'default'
749 p = SilentPopen(['hg', 'update', '-C', rev], cwd=self.local)
750 if p.returncode != 0:
751 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
752 p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
753 # Also delete untracked files, we have to enable purge extension for that:
754 if "'purge' is provided by the following extension" in p.output:
755 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
756 myfile.write("\n[extensions]\nhgext.purge=\n")
757 p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
758 if p.returncode != 0:
759 raise VCSException("HG purge failed", p.output)
760 elif p.returncode != 0:
761 raise VCSException("HG purge failed", p.output)
764 p = SilentPopen(['hg', 'tags', '-q'], cwd=self.local)
765 return p.output.splitlines()[1:]
773 def gotorevisionx(self, rev):
774 if not os.path.exists(self.local):
775 p = SilentPopen(['bzr', 'branch', self.remote, self.local])
776 if p.returncode != 0:
777 self.clone_failed = True
778 raise VCSException("Bzr branch failed", p.output)
780 p = SilentPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local)
781 if p.returncode != 0:
782 raise VCSException("Bzr revert failed", p.output)
783 if not self.refreshed:
784 p = SilentPopen(['bzr', 'pull'], cwd=self.local)
785 if p.returncode != 0:
786 raise VCSException("Bzr update failed", p.output)
787 self.refreshed = True
789 revargs = list(['-r', rev] if rev else [])
790 p = SilentPopen(['bzr', 'revert'] + revargs, cwd=self.local)
791 if p.returncode != 0:
792 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
795 p = SilentPopen(['bzr', 'tags'], cwd=self.local)
796 return [tag.split(' ')[0].strip() for tag in
797 p.output.splitlines()]
800 def retrieve_string(app_dir, string, xmlfiles=None):
803 os.path.join(app_dir, 'res'),
804 os.path.join(app_dir, 'src', 'main'),
809 for res_dir in res_dirs:
810 for r, d, f in os.walk(res_dir):
811 if os.path.basename(r) == 'values':
812 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
815 if string.startswith('@string/'):
816 string_search = re.compile(r'.*name="' + string[8:] + '".*?>"?([^<]+?)"?<.*').search
817 elif string.startswith('&') and string.endswith(';'):
818 string_search = re.compile(r'.*<!ENTITY.*' + string[1:-1] + '.*?"([^"]+?)".*>').search
820 if string_search is not None:
821 for xmlfile in xmlfiles:
822 for line in file(xmlfile):
823 matches = string_search(line)
825 return retrieve_string(app_dir, matches.group(1), xmlfiles)
828 return string.replace("\\'", "'")
831 # Return list of existing files that will be used to find the highest vercode
832 def manifest_paths(app_dir, flavours):
834 possible_manifests = \
835 [os.path.join(app_dir, 'AndroidManifest.xml'),
836 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
837 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
838 os.path.join(app_dir, 'build.gradle')]
840 for flavour in flavours:
843 possible_manifests.append(
844 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
846 return [path for path in possible_manifests if os.path.isfile(path)]
849 # Retrieve the package name. Returns the name, or None if not found.
850 def fetch_real_name(app_dir, flavours):
851 app_search = re.compile(r'.*<application.*').search
852 name_search = re.compile(r'.*android:label="([^"]+)".*').search
854 for f in manifest_paths(app_dir, flavours):
855 if not has_extension(f, 'xml'):
857 logging.debug("fetch_real_name: Checking manifest at " + f)
863 matches = name_search(line)
865 stringname = matches.group(1)
866 logging.debug("fetch_real_name: using string " + stringname)
867 result = retrieve_string(app_dir, stringname)
869 result = result.strip()
874 # Retrieve the version name
875 def version_name(original, app_dir, flavours):
876 for f in manifest_paths(app_dir, flavours):
877 if not has_extension(f, 'xml'):
879 string = retrieve_string(app_dir, original)
885 def get_library_references(root_dir):
887 proppath = os.path.join(root_dir, 'project.properties')
888 if not os.path.isfile(proppath):
890 with open(proppath) as f:
891 for line in f.readlines():
892 if not line.startswith('android.library.reference.'):
894 path = line.split('=')[1].strip()
895 relpath = os.path.join(root_dir, path)
896 if not os.path.isdir(relpath):
898 logging.debug("Found subproject at %s" % path)
899 libraries.append(path)
903 def ant_subprojects(root_dir):
904 subprojects = get_library_references(root_dir)
905 for subpath in subprojects:
906 subrelpath = os.path.join(root_dir, subpath)
907 for p in get_library_references(subrelpath):
908 relp = os.path.normpath(os.path.join(subpath, p))
909 if relp not in subprojects:
910 subprojects.insert(0, relp)
914 def remove_debuggable_flags(root_dir):
915 # Remove forced debuggable flags
916 logging.debug("Removing debuggable flags from %s" % root_dir)
917 for root, dirs, files in os.walk(root_dir):
918 if 'AndroidManifest.xml' in files:
919 path = os.path.join(root, 'AndroidManifest.xml')
920 p = SilentPopen(['sed', '-i', 's/android:debuggable="[^"]*"//g', path])
921 if p.returncode != 0:
922 raise BuildException("Failed to remove debuggable flags of %s" % path)
925 # Extract some information from the AndroidManifest.xml at the given path.
926 # Returns (version, vercode, package), any or all of which might be None.
927 # All values returned are strings.
928 def parse_androidmanifests(paths, ignoreversions=None):
931 return (None, None, None)
933 vcsearch = re.compile(r'.*:versionCode="([0-9]+?)".*').search
934 vnsearch = re.compile(r'.*:versionName="([^"]+?)".*').search
935 psearch = re.compile(r'.*package="([^"]+)".*').search
937 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
938 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
939 psearch_g = re.compile(r'.*packageName *=* *["\']([^"]+)["\'].*').search
941 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
949 gradle = has_extension(path, 'gradle')
952 # Remember package name, may be defined separately from version+vercode
953 package = max_package
955 for line in file(path):
958 matches = psearch_g(line)
960 matches = psearch(line)
962 package = matches.group(1)
965 matches = vnsearch_g(line)
967 matches = vnsearch(line)
969 version = matches.group(2 if gradle else 1)
972 matches = vcsearch_g(line)
974 matches = vcsearch(line)
976 vercode = matches.group(1)
978 # Always grab the package name and version name in case they are not
979 # together with the highest version code
980 if max_package is None and package is not None:
981 max_package = package
982 if max_version is None and version is not None:
983 max_version = version
985 if max_vercode is None or (vercode is not None and vercode > max_vercode):
986 if not ignoresearch or not ignoresearch(version):
987 if version is not None:
988 max_version = version
989 if vercode is not None:
990 max_vercode = vercode
991 if package is not None:
992 max_package = package
994 max_version = "Ignore"
996 if max_version is None:
997 max_version = "Unknown"
999 return (max_version, max_vercode, max_package)
1002 class FDroidException(Exception):
1003 def __init__(self, value, detail=None):
1005 self.detail = detail
1007 def get_wikitext(self):
1008 ret = repr(self.value) + "\n"
1012 txt = self.detail[-8192:] if len(self.detail) > 8192 else self.detail
1020 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1024 class VCSException(FDroidException):
1028 class BuildException(FDroidException):
1032 # Get the specified source library.
1033 # Returns the path to it. Normally this is the path to be used when referencing
1034 # it, which may be a subdirectory of the actual project. If you want the base
1035 # directory of the project, pass 'basepath=True'.
1036 def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
1037 basepath=False, raw=False, prepare=True, preponly=False):
1045 name, ref = spec.split('@')
1047 number, name = name.split(':', 1)
1049 name, subdir = name.split('/', 1)
1051 if name not in metadata.srclibs:
1052 raise VCSException('srclib ' + name + ' not found.')
1054 srclib = metadata.srclibs[name]
1056 sdir = os.path.join(srclib_dir, name)
1059 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1060 vcs.srclib = (name, number, sdir)
1062 vcs.gotorevision(ref)
1069 libdir = os.path.join(sdir, subdir)
1070 elif srclib["Subdir"]:
1071 for subdir in srclib["Subdir"]:
1072 libdir_candidate = os.path.join(sdir, subdir)
1073 if os.path.exists(libdir_candidate):
1074 libdir = libdir_candidate
1080 if srclib["Srclibs"]:
1082 for lib in srclib["Srclibs"].replace(';', ',').split(','):
1084 for t in srclibpaths:
1089 raise VCSException('Missing recursive srclib %s for %s' % (
1091 place_srclib(libdir, n, s_tuple[2])
1094 remove_signing_keys(sdir)
1095 remove_debuggable_flags(sdir)
1099 if srclib["Prepare"]:
1100 cmd = replace_config_vars(srclib["Prepare"])
1102 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1103 if p.returncode != 0:
1104 raise BuildException("Error running prepare command for srclib %s"
1110 return (name, number, libdir)
1113 # Prepare the source code for a particular build
1114 # 'vcs' - the appropriate vcs object for the application
1115 # 'app' - the application details from the metadata
1116 # 'build' - the build details from the metadata
1117 # 'build_dir' - the path to the build directory, usually
1119 # 'srclib_dir' - the path to the source libraries directory, usually
1121 # 'extlib_dir' - the path to the external libraries directory, usually
1123 # Returns the (root, srclibpaths) where:
1124 # 'root' is the root directory, which may be the same as 'build_dir' or may
1125 # be a subdirectory of it.
1126 # 'srclibpaths' is information on the srclibs being used
1127 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False):
1129 # Optionally, the actual app source can be in a subdirectory
1131 root_dir = os.path.join(build_dir, build['subdir'])
1133 root_dir = build_dir
1135 # Get a working copy of the right revision
1136 logging.info("Getting source for revision " + build['commit'])
1137 vcs.gotorevision(build['commit'])
1139 # Initialise submodules if requred
1140 if build['submodules']:
1141 logging.info("Initialising submodules")
1142 vcs.initsubmodules()
1144 # Check that a subdir (if we're using one) exists. This has to happen
1145 # after the checkout, since it might not exist elsewhere
1146 if not os.path.exists(root_dir):
1147 raise BuildException('Missing subdir ' + root_dir)
1149 # Run an init command if one is required
1151 cmd = replace_config_vars(build['init'])
1152 logging.info("Running 'init' commands in %s" % root_dir)
1154 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1155 if p.returncode != 0:
1156 raise BuildException("Error running init command for %s:%s" %
1157 (app['id'], build['version']), p.output)
1159 # Apply patches if any
1161 logging.info("Applying patches")
1162 for patch in build['patch']:
1163 patch = patch.strip()
1164 logging.info("Applying " + patch)
1165 patch_path = os.path.join('metadata', app['id'], patch)
1166 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1167 if p.returncode != 0:
1168 raise BuildException("Failed to apply patch %s" % patch_path)
1170 # Get required source libraries
1172 if build['srclibs']:
1173 logging.info("Collecting source libraries")
1174 for lib in build['srclibs']:
1175 srclibpaths.append(getsrclib(lib, srclib_dir, srclibpaths,
1178 for name, number, libpath in srclibpaths:
1179 place_srclib(root_dir, int(number) if number else None, libpath)
1181 basesrclib = vcs.getsrclib()
1182 # If one was used for the main source, add that too.
1184 srclibpaths.append(basesrclib)
1186 # Update the local.properties file
1187 localprops = [os.path.join(build_dir, 'local.properties')]
1189 localprops += [os.path.join(root_dir, 'local.properties')]
1190 for path in localprops:
1192 if os.path.isfile(path):
1193 logging.info("Updating local.properties file at %s" % path)
1199 logging.info("Creating local.properties file at %s" % path)
1200 # Fix old-fashioned 'sdk-location' by copying
1201 # from sdk.dir, if necessary
1202 if build['oldsdkloc']:
1203 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1204 re.S | re.M).group(1)
1205 props += "sdk-location=%s\n" % sdkloc
1207 props += "sdk.dir=%s\n" % config['sdk_path']
1208 props += "sdk-location=%s\n" % config['sdk_path']
1209 if config['ndk_path']:
1211 props += "ndk.dir=%s\n" % config['ndk_path']
1212 props += "ndk-location=%s\n" % config['ndk_path']
1213 # Add java.encoding if necessary
1214 if build['encoding']:
1215 props += "java.encoding=%s\n" % build['encoding']
1221 if build['type'] == 'gradle':
1222 flavours = build['gradle']
1224 version_regex = re.compile(r".*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1225 gradlepluginver = None
1227 gradle_files = [os.path.join(root_dir, 'build.gradle')]
1229 # Parent dir build.gradle
1230 parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
1231 if parent_dir.startswith(build_dir):
1232 gradle_files.append(os.path.join(parent_dir, 'build.gradle'))
1234 for path in gradle_files:
1237 if not os.path.isfile(path):
1239 with open(path) as f:
1241 match = version_regex.match(line)
1243 gradlepluginver = match.group(1)
1247 build['gradlepluginver'] = LooseVersion(gradlepluginver)
1249 logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
1250 build['gradlepluginver'] = LooseVersion('0.11')
1253 n = build["target"].split('-')[1]
1254 SilentPopen(['sed', '-i',
1255 's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g',
1259 # Remove forced debuggable flags
1260 remove_debuggable_flags(root_dir)
1262 # Insert version code and number into the manifest if necessary
1263 if build['forceversion']:
1264 logging.info("Changing the version name")
1265 for path in manifest_paths(root_dir, flavours):
1266 if not os.path.isfile(path):
1268 if has_extension(path, 'xml'):
1269 p = SilentPopen(['sed', '-i',
1270 's/android:versionName="[^"]*"/android:versionName="'
1271 + build['version'] + '"/g',
1273 if p.returncode != 0:
1274 raise BuildException("Failed to amend manifest")
1275 elif has_extension(path, 'gradle'):
1276 p = SilentPopen(['sed', '-i',
1277 's/versionName *=* *"[^"]*"/versionName = "'
1278 + build['version'] + '"/g',
1280 if p.returncode != 0:
1281 raise BuildException("Failed to amend build.gradle")
1282 if build['forcevercode']:
1283 logging.info("Changing the version code")
1284 for path in manifest_paths(root_dir, flavours):
1285 if not os.path.isfile(path):
1287 if has_extension(path, 'xml'):
1288 p = SilentPopen(['sed', '-i',
1289 's/android:versionCode="[^"]*"/android:versionCode="'
1290 + build['vercode'] + '"/g',
1292 if p.returncode != 0:
1293 raise BuildException("Failed to amend manifest")
1294 elif has_extension(path, 'gradle'):
1295 p = SilentPopen(['sed', '-i',
1296 's/versionCode *=* *[0-9]*/versionCode = '
1297 + build['vercode'] + '/g',
1299 if p.returncode != 0:
1300 raise BuildException("Failed to amend build.gradle")
1302 # Delete unwanted files
1304 logging.info("Removing specified files")
1305 for part in getpaths(build_dir, build, 'rm'):
1306 dest = os.path.join(build_dir, part)
1307 logging.info("Removing {0}".format(part))
1308 if os.path.lexists(dest):
1309 if os.path.islink(dest):
1310 SilentPopen(['unlink ' + dest], shell=True)
1312 SilentPopen(['rm -rf ' + dest], shell=True)
1314 logging.info("...but it didn't exist")
1316 remove_signing_keys(build_dir)
1318 # Add required external libraries
1319 if build['extlibs']:
1320 logging.info("Collecting prebuilt libraries")
1321 libsdir = os.path.join(root_dir, 'libs')
1322 if not os.path.exists(libsdir):
1324 for lib in build['extlibs']:
1326 logging.info("...installing extlib {0}".format(lib))
1327 libf = os.path.basename(lib)
1328 libsrc = os.path.join(extlib_dir, lib)
1329 if not os.path.exists(libsrc):
1330 raise BuildException("Missing extlib file {0}".format(libsrc))
1331 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1333 # Run a pre-build command if one is required
1334 if build['prebuild']:
1335 logging.info("Running 'prebuild' commands in %s" % root_dir)
1337 cmd = replace_config_vars(build['prebuild'])
1339 # Substitute source library paths into prebuild commands
1340 for name, number, libpath in srclibpaths:
1341 libpath = os.path.relpath(libpath, root_dir)
1342 cmd = cmd.replace('$$' + name + '$$', libpath)
1344 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1345 if p.returncode != 0:
1346 raise BuildException("Error running prebuild command for %s:%s" %
1347 (app['id'], build['version']), p.output)
1349 # Generate (or update) the ant build file, build.xml...
1350 if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
1351 parms = [config['android'], 'update', 'lib-project']
1352 lparms = [config['android'], 'update', 'project']
1355 parms += ['-t', build['target']]
1356 lparms += ['-t', build['target']]
1357 if build['update'] == ['auto']:
1358 update_dirs = ant_subprojects(root_dir) + ['.']
1360 update_dirs = build['update']
1362 for d in update_dirs:
1363 subdir = os.path.join(root_dir, d)
1365 logging.debug("Updating main project")
1366 cmd = parms + ['-p', d]
1368 logging.debug("Updating subproject %s" % d)
1369 cmd = lparms + ['-p', d]
1370 p = FDroidPopen(cmd, cwd=root_dir)
1371 # Check to see whether an error was returned without a proper exit
1372 # code (this is the case for the 'no target set or target invalid'
1374 if p.returncode != 0 or p.output.startswith("Error: "):
1375 raise BuildException("Failed to update project at %s" % d, p.output)
1376 # Clean update dirs via ant
1378 logging.info("Cleaning subproject %s" % d)
1379 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1381 return (root_dir, srclibpaths)
1384 # Split and extend via globbing the paths from a field
1385 def getpaths(build_dir, build, field):
1387 for p in build[field]:
1389 full_path = os.path.join(build_dir, p)
1390 full_path = os.path.normpath(full_path)
1391 paths += [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1395 # Scan the source code in the given directory (and all subdirectories)
1396 # and return the number of fatal problems encountered
1397 def scan_source(build_dir, root_dir, thisbuild):
1401 # Common known non-free blobs (always lower case):
1403 re.compile(r'flurryagent', re.IGNORECASE),
1404 re.compile(r'paypal.*mpl', re.IGNORECASE),
1405 re.compile(r'google.*analytics', re.IGNORECASE),
1406 re.compile(r'admob.*sdk.*android', re.IGNORECASE),
1407 re.compile(r'google.*ad.*view', re.IGNORECASE),
1408 re.compile(r'google.*admob', re.IGNORECASE),
1409 re.compile(r'google.*play.*services', re.IGNORECASE),
1410 re.compile(r'crittercism', re.IGNORECASE),
1411 re.compile(r'heyzap', re.IGNORECASE),
1412 re.compile(r'jpct.*ae', re.IGNORECASE),
1413 re.compile(r'youtube.*android.*player.*api', re.IGNORECASE),
1414 re.compile(r'bugsense', re.IGNORECASE),
1415 re.compile(r'crashlytics', re.IGNORECASE),
1416 re.compile(r'ouya.*sdk', re.IGNORECASE),
1417 re.compile(r'libspen23', re.IGNORECASE),
1420 scanignore = getpaths(build_dir, thisbuild, 'scanignore')
1421 scandelete = getpaths(build_dir, thisbuild, 'scandelete')
1424 ms = magic.open(magic.MIME_TYPE)
1426 except AttributeError:
1430 for i in scanignore:
1431 if fd.startswith(i):
1436 for i in scandelete:
1437 if fd.startswith(i):
1441 def removeproblem(what, fd, fp):
1442 logging.info('Removing %s at %s' % (what, fd))
1445 def warnproblem(what, fd):
1446 logging.warn('Found %s at %s' % (what, fd))
1448 def handleproblem(what, fd, fp):
1450 logging.info('Ignoring %s at %s' % (what, fd))
1452 removeproblem(what, fd, fp)
1454 logging.error('Found %s at %s' % (what, fd))
1458 # Iterate through all files in the source code
1459 for r, d, f in os.walk(build_dir, topdown=True):
1461 # It's topdown, so checking the basename is enough
1462 for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
1468 # Path (relative) to the file
1469 fp = os.path.join(r, curfile)
1470 fd = fp[len(build_dir) + 1:]
1473 mime = magic.from_file(fp, mime=True) if ms is None else ms.file(fp)
1474 except UnicodeError:
1475 warnproblem('malformed magic number', fd)
1477 if mime == 'application/x-sharedlib':
1478 count += handleproblem('shared library', fd, fp)
1480 elif mime == 'application/x-archive':
1481 count += handleproblem('static library', fd, fp)
1483 elif mime == 'application/x-executable':
1484 count += handleproblem('binary executable', fd, fp)
1486 elif mime == 'application/x-java-applet':
1487 count += handleproblem('Java compiled class', fd, fp)
1492 'application/java-archive',
1493 'application/octet-stream',
1497 if has_extension(fp, 'apk'):
1498 removeproblem('APK file', fd, fp)
1500 elif has_extension(fp, 'jar'):
1502 if any(suspect.match(curfile) for suspect in usual_suspects):
1503 count += handleproblem('usual supect', fd, fp)
1505 warnproblem('JAR file', fd)
1507 elif has_extension(fp, 'zip'):
1508 warnproblem('ZIP file', fd)
1511 warnproblem('unknown compressed or binary file', fd)
1513 elif has_extension(fp, 'java'):
1514 for line in file(fp):
1515 if 'DexClassLoader' in line:
1516 count += handleproblem('DexClassLoader', fd, fp)
1521 # Presence of a jni directory without buildjni=yes might
1522 # indicate a problem (if it's not a problem, explicitly use
1523 # buildjni=no to bypass this check)
1524 if (os.path.exists(os.path.join(root_dir, 'jni')) and
1525 not thisbuild['buildjni']):
1526 logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
1535 self.path = os.path.join('stats', 'known_apks.txt')
1537 if os.path.exists(self.path):
1538 for line in file(self.path):
1539 t = line.rstrip().split(' ')
1541 self.apks[t[0]] = (t[1], None)
1543 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1544 self.changed = False
1546 def writeifchanged(self):
1548 if not os.path.exists('stats'):
1550 f = open(self.path, 'w')
1552 for apk, app in self.apks.iteritems():
1554 line = apk + ' ' + appid
1556 line += ' ' + time.strftime('%Y-%m-%d', added)
1558 for line in sorted(lst):
1559 f.write(line + '\n')
1562 # Record an apk (if it's new, otherwise does nothing)
1563 # Returns the date it was added.
1564 def recordapk(self, apk, app):
1565 if apk not in self.apks:
1566 self.apks[apk] = (app, time.gmtime(time.time()))
1568 _, added = self.apks[apk]
1571 # Look up information - given the 'apkname', returns (app id, date added/None).
1572 # Or returns None for an unknown apk.
1573 def getapp(self, apkname):
1574 if apkname in self.apks:
1575 return self.apks[apkname]
1578 # Get the most recent 'num' apps added to the repo, as a list of package ids
1579 # with the most recent first.
1580 def getlatest(self, num):
1582 for apk, app in self.apks.iteritems():
1586 if apps[appid] > added:
1590 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1591 lst = [app for app, _ in sortedapps]
1596 def isApkDebuggable(apkfile, config):
1597 """Returns True if the given apk file is debuggable
1599 :param apkfile: full path to the apk to check"""
1601 p = SilentPopen([os.path.join(config['sdk_path'], 'build-tools',
1602 config['build_tools'], 'aapt'),
1603 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'])
1604 if p.returncode != 0:
1605 logging.critical("Failed to get apk manifest information")
1607 for line in p.output.splitlines():
1608 if 'android:debuggable' in line and not line.endswith('0x0'):
1613 class AsynchronousFileReader(threading.Thread):
1615 Helper class to implement asynchronous reading of a file
1616 in a separate thread. Pushes read lines on a queue to
1617 be consumed in another thread.
1620 def __init__(self, fd, queue):
1621 assert isinstance(queue, Queue.Queue)
1622 assert callable(fd.readline)
1623 threading.Thread.__init__(self)
1628 '''The body of the tread: read lines and put them on the queue.'''
1629 for line in iter(self._fd.readline, ''):
1630 self._queue.put(line)
1633 '''Check whether there is no more content to expect.'''
1634 return not self.is_alive() and self._queue.empty()
1642 def SilentPopen(commands, cwd=None, shell=False):
1643 return FDroidPopen(commands, cwd=cwd, shell=shell, output=False)
1646 def FDroidPopen(commands, cwd=None, shell=False, output=True):
1648 Run a command and capture the possibly huge output.
1650 :param commands: command and argument list like in subprocess.Popen
1651 :param cwd: optionally specifies a working directory
1652 :returns: A PopenResult.
1658 cwd = os.path.normpath(cwd)
1659 logging.debug("Directory: %s" % cwd)
1660 logging.debug("> %s" % ' '.join(commands))
1662 result = PopenResult()
1665 p = subprocess.Popen(commands, cwd=cwd, shell=shell, env=env,
1666 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1668 raise BuildException("OSError while trying to execute " + ' '.join(commands) + ': ' + str(e))
1670 stdout_queue = Queue.Queue()
1671 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1672 stdout_reader.start()
1674 # Check the queue for output (until there is no more to get)
1675 while not stdout_reader.eof():
1676 while not stdout_queue.empty():
1677 line = stdout_queue.get()
1678 if output and options.verbose:
1679 # Output directly to console
1680 sys.stderr.write(line)
1682 result.output += line
1686 result.returncode = p.wait()
1690 def remove_signing_keys(build_dir):
1691 comment = re.compile(r'[ ]*//')
1692 signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1694 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1695 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1696 re.compile(r'.*variant\.outputFile = .*'),
1697 re.compile(r'.*\.readLine\(.*'),
1699 for root, dirs, files in os.walk(build_dir):
1700 if 'build.gradle' in files:
1701 path = os.path.join(root, 'build.gradle')
1703 with open(path, "r") as o:
1704 lines = o.readlines()
1709 with open(path, "w") as o:
1711 if comment.match(line):
1715 opened += line.count('{')
1716 opened -= line.count('}')
1719 if signing_configs.match(line):
1724 if any(s.match(line) for s in line_matches):
1732 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1735 'project.properties',
1737 'default.properties',
1740 if propfile in files:
1741 path = os.path.join(root, propfile)
1743 with open(path, "r") as o:
1744 lines = o.readlines()
1748 with open(path, "w") as o:
1750 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1757 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1760 def replace_config_vars(cmd):
1761 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1762 cmd = cmd.replace('$$NDK$$', config['ndk_path'])
1763 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1767 def place_srclib(root_dir, number, libpath):
1770 relpath = os.path.relpath(libpath, root_dir)
1771 proppath = os.path.join(root_dir, 'project.properties')
1774 if os.path.isfile(proppath):
1775 with open(proppath, "r") as o:
1776 lines = o.readlines()
1778 with open(proppath, "w") as o:
1781 if line.startswith('android.library.reference.%d=' % number):
1782 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1787 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1790 def compare_apks(apk1, apk2, tmp_dir):
1793 Returns None if the apk content is the same (apart from the signing key),
1794 otherwise a string describing what's different, or what went wrong when
1795 trying to do the comparison.
1798 thisdir = os.path.join(tmp_dir, 'this_apk')
1799 thatdir = os.path.join(tmp_dir, 'that_apk')
1800 for d in [thisdir, thatdir]:
1801 if os.path.exists(d):
1805 if subprocess.call(['jar', 'xf',
1806 os.path.abspath(apk1)],
1808 return("Failed to unpack " + apk1)
1809 if subprocess.call(['jar', 'xf',
1810 os.path.abspath(apk2)],
1812 return("Failed to unpack " + apk2)
1814 p = FDroidPopen(['diff', '-r', 'this_apk', 'that_apk'], cwd=tmp_dir,
1816 lines = p.output.splitlines()
1817 if len(lines) != 1 or 'META-INF' not in lines[0]:
1818 return("Unexpected diff output - " + p.output)
1820 # If we get here, it seems like they're the same!