1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
37 def get_default_config():
39 'sdk_path': os.getenv("ANDROID_HOME"),
40 'ndk_path': "$ANDROID_NDK",
41 'build_tools': "19.0.3",
46 'update_stats': False,
47 'stats_to_carbon': False,
49 'build_server_always': False,
50 'keystore': '$HOME/.local/share/fdroidserver/keystore.jks',
51 'smartcardoptions': [],
59 def read_config(opts, config_file='config.py'):
60 """Read the repository config
62 The config is read from config_file, which is in the current directory when
63 any of the repo management commands are used.
65 global config, options
67 if config is not None:
69 if not os.path.isfile(config_file):
70 logging.critical("Missing config file - is this a repo directory?")
77 logging.debug("Reading %s" % config_file)
78 execfile(config_file, config)
80 # smartcardoptions must be a list since its command line args for Popen
81 if 'smartcardoptions' in config:
82 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
83 elif 'keystore' in config and config['keystore'] == 'NONE':
84 # keystore='NONE' means use smartcard, these are required defaults
85 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
86 'SunPKCS11-OpenSC', '-providerClass',
87 'sun.security.pkcs11.SunPKCS11',
88 '-providerArg', 'opensc-fdroid.cfg']
90 defconfig = get_default_config()
91 for k, v in defconfig.items():
95 # Expand environment variables
96 for k, v in config.items():
99 v = os.path.expanduser(v)
100 config[k] = os.path.expandvars(v)
102 if not test_sdk_exists(config):
105 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
106 st = os.stat(config_file)
107 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
108 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
110 for k in ["keystorepass", "keypass"]:
112 write_password_file(k)
114 # since this is used with rsync, where trailing slashes have meaning,
115 # ensure there is always a trailing slash
116 if 'serverwebroot' in config:
117 if config['serverwebroot'][-1] != '/':
118 config['serverwebroot'] += '/'
119 config['serverwebroot'] = config['serverwebroot'].replace('//', '/')
123 def test_sdk_exists(c):
124 if c['sdk_path'] == None:
125 # c['sdk_path'] is set to the value of ANDROID_HOME by default
126 logging.critical('No Android SDK found! ANDROID_HOME is not set and sdk_path is not in config.py!')
127 logging.info('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
128 logging.info('\texport ANDROID_HOME=/opt/android-sdk')
130 if not os.path.exists(c['sdk_path']):
131 logging.critical('Android SDK path "' + c['sdk_path'] + '" does not exist!')
133 if not os.path.isdir(c['sdk_path']):
134 logging.critical('Android SDK path "' + c['sdk_path'] + '" is not a directory!')
136 if not os.path.isdir(os.path.join(c['sdk_path'], 'build-tools')):
137 logging.critical('Android SDK path "' + c['sdk_path'] + '" does not contain "build-tools/"!')
141 def write_password_file(pwtype, password=None):
143 writes out passwords to a protected file instead of passing passwords as
144 command line argments
146 filename = '.fdroid.' + pwtype + '.txt'
147 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
149 os.write(fd, config[pwtype])
151 os.write(fd, password)
153 config[pwtype + 'file'] = filename
155 # Given the arguments in the form of multiple appid:[vc] strings, this returns
156 # a dictionary with the set of vercodes specified for each package.
157 def read_pkg_args(args, allow_vercodes=False):
164 if allow_vercodes and ':' in p:
165 package, vercode = p.split(':')
167 package, vercode = p, None
168 if package not in vercodes:
169 vercodes[package] = [vercode] if vercode else []
171 elif vercode and vercode not in vercodes[package]:
172 vercodes[package] += [vercode] if vercode else []
176 # On top of what read_pkg_args does, this returns the whole app metadata, but
177 # limiting the builds list to the builds matching the vercodes specified.
178 def read_app_args(args, allapps, allow_vercodes=False):
180 vercodes = read_pkg_args(args, allow_vercodes)
185 apps = [app for app in allapps if app['id'] in vercodes]
187 if len(apps) != len(vercodes):
188 allids = [app["id"] for app in allapps]
191 logging.critical("No such package: %s" % p)
192 raise Exception("Found invalid app ids in arguments")
194 raise Exception("No packages specified")
198 vc = vercodes[app['id']]
201 app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
202 if len(app['builds']) != len(vercodes[app['id']]):
204 allvcs = [b['vercode'] for b in app['builds']]
205 for v in vercodes[app['id']]:
207 logging.critical("No such vercode %s for app %s" % (v, app['id']))
210 raise Exception("Found invalid vercodes for some apps")
214 def has_extension(filename, extension):
215 name, ext = os.path.splitext(filename)
216 ext = ext.lower()[1:]
217 return ext == extension
221 def apknameinfo(filename):
223 filename = os.path.basename(filename)
224 if apk_regex is None:
225 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
226 m = apk_regex.match(filename)
228 result = (m.group(1), m.group(2))
229 except AttributeError:
230 raise Exception("Invalid apk name: %s" % filename)
233 def getapkname(app, build):
234 return "%s_%s.apk" % (app['id'], build['vercode'])
236 def getsrcname(app, build):
237 return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
243 return app['Auto Name']
247 return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
249 def getvcs(vcstype, remote, local):
251 return vcs_git(remote, local)
253 return vcs_svn(remote, local)
254 if vcstype == 'git-svn':
255 return vcs_gitsvn(remote, local)
257 return vcs_hg(remote, local)
259 return vcs_bzr(remote, local)
260 if vcstype == 'srclib':
261 if local != 'build/srclib/' + remote:
262 raise VCSException("Error: srclib paths are hard-coded!")
263 return getsrclib(remote, 'build/srclib', raw=True)
264 raise VCSException("Invalid vcs type " + vcstype)
266 def getsrclibvcs(name):
267 srclib_path = os.path.join('srclibs', name + ".txt")
268 if not os.path.exists(srclib_path):
269 raise VCSException("Missing srclib " + name)
270 return metadata.parse_srclib(srclib_path)['Repo Type']
273 def __init__(self, remote, local):
275 # svn, git-svn and bzr may require auth
277 if self.repotype() in ('svn', 'git-svn', 'bzr'):
279 self.username, remote = remote.split('@')
280 if ':' not in self.username:
281 raise VCSException("Password required with username")
282 self.username, self.password = self.username.split(':')
286 self.refreshed = False
292 # Take the local repository to a clean version of the given revision, which
293 # is specificed in the VCS's native format. Beforehand, the repository can
294 # be dirty, or even non-existent. If the repository does already exist
295 # locally, it will be updated from the origin, but only once in the
296 # lifetime of the vcs object.
297 # None is acceptable for 'rev' if you know you are cloning a clean copy of
298 # the repo - otherwise it must specify a valid revision.
299 def gotorevision(self, rev):
301 # The .fdroidvcs-id file for a repo tells us what VCS type
302 # and remote that directory was created from, allowing us to drop it
303 # automatically if either of those things changes.
304 fdpath = os.path.join(self.local, '..',
305 '.fdroidvcs-' + os.path.basename(self.local))
306 cdata = self.repotype() + ' ' + self.remote
309 if os.path.exists(self.local):
310 if os.path.exists(fdpath):
311 with open(fdpath, 'r') as f:
312 fsdata = f.read().strip()
317 logging.info("Repository details changed - deleting")
320 logging.info("Repository details missing - deleting")
322 shutil.rmtree(self.local)
324 self.gotorevisionx(rev)
326 # If necessary, write the .fdroidvcs file.
328 with open(fdpath, 'w') as f:
331 # Derived classes need to implement this. It's called once basic checking
332 # has been performend.
333 def gotorevisionx(self, rev):
334 raise VCSException("This VCS type doesn't define gotorevisionx")
336 # Initialise and update submodules
337 def initsubmodules(self):
338 raise VCSException('Submodules not supported for this vcs type')
340 # Get a list of all known tags
342 raise VCSException('gettags not supported for this vcs type')
344 # Get a list of latest number tags
345 def latesttags(self, number):
346 raise VCSException('latesttags not supported for this vcs type')
348 # Get current commit reference (hash, revision, etc)
350 raise VCSException('getref not supported for this vcs type')
352 # Returns the srclib (name, path) used in setting up the current
362 # If the local directory exists, but is somehow not a git repository, git
363 # will traverse up the directory tree until it finds one that is (i.e.
364 # fdroidserver) and then we'll proceed to destroy it! This is called as
367 p = SilentPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
368 result = p.stdout.rstrip()
369 if not result.endswith(self.local):
370 raise VCSException('Repository mismatch')
372 def gotorevisionx(self, rev):
373 if not os.path.exists(self.local):
375 p = FDroidPopen(['git', 'clone', self.remote, self.local])
376 if p.returncode != 0:
377 raise VCSException("Git clone failed")
381 # Discard any working tree changes
382 p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
383 if p.returncode != 0:
384 raise VCSException("Git reset failed")
385 # Remove untracked files now, in case they're tracked in the target
386 # revision (it happens!)
387 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
388 if p.returncode != 0:
389 raise VCSException("Git clean failed")
390 if not self.refreshed:
391 # Get latest commits and tags from remote
392 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
393 if p.returncode != 0:
394 raise VCSException("Git fetch failed")
395 p = SilentPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local)
396 if p.returncode != 0:
397 raise VCSException("Git fetch failed")
398 self.refreshed = True
399 # Check out the appropriate revision
400 rev = str(rev if rev else 'origin/master')
401 p = SilentPopen(['git', 'checkout', '-f', rev], cwd=self.local)
402 if p.returncode != 0:
403 raise VCSException("Git checkout failed")
404 # Get rid of any uncontrolled files left behind
405 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
406 if p.returncode != 0:
407 raise VCSException("Git clean failed")
409 def initsubmodules(self):
411 submfile = os.path.join(self.local, '.gitmodules')
412 if not os.path.isfile(submfile):
413 raise VCSException("No git submodules available")
415 # fix submodules not accessible without an account and public key auth
416 with open(submfile, 'r') as f:
417 lines = f.readlines()
418 with open(submfile, 'w') as f:
420 if 'git@github.com' in line:
421 line = line.replace('git@github.com:', 'https://github.com/')
425 ['git', 'reset', '--hard'],
426 ['git', 'clean', '-dffx'],
428 p = SilentPopen(['git', 'submodule', 'foreach', '--recursive'] + cmd, cwd=self.local)
429 if p.returncode != 0:
430 raise VCSException("Git submodule reset failed")
431 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
432 if p.returncode != 0:
433 raise VCSException("Git submodule update failed")
437 p = SilentPopen(['git', 'tag'], cwd=self.local)
438 return p.stdout.splitlines()
440 def latesttags(self, alltags, number):
442 p = SilentPopen(['echo "'+'\n'.join(alltags)+'" | \
443 xargs -I@ git log --format=format:"%at @%n" -1 @ | \
444 sort -n | awk \'{print $2}\''],
445 cwd=self.local, shell=True)
446 return p.stdout.splitlines()[-number:]
449 class vcs_gitsvn(vcs):
454 # Damn git-svn tries to use a graphical password prompt, so we have to
455 # trick it into taking the password from stdin
457 if self.username is None:
459 return ('echo "%s" | DISPLAY="" ' % self.password, ' --username "%s"' % self.username)
461 # If the local directory exists, but is somehow not a git repository, git
462 # will traverse up the directory tree until it finds one that is (i.e.
463 # fdroidserver) and then we'll proceed to destory it! This is called as
466 p = SilentPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
467 result = p.stdout.rstrip()
468 if not result.endswith(self.local):
469 raise VCSException('Repository mismatch')
471 def gotorevisionx(self, rev):
472 if not os.path.exists(self.local):
474 gitsvn_cmd = '%sgit svn clone%s' % self.userargs()
475 if ';' in self.remote:
476 remote_split = self.remote.split(';')
477 for i in remote_split[1:]:
478 if i.startswith('trunk='):
479 gitsvn_cmd += ' -T %s' % i[6:]
480 elif i.startswith('tags='):
481 gitsvn_cmd += ' -t %s' % i[5:]
482 elif i.startswith('branches='):
483 gitsvn_cmd += ' -b %s' % i[9:]
484 p = SilentPopen([gitsvn_cmd + " %s %s" % (remote_split[0], self.local)], shell=True)
485 if p.returncode != 0:
486 raise VCSException("Git clone failed")
488 p = SilentPopen([gitsvn_cmd + " %s %s" % (self.remote, self.local)], shell=True)
489 if p.returncode != 0:
490 raise VCSException("Git clone failed")
494 # Discard any working tree changes
495 p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
496 if p.returncode != 0:
497 raise VCSException("Git reset failed")
498 # Remove untracked files now, in case they're tracked in the target
499 # revision (it happens!)
500 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
501 if p.returncode != 0:
502 raise VCSException("Git clean failed")
503 if not self.refreshed:
504 # Get new commits, branches and tags from repo
505 p = SilentPopen(['%sgit svn fetch %s' % self.userargs()], cwd=self.local, shell=True)
506 if p.returncode != 0:
507 raise VCSException("Git svn fetch failed")
508 p = SilentPopen(['%sgit svn rebase %s' % self.userargs()], cwd=self.local, shell=True)
509 if p.returncode != 0:
510 raise VCSException("Git svn rebase failed")
511 self.refreshed = True
513 rev = str(rev if rev else 'master')
515 nospaces_rev = rev.replace(' ', '%20')
516 # Try finding a svn tag
517 p = SilentPopen(['git', 'checkout', 'tags/' + nospaces_rev], cwd=self.local)
518 if p.returncode != 0:
519 # No tag found, normal svn rev translation
520 # Translate svn rev into git format
521 rev_split = rev.split('/')
522 if len(rev_split) > 1:
523 treeish = rev_split[0]
524 svn_rev = rev_split[1]
527 # if no branch is specified, then assume trunk (ie. 'master'
532 p = SilentPopen(['git', 'svn', 'find-rev', 'r' + svn_rev, treeish], cwd=self.local)
533 git_rev = p.stdout.rstrip()
535 if p.returncode != 0 or not git_rev:
536 # Try a plain git checkout as a last resort
537 p = SilentPopen(['git', 'checkout', rev], cwd=self.local)
538 if p.returncode != 0:
539 raise VCSException("No git treeish found and direct git checkout failed")
541 # Check out the git rev equivalent to the svn rev
542 p = SilentPopen(['git', 'checkout', git_rev], cwd=self.local)
543 if p.returncode != 0:
544 raise VCSException("Git svn checkout failed")
546 # Get rid of any uncontrolled files left behind
547 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
548 if p.returncode != 0:
549 raise VCSException("Git clean failed")
553 return os.listdir(os.path.join(self.local, '.git/svn/refs/remotes/tags'))
557 p = SilentPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local)
558 if p.returncode != 0:
560 return p.stdout.strip()
568 if self.username is None:
569 return ['--non-interactive']
570 return ['--username', self.username,
571 '--password', self.password,
574 def gotorevisionx(self, rev):
575 if not os.path.exists(self.local):
576 p = SilentPopen(['svn', 'checkout', self.remote, self.local] + self.userargs())
577 if p.returncode != 0:
578 raise VCSException("Svn checkout failed")
582 r"svn status | awk '/\?/ {print $2}' | xargs rm -rf"):
583 p = SilentPopen([svncommand], cwd=self.local, shell=True)
584 if p.returncode != 0:
585 raise VCSException("Svn reset ({0}) failed in {1}".format(svncommand, self.local))
586 if not self.refreshed:
587 p = SilentPopen(['svn', 'update'] + self.userargs(), cwd=self.local)
588 if p.returncode != 0:
589 raise VCSException("Svn update failed")
590 self.refreshed = True
592 revargs = list(['-r', rev] if rev else [])
593 p = SilentPopen(['svn', 'update', '--force'] + revargs + self.userargs(), cwd=self.local)
594 if p.returncode != 0:
595 raise VCSException("Svn update failed")
598 p = SilentPopen(['svn', 'info'], cwd=self.local)
599 for line in p.stdout.splitlines():
600 if line and line.startswith('Last Changed Rev: '):
609 def gotorevisionx(self, rev):
610 if not os.path.exists(self.local):
611 p = SilentPopen(['hg', 'clone', self.remote, self.local])
612 if p.returncode != 0:
613 raise VCSException("Hg clone failed")
615 p = SilentPopen(['hg status -uS | xargs rm -rf'], cwd=self.local, shell=True)
616 if p.returncode != 0:
617 raise VCSException("Hg clean failed")
618 if not self.refreshed:
619 p = SilentPopen(['hg', 'pull'], cwd=self.local)
620 if p.returncode != 0:
621 raise VCSException("Hg pull failed")
622 self.refreshed = True
624 rev = str(rev if rev else 'default')
627 p = SilentPopen(['hg', 'update', '-C', rev], cwd=self.local)
628 if p.returncode != 0:
629 raise VCSException("Hg checkout failed")
630 p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
631 # Also delete untracked files, we have to enable purge extension for that:
632 if "'purge' is provided by the following extension" in p.stdout:
633 with open(self.local+"/.hg/hgrc", "a") as myfile:
634 myfile.write("\n[extensions]\nhgext.purge=\n")
635 p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
636 if p.returncode != 0:
637 raise VCSException("HG purge failed")
638 elif p.returncode != 0:
639 raise VCSException("HG purge failed")
642 p = SilentPopen(['hg', 'tags', '-q'], cwd=self.local)
643 return p.stdout.splitlines()[1:]
651 def gotorevisionx(self, rev):
652 if not os.path.exists(self.local):
653 p = SilentPopen(['bzr', 'branch', self.remote, self.local])
654 if p.returncode != 0:
655 raise VCSException("Bzr branch failed")
657 p = SilentPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local)
658 if p.returncode != 0:
659 raise VCSException("Bzr revert failed")
660 if not self.refreshed:
661 p = SilentPopen(['bzr', 'pull'], cwd=self.local)
662 if p.returncode != 0:
663 raise VCSException("Bzr update failed")
664 self.refreshed = True
666 revargs = list(['-r', rev] if rev else [])
667 p = SilentPopen(['bzr', 'revert'] + revargs, cwd=self.local)
668 if p.returncode != 0:
669 raise VCSException("Bzr revert failed")
672 p = SilentPopen(['bzr', 'tags'], cwd=self.local)
673 return [tag.split(' ')[0].strip() for tag in
674 p.stdout.splitlines()]
676 def retrieve_string(app_dir, string, xmlfiles=None):
679 os.path.join(app_dir, 'res'),
680 os.path.join(app_dir, 'src/main'),
685 for res_dir in res_dirs:
686 for r,d,f in os.walk(res_dir):
687 if r.endswith('/values'):
688 xmlfiles += [os.path.join(r,x) for x in f if x.endswith('.xml')]
691 if string.startswith('@string/'):
692 string_search = re.compile(r'.*"'+string[8:]+'".*?>([^<]+?)<.*').search
693 elif string.startswith('&') and string.endswith(';'):
694 string_search = re.compile(r'.*<!ENTITY.*'+string[1:-1]+'.*?"([^"]+?)".*>').search
696 if string_search is not None:
697 for xmlfile in xmlfiles:
698 for line in file(xmlfile):
699 matches = string_search(line)
701 return retrieve_string(app_dir, matches.group(1), xmlfiles)
704 return string.replace("\\'","'")
706 # Return list of existing files that will be used to find the highest vercode
707 def manifest_paths(app_dir, flavour):
709 possible_manifests = [ os.path.join(app_dir, 'AndroidManifest.xml'),
710 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
711 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
712 os.path.join(app_dir, 'build.gradle') ]
715 possible_manifests.append(
716 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
718 return [path for path in possible_manifests if os.path.isfile(path)]
720 # Retrieve the package name. Returns the name, or None if not found.
721 def fetch_real_name(app_dir, flavour):
722 app_search = re.compile(r'.*<application.*').search
723 name_search = re.compile(r'.*android:label="([^"]+)".*').search
725 for f in manifest_paths(app_dir, flavour):
726 if not has_extension(f, 'xml'):
728 logging.debug("fetch_real_name: Checking manifest at " + f)
734 matches = name_search(line)
736 stringname = matches.group(1)
737 logging.debug("fetch_real_name: using string " + stringname)
738 result = retrieve_string(app_dir, stringname)
740 result = result.strip()
744 # Retrieve the version name
745 def version_name(original, app_dir, flavour):
746 for f in manifest_paths(app_dir, flavour):
747 if not has_extension(f, 'xml'):
749 string = retrieve_string(app_dir, original)
754 def get_library_references(root_dir):
756 proppath = os.path.join(root_dir, 'project.properties')
757 if not os.path.isfile(proppath):
759 with open(proppath) as f:
760 for line in f.readlines():
761 if not line.startswith('android.library.reference.'):
763 path = line.split('=')[1].strip()
764 relpath = os.path.join(root_dir, path)
765 if not os.path.isdir(relpath):
767 logging.info("Found subproject at %s" % path)
768 libraries.append(path)
771 def ant_subprojects(root_dir):
772 subprojects = get_library_references(root_dir)
773 for subpath in subprojects:
774 subrelpath = os.path.join(root_dir, subpath)
775 for p in get_library_references(subrelpath):
776 relp = os.path.normpath(os.path.join(subpath,p))
777 if relp not in subprojects:
778 subprojects.insert(0, relp)
781 def remove_debuggable_flags(root_dir):
782 # Remove forced debuggable flags
783 logging.info("Removing debuggable flags")
784 for root, dirs, files in os.walk(root_dir):
785 if 'AndroidManifest.xml' in files:
786 path = os.path.join(root, 'AndroidManifest.xml')
787 p = FDroidPopen(['sed','-i', 's/android:debuggable="[^"]*"//g', path])
788 if p.returncode != 0:
789 raise BuildException("Failed to remove debuggable flags of %s" % path)
791 # Extract some information from the AndroidManifest.xml at the given path.
792 # Returns (version, vercode, package), any or all of which might be None.
793 # All values returned are strings.
794 def parse_androidmanifests(paths):
797 return (None, None, None)
799 vcsearch = re.compile(r'.*:versionCode="([0-9]+?)".*').search
800 vnsearch = re.compile(r'.*:versionName="([^"]+?)".*').search
801 psearch = re.compile(r'.*package="([^"]+)".*').search
803 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
804 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
805 psearch_g = re.compile(r'.*packageName *=* *["\']([^"]+)["\'].*').search
813 gradle = has_extension(path, 'gradle')
816 # Remember package name, may be defined separately from version+vercode
817 package = max_package
819 for line in file(path):
822 matches = psearch_g(line)
824 matches = psearch(line)
826 package = matches.group(1)
829 matches = vnsearch_g(line)
831 matches = vnsearch(line)
833 version = matches.group(2 if gradle else 1)
836 matches = vcsearch_g(line)
838 matches = vcsearch(line)
840 vercode = matches.group(1)
842 # Better some package name than nothing
843 if max_package is None:
844 max_package = package
846 if max_vercode is None or (vercode is not None and vercode > max_vercode):
847 max_version = version
848 max_vercode = vercode
849 max_package = package
851 if max_version is None:
852 max_version = "Unknown"
854 return (max_version, max_vercode, max_package)
856 class BuildException(Exception):
857 def __init__(self, value, detail = None):
861 def get_wikitext(self):
862 ret = repr(self.value) + "\n"
866 txt = self.detail[-8192:] if len(self.detail) > 8192 else self.detail
872 ret = repr(self.value)
874 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
877 class VCSException(Exception):
878 def __init__(self, value):
882 return repr(self.value)
884 # Get the specified source library.
885 # Returns the path to it. Normally this is the path to be used when referencing
886 # it, which may be a subdirectory of the actual project. If you want the base
887 # directory of the project, pass 'basepath=True'.
888 def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
889 basepath=False, raw=False, prepare=True, preponly=False):
897 name, ref = spec.split('@')
899 number, name = name.split(':', 1)
901 name, subdir = name.split('/',1)
903 srclib_path = os.path.join('srclibs', name + ".txt")
905 if not os.path.exists(srclib_path):
906 raise BuildException('srclib ' + name + ' not found.')
908 srclib = metadata.parse_srclib(srclib_path)
910 sdir = os.path.join(srclib_dir, name)
913 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
914 vcs.srclib = (name, number, sdir)
916 vcs.gotorevision(ref)
923 libdir = os.path.join(sdir, subdir)
924 elif srclib["Subdir"]:
925 for subdir in srclib["Subdir"]:
926 libdir_candidate = os.path.join(sdir, subdir)
927 if os.path.exists(libdir_candidate):
928 libdir = libdir_candidate
934 if srclib["Srclibs"]:
936 for lib in srclib["Srclibs"].replace(';',',').split(','):
938 for t in srclibpaths:
943 raise BuildException('Missing recursive srclib %s for %s' % (
945 place_srclib(libdir, n, s_tuple[2])
948 remove_signing_keys(sdir)
949 remove_debuggable_flags(sdir)
953 if srclib["Prepare"]:
954 cmd = replace_config_vars(srclib["Prepare"])
956 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
957 if p.returncode != 0:
958 raise BuildException("Error running prepare command for srclib %s"
964 return (name, number, libdir)
967 # Prepare the source code for a particular build
968 # 'vcs' - the appropriate vcs object for the application
969 # 'app' - the application details from the metadata
970 # 'build' - the build details from the metadata
971 # 'build_dir' - the path to the build directory, usually
973 # 'srclib_dir' - the path to the source libraries directory, usually
975 # 'extlib_dir' - the path to the external libraries directory, usually
977 # Returns the (root, srclibpaths) where:
978 # 'root' is the root directory, which may be the same as 'build_dir' or may
979 # be a subdirectory of it.
980 # 'srclibpaths' is information on the srclibs being used
981 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False):
983 # Optionally, the actual app source can be in a subdirectory
984 if 'subdir' in build:
985 root_dir = os.path.join(build_dir, build['subdir'])
989 # Get a working copy of the right revision
990 logging.info("Getting source for revision " + build['commit'])
991 vcs.gotorevision(build['commit'])
993 # Initialise submodules if requred
994 if build['submodules']:
995 logging.info("Initialising submodules")
998 # Check that a subdir (if we're using one) exists. This has to happen
999 # after the checkout, since it might not exist elsewhere
1000 if not os.path.exists(root_dir):
1001 raise BuildException('Missing subdir ' + root_dir)
1003 # Run an init command if one is required
1005 cmd = replace_config_vars(build['init'])
1006 logging.info("Running 'init' commands in %s" % root_dir)
1008 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1009 if p.returncode != 0:
1010 raise BuildException("Error running init command for %s:%s" %
1011 (app['id'], build['version']), p.stdout)
1013 # Apply patches if any
1014 if 'patch' in build:
1015 for patch in build['patch']:
1016 patch = patch.strip()
1017 logging.info("Applying " + patch)
1018 patch_path = os.path.join('metadata', app['id'], patch)
1019 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1020 if p.returncode != 0:
1021 raise BuildException("Failed to apply patch %s" % patch_path)
1023 # Get required source libraries
1025 if 'srclibs' in build:
1026 logging.info("Collecting source libraries")
1027 for lib in build['srclibs']:
1028 srclibpaths.append(getsrclib(lib, srclib_dir, srclibpaths,
1031 for name, number, libpath in srclibpaths:
1032 place_srclib(root_dir, int(number) if number else None, libpath)
1034 basesrclib = vcs.getsrclib()
1035 # If one was used for the main source, add that too.
1037 srclibpaths.append(basesrclib)
1039 # Update the local.properties file
1040 localprops = [ os.path.join(build_dir, 'local.properties') ]
1041 if 'subdir' in build:
1042 localprops += [ os.path.join(root_dir, 'local.properties') ]
1043 for path in localprops:
1044 if not os.path.isfile(path):
1046 logging.info("Updating properties file at %s" % path)
1051 # Fix old-fashioned 'sdk-location' by copying
1052 # from sdk.dir, if necessary
1053 if build['oldsdkloc']:
1054 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1056 props += "sdk-location=%s\n" % sdkloc
1058 props += "sdk.dir=%s\n" % config['sdk_path']
1059 props += "sdk-location=%s\n" % config['sdk_path']
1060 if 'ndk_path' in config:
1062 props += "ndk.dir=%s\n" % config['ndk_path']
1063 props += "ndk-location=%s\n" % config['ndk_path']
1064 # Add java.encoding if necessary
1065 if 'encoding' in build:
1066 props += "java.encoding=%s\n" % build['encoding']
1072 if build['type'] == 'gradle':
1073 flavour = build['gradle'].split('@')[0]
1074 if flavour in ['main', 'yes', '']:
1077 if 'target' in build:
1078 n = build["target"].split('-')[1]
1079 FDroidPopen(['sed', '-i',
1080 's@compileSdkVersion *[0-9]*@compileSdkVersion '+n+'@g',
1081 'build.gradle'], cwd=root_dir)
1082 if '@' in build['gradle']:
1083 gradle_dir = os.path.join(root_dir, build['gradle'].split('@',1)[1])
1084 gradle_dir = os.path.normpath(gradle_dir)
1085 FDroidPopen(['sed', '-i',
1086 's@compileSdkVersion *[0-9]*@compileSdkVersion '+n+'@g',
1087 'build.gradle'], cwd=gradle_dir)
1089 # Remove forced debuggable flags
1090 remove_debuggable_flags(root_dir)
1092 # Insert version code and number into the manifest if necessary
1093 if build['forceversion']:
1094 logging.info("Changing the version name")
1095 for path in manifest_paths(root_dir, flavour):
1096 if not os.path.isfile(path):
1098 if has_extension(path, 'xml'):
1099 p = SilentPopen(['sed', '-i',
1100 's/android:versionName="[^"]*"/android:versionName="' + build['version'] + '"/g',
1102 if p.returncode != 0:
1103 raise BuildException("Failed to amend manifest")
1104 elif has_extension(path, 'gradle'):
1105 p = SilentPopen(['sed', '-i',
1106 's/versionName *=* *"[^"]*"/versionName = "' + build['version'] + '"/g',
1108 if p.returncode != 0:
1109 raise BuildException("Failed to amend build.gradle")
1110 if build['forcevercode']:
1111 logging.info("Changing the version code")
1112 for path in manifest_paths(root_dir, flavour):
1113 if not os.path.isfile(path):
1115 if has_extension(path, 'xml'):
1116 p = SilentPopen(['sed', '-i',
1117 's/android:versionCode="[^"]*"/android:versionCode="' + build['vercode'] + '"/g',
1119 if p.returncode != 0:
1120 raise BuildException("Failed to amend manifest")
1121 elif has_extension(path, 'gradle'):
1122 p = SilentPopen(['sed', '-i',
1123 's/versionCode *=* *[0-9]*/versionCode = ' + build['vercode'] + '/g',
1125 if p.returncode != 0:
1126 raise BuildException("Failed to amend build.gradle")
1128 # Delete unwanted files
1130 for part in getpaths(build_dir, build, 'rm'):
1131 dest = os.path.join(build_dir, part)
1132 logging.info("Removing {0}".format(part))
1133 if os.path.lexists(dest):
1134 if os.path.islink(dest):
1135 SilentPopen(['unlink ' + dest], shell=True)
1137 SilentPopen(['rm -rf ' + dest], shell=True)
1139 logging.info("...but it didn't exist")
1141 remove_signing_keys(build_dir)
1143 # Add required external libraries
1144 if 'extlibs' in build:
1145 logging.info("Collecting prebuilt libraries")
1146 libsdir = os.path.join(root_dir, 'libs')
1147 if not os.path.exists(libsdir):
1149 for lib in build['extlibs']:
1151 logging.info("...installing extlib {0}".format(lib))
1152 libf = os.path.basename(lib)
1153 libsrc = os.path.join(extlib_dir, lib)
1154 if not os.path.exists(libsrc):
1155 raise BuildException("Missing extlib file {0}".format(libsrc))
1156 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1158 # Run a pre-build command if one is required
1159 if 'prebuild' in build:
1160 cmd = replace_config_vars(build['prebuild'])
1162 # Substitute source library paths into prebuild commands
1163 for name, number, libpath in srclibpaths:
1164 libpath = os.path.relpath(libpath, root_dir)
1165 cmd = cmd.replace('$$' + name + '$$', libpath)
1167 logging.info("Running 'prebuild' commands in %s" % root_dir)
1169 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1170 if p.returncode != 0:
1171 raise BuildException("Error running prebuild command for %s:%s" %
1172 (app['id'], build['version']), p.stdout)
1174 updatemode = build.get('update', ['auto'])
1175 # Generate (or update) the ant build file, build.xml...
1176 if updatemode != ['no'] and build['type'] == 'ant':
1177 parms = [os.path.join(config['sdk_path'], 'tools', 'android'), 'update']
1178 lparms = parms + ['lib-project']
1179 parms = parms + ['project']
1181 if 'target' in build and build['target']:
1182 parms += ['-t', build['target']]
1183 lparms += ['-t', build['target']]
1184 if updatemode == ['auto']:
1185 update_dirs = ant_subprojects(root_dir) + ['.']
1187 update_dirs = updatemode
1189 for d in update_dirs:
1190 subdir = os.path.join(root_dir, d)
1192 print("Updating main project")
1193 cmd = parms + ['-p', d]
1195 print("Updating subproject %s" % d)
1196 cmd = lparms + ['-p', d]
1197 p = FDroidPopen(cmd, cwd=root_dir)
1198 # Check to see whether an error was returned without a proper exit
1199 # code (this is the case for the 'no target set or target invalid'
1201 if p.returncode != 0 or p.stdout.startswith("Error: "):
1202 raise BuildException("Failed to update project at %s" % d, p.stdout)
1203 # Clean update dirs via ant
1205 logging.info("Cleaning subproject %s" % d)
1206 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1208 return (root_dir, srclibpaths)
1210 # Split and extend via globbing the paths from a field
1211 def getpaths(build_dir, build, field):
1213 if field not in build:
1215 for p in build[field]:
1217 full_path = os.path.join(build_dir, p)
1218 full_path = os.path.normpath(full_path)
1219 paths += [r[len(build_dir)+1:] for r in glob.glob(full_path)]
1222 # Scan the source code in the given directory (and all subdirectories)
1223 # and return the number of fatal problems encountered
1224 def scan_source(build_dir, root_dir, thisbuild):
1228 # Common known non-free blobs (always lower case):
1230 re.compile(r'flurryagent', re.IGNORECASE),
1231 re.compile(r'paypal.*mpl', re.IGNORECASE),
1232 re.compile(r'libgoogleanalytics', re.IGNORECASE),
1233 re.compile(r'admob.*sdk.*android', re.IGNORECASE),
1234 re.compile(r'googleadview', re.IGNORECASE),
1235 re.compile(r'googleadmobadssdk', re.IGNORECASE),
1236 re.compile(r'google.*play.*services', re.IGNORECASE),
1237 re.compile(r'crittercism', re.IGNORECASE),
1238 re.compile(r'heyzap', re.IGNORECASE),
1239 re.compile(r'jpct.*ae', re.IGNORECASE),
1240 re.compile(r'youtubeandroidplayerapi', re.IGNORECASE),
1241 re.compile(r'bugsense', re.IGNORECASE),
1242 re.compile(r'crashlytics', re.IGNORECASE),
1243 re.compile(r'ouya.*sdk', re.IGNORECASE),
1246 scanignore = getpaths(build_dir, thisbuild, 'scanignore')
1247 scandelete = getpaths(build_dir, thisbuild, 'scandelete')
1250 ms = magic.open(magic.MIME_TYPE)
1252 except AttributeError:
1256 for i in scanignore:
1257 if fd.startswith(i):
1262 for i in scandelete:
1263 if fd.startswith(i):
1267 def removeproblem(what, fd, fp):
1268 logging.info('Removing %s at %s' % (what, fd))
1271 def warnproblem(what, fd):
1272 logging.warn('Found %s at %s' % (what, fd))
1274 def handleproblem(what, fd, fp):
1276 removeproblem(what, fd, fp)
1278 logging.error('Found %s at %s' % (what, fd))
1282 def insidedir(path, dirname):
1283 return path.endswith('/%s' % dirname) or '/%s/' % dirname in path
1285 # Iterate through all files in the source code
1286 for r,d,f in os.walk(build_dir):
1288 if any(insidedir(r, d) for d in ('.hg', '.git', '.svn', '.bzr')):
1293 # Path (relative) to the file
1294 fp = os.path.join(r, curfile)
1295 fd = fp[len(build_dir)+1:]
1297 # Check if this file has been explicitly excluded from scanning
1301 mime = magic.from_file(fp, mime=True) if ms is None else ms.file(fp)
1303 if mime == 'application/x-sharedlib':
1304 count += handleproblem('shared library', fd, fp)
1306 elif mime == 'application/x-archive':
1307 count += handleproblem('static library', fd, fp)
1309 elif mime == 'application/x-executable':
1310 count += handleproblem('binary executable', fd, fp)
1312 elif mime == 'application/x-java-applet':
1313 count += handleproblem('Java compiled class', fd, fp)
1318 'application/java-archive',
1319 'application/octet-stream',
1323 if has_extension(fp, 'apk'):
1324 removeproblem('APK file', fd, fp)
1326 elif has_extension(fp, 'jar'):
1328 if any(suspect.match(curfile) for suspect in usual_suspects):
1329 count += handleproblem('usual supect', fd, fp)
1331 warnproblem('JAR file', fd)
1333 elif has_extension(fp, 'zip'):
1334 warnproblem('ZIP file', fd)
1337 warnproblem('unknown compressed or binary file', fd)
1339 elif has_extension(fp, 'java'):
1340 for line in file(fp):
1341 if 'DexClassLoader' in line:
1342 count += handleproblem('DexClassLoader', fd, fp)
1347 # Presence of a jni directory without buildjni=yes might
1348 # indicate a problem (if it's not a problem, explicitly use
1349 # buildjni=no to bypass this check)
1350 if (os.path.exists(os.path.join(root_dir, 'jni')) and
1351 thisbuild.get('buildjni') is None):
1352 logging.warn('Found jni directory, but buildjni is not enabled')
1361 self.path = os.path.join('stats', 'known_apks.txt')
1363 if os.path.exists(self.path):
1364 for line in file( self.path):
1365 t = line.rstrip().split(' ')
1367 self.apks[t[0]] = (t[1], None)
1369 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1370 self.changed = False
1372 def writeifchanged(self):
1374 if not os.path.exists('stats'):
1376 f = open(self.path, 'w')
1378 for apk, app in self.apks.iteritems():
1380 line = apk + ' ' + appid
1382 line += ' ' + time.strftime('%Y-%m-%d', added)
1384 for line in sorted(lst):
1385 f.write(line + '\n')
1388 # Record an apk (if it's new, otherwise does nothing)
1389 # Returns the date it was added.
1390 def recordapk(self, apk, app):
1391 if not apk in self.apks:
1392 self.apks[apk] = (app, time.gmtime(time.time()))
1394 _, added = self.apks[apk]
1397 # Look up information - given the 'apkname', returns (app id, date added/None).
1398 # Or returns None for an unknown apk.
1399 def getapp(self, apkname):
1400 if apkname in self.apks:
1401 return self.apks[apkname]
1404 # Get the most recent 'num' apps added to the repo, as a list of package ids
1405 # with the most recent first.
1406 def getlatest(self, num):
1408 for apk, app in self.apks.iteritems():
1412 if apps[appid] > added:
1416 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1417 lst = [app for app,_ in sortedapps]
1421 def isApkDebuggable(apkfile, config):
1422 """Returns True if the given apk file is debuggable
1424 :param apkfile: full path to the apk to check"""
1426 p = SilentPopen([os.path.join(config['sdk_path'],
1427 'build-tools', config['build_tools'], 'aapt'),
1428 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'])
1429 if p.returncode != 0:
1430 logging.critical("Failed to get apk manifest information")
1432 for line in p.stdout.splitlines():
1433 if 'android:debuggable' in line and not line.endswith('0x0'):
1438 class AsynchronousFileReader(threading.Thread):
1440 Helper class to implement asynchronous reading of a file
1441 in a separate thread. Pushes read lines on a queue to
1442 be consumed in another thread.
1445 def __init__(self, fd, queue):
1446 assert isinstance(queue, Queue.Queue)
1447 assert callable(fd.readline)
1448 threading.Thread.__init__(self)
1453 '''The body of the tread: read lines and put them on the queue.'''
1454 for line in iter(self._fd.readline, ''):
1455 self._queue.put(line)
1458 '''Check whether there is no more content to expect.'''
1459 return not self.is_alive() and self._queue.empty()
1465 def SilentPopen(commands, cwd=None, shell=False):
1466 return FDroidPopen(commands, cwd=cwd, shell=shell, output=False)
1468 def FDroidPopen(commands, cwd=None, shell=False, output=True):
1470 Run a command and capture the possibly huge output.
1472 :param commands: command and argument list like in subprocess.Popen
1473 :param cwd: optionally specifies a working directory
1474 :returns: A PopenResult.
1479 cwd = os.path.normpath(cwd)
1480 logging.info("Directory: %s" % cwd)
1481 logging.info("> %s" % ' '.join(commands))
1483 result = PopenResult()
1484 p = subprocess.Popen(commands, cwd=cwd, shell=shell,
1485 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1487 stdout_queue = Queue.Queue()
1488 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1489 stdout_reader.start()
1491 # Check the queue for output (until there is no more to get)
1492 while not stdout_reader.eof():
1493 while not stdout_queue.empty():
1494 line = stdout_queue.get()
1495 if output and options.verbose:
1496 # Output directly to console
1497 sys.stdout.write(line)
1499 result.stdout += line
1504 result.returncode = p.returncode
1507 def remove_signing_keys(build_dir):
1508 comment = re.compile(r'[ ]*//')
1509 signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1511 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1512 re.compile(r'.*android\.signingConfigs\..*'),
1513 re.compile(r'.*variant\.outputFile = .*'),
1514 re.compile(r'.*\.readLine\(.*'),
1516 for root, dirs, files in os.walk(build_dir):
1517 if 'build.gradle' in files:
1518 path = os.path.join(root, 'build.gradle')
1520 with open(path, "r") as o:
1521 lines = o.readlines()
1524 with open(path, "w") as o:
1526 if comment.match(line):
1530 opened += line.count('{')
1531 opened -= line.count('}')
1534 if signing_configs.match(line):
1538 if any(s.match(line) for s in line_matches):
1544 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1547 'project.properties',
1549 'default.properties',
1552 if propfile in files:
1553 path = os.path.join(root, propfile)
1555 with open(path, "r") as o:
1556 lines = o.readlines()
1558 with open(path, "w") as o:
1560 if line.startswith('key.store'):
1562 if line.startswith('key.alias'):
1566 logging.info("Cleaned %s of keysigning configs at %s" % (propfile,path))
1568 def replace_config_vars(cmd):
1569 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1570 cmd = cmd.replace('$$NDK$$', config['ndk_path'])
1571 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1574 def place_srclib(root_dir, number, libpath):
1577 relpath = os.path.relpath(libpath, root_dir)
1578 proppath = os.path.join(root_dir, 'project.properties')
1581 if os.path.isfile(proppath):
1582 with open(proppath, "r") as o:
1583 lines = o.readlines()
1585 with open(proppath, "w") as o:
1588 if line.startswith('android.library.reference.%d=' % number):
1589 o.write('android.library.reference.%d=%s\n' % (number,relpath))
1594 o.write('android.library.reference.%d=%s\n' % (number,relpath))