1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
40 def get_default_config():
42 'sdk_path': os.getenv("ANDROID_HOME"),
43 'ndk_path': "$ANDROID_NDK",
44 'build_tools': "19.0.3",
49 'update_stats': False,
50 'stats_to_carbon': False,
52 'build_server_always': False,
53 'keystore': '$HOME/.local/share/fdroidserver/keystore.jks',
54 'smartcardoptions': [],
63 def read_config(opts, config_file='config.py'):
64 """Read the repository config
66 The config is read from config_file, which is in the current directory when
67 any of the repo management commands are used.
69 global config, options
71 if config is not None:
73 if not os.path.isfile(config_file):
74 logging.critical("Missing config file - is this a repo directory?")
81 logging.debug("Reading %s" % config_file)
82 execfile(config_file, config)
84 # smartcardoptions must be a list since its command line args for Popen
85 if 'smartcardoptions' in config:
86 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
87 elif 'keystore' in config and config['keystore'] == 'NONE':
88 # keystore='NONE' means use smartcard, these are required defaults
89 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
90 'SunPKCS11-OpenSC', '-providerClass',
91 'sun.security.pkcs11.SunPKCS11',
92 '-providerArg', 'opensc-fdroid.cfg']
94 defconfig = get_default_config()
95 for k, v in defconfig.items():
99 # Expand environment variables
100 for k, v in config.items():
103 v = os.path.expanduser(v)
104 config[k] = os.path.expandvars(v)
106 if not test_sdk_exists(config):
109 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
110 st = os.stat(config_file)
111 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
112 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
114 for k in ["keystorepass", "keypass"]:
116 write_password_file(k)
118 # since this is used with rsync, where trailing slashes have meaning,
119 # ensure there is always a trailing slash
120 if 'serverwebroot' in config:
121 if config['serverwebroot'][-1] != '/':
122 config['serverwebroot'] += '/'
123 config['serverwebroot'] = config['serverwebroot'].replace('//', '/')
128 def test_sdk_exists(c):
129 if c['sdk_path'] is None:
130 # c['sdk_path'] is set to the value of ANDROID_HOME by default
131 logging.critical('No Android SDK found! ANDROID_HOME is not set and sdk_path is not in config.py!')
132 logging.info('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
133 logging.info('\texport ANDROID_HOME=/opt/android-sdk')
135 if not os.path.exists(c['sdk_path']):
136 logging.critical('Android SDK path "' + c['sdk_path'] + '" does not exist!')
138 if not os.path.isdir(c['sdk_path']):
139 logging.critical('Android SDK path "' + c['sdk_path'] + '" is not a directory!')
141 if not os.path.isdir(os.path.join(c['sdk_path'], 'build-tools')):
142 logging.critical('Android SDK path "' + c['sdk_path'] + '" does not contain "build-tools/"!')
147 def write_password_file(pwtype, password=None):
149 writes out passwords to a protected file instead of passing passwords as
150 command line argments
152 filename = '.fdroid.' + pwtype + '.txt'
153 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
155 os.write(fd, config[pwtype])
157 os.write(fd, password)
159 config[pwtype + 'file'] = filename
162 # Given the arguments in the form of multiple appid:[vc] strings, this returns
163 # a dictionary with the set of vercodes specified for each package.
164 def read_pkg_args(args, allow_vercodes=False):
171 if allow_vercodes and ':' in p:
172 package, vercode = p.split(':')
174 package, vercode = p, None
175 if package not in vercodes:
176 vercodes[package] = [vercode] if vercode else []
178 elif vercode and vercode not in vercodes[package]:
179 vercodes[package] += [vercode] if vercode else []
184 # On top of what read_pkg_args does, this returns the whole app metadata, but
185 # limiting the builds list to the builds matching the vercodes specified.
186 def read_app_args(args, allapps, allow_vercodes=False):
188 vercodes = read_pkg_args(args, allow_vercodes)
193 apps = [app for app in allapps if app['id'] in vercodes]
195 if len(apps) != len(vercodes):
196 allids = [app["id"] for app in allapps]
199 logging.critical("No such package: %s" % p)
200 raise Exception("Found invalid app ids in arguments")
202 raise Exception("No packages specified")
206 vc = vercodes[app['id']]
209 app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
210 if len(app['builds']) != len(vercodes[app['id']]):
212 allvcs = [b['vercode'] for b in app['builds']]
213 for v in vercodes[app['id']]:
215 logging.critical("No such vercode %s for app %s" % (v, app['id']))
218 raise Exception("Found invalid vercodes for some apps")
223 def has_extension(filename, extension):
224 name, ext = os.path.splitext(filename)
225 ext = ext.lower()[1:]
226 return ext == extension
231 def apknameinfo(filename):
233 filename = os.path.basename(filename)
234 if apk_regex is None:
235 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
236 m = apk_regex.match(filename)
238 result = (m.group(1), m.group(2))
239 except AttributeError:
240 raise Exception("Invalid apk name: %s" % filename)
244 def getapkname(app, build):
245 return "%s_%s.apk" % (app['id'], build['vercode'])
248 def getsrcname(app, build):
249 return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
256 return app['Auto Name']
261 return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
264 def getvcs(vcstype, remote, local):
266 return vcs_git(remote, local)
268 return vcs_svn(remote, local)
269 if vcstype == 'git-svn':
270 return vcs_gitsvn(remote, local)
272 return vcs_hg(remote, local)
274 return vcs_bzr(remote, local)
275 if vcstype == 'srclib':
276 if local != 'build/srclib/' + remote:
277 raise VCSException("Error: srclib paths are hard-coded!")
278 return getsrclib(remote, 'build/srclib', raw=True)
279 raise VCSException("Invalid vcs type " + vcstype)
282 def getsrclibvcs(name):
283 if not name in metadata.srclibs:
284 raise VCSException("Missing srclib " + name)
285 return metadata.srclibs[name]['Repo Type']
289 def __init__(self, remote, local):
291 # svn, git-svn and bzr may require auth
293 if self.repotype() in ('svn', 'git-svn', 'bzr'):
295 self.username, remote = remote.split('@')
296 if ':' not in self.username:
297 raise VCSException("Password required with username")
298 self.username, self.password = self.username.split(':')
302 self.refreshed = False
308 # Take the local repository to a clean version of the given revision, which
309 # is specificed in the VCS's native format. Beforehand, the repository can
310 # be dirty, or even non-existent. If the repository does already exist
311 # locally, it will be updated from the origin, but only once in the
312 # lifetime of the vcs object.
313 # None is acceptable for 'rev' if you know you are cloning a clean copy of
314 # the repo - otherwise it must specify a valid revision.
315 def gotorevision(self, rev):
317 # The .fdroidvcs-id file for a repo tells us what VCS type
318 # and remote that directory was created from, allowing us to drop it
319 # automatically if either of those things changes.
320 fdpath = os.path.join(self.local, '..',
321 '.fdroidvcs-' + os.path.basename(self.local))
322 cdata = self.repotype() + ' ' + self.remote
325 if os.path.exists(self.local):
326 if os.path.exists(fdpath):
327 with open(fdpath, 'r') as f:
328 fsdata = f.read().strip()
333 logging.info("Repository details changed - deleting")
336 logging.info("Repository details missing - deleting")
338 shutil.rmtree(self.local)
340 self.gotorevisionx(rev)
342 # If necessary, write the .fdroidvcs file.
344 with open(fdpath, 'w') as f:
347 # Derived classes need to implement this. It's called once basic checking
348 # has been performend.
349 def gotorevisionx(self, rev):
350 raise VCSException("This VCS type doesn't define gotorevisionx")
352 # Initialise and update submodules
353 def initsubmodules(self):
354 raise VCSException('Submodules not supported for this vcs type')
356 # Get a list of all known tags
358 raise VCSException('gettags not supported for this vcs type')
360 # Get a list of latest number tags
361 def latesttags(self, number):
362 raise VCSException('latesttags not supported for this vcs type')
364 # Get current commit reference (hash, revision, etc)
366 raise VCSException('getref not supported for this vcs type')
368 # Returns the srclib (name, path) used in setting up the current
379 # If the local directory exists, but is somehow not a git repository, git
380 # will traverse up the directory tree until it finds one that is (i.e.
381 # fdroidserver) and then we'll proceed to destroy it! This is called as
384 p = SilentPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
385 result = p.stdout.rstrip()
386 if not result.endswith(self.local):
387 raise VCSException('Repository mismatch')
389 def gotorevisionx(self, rev):
390 if not os.path.exists(self.local):
392 p = FDroidPopen(['git', 'clone', self.remote, self.local])
393 if p.returncode != 0:
394 raise VCSException("Git clone failed")
398 # Discard any working tree changes
399 p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
400 if p.returncode != 0:
401 raise VCSException("Git reset failed")
402 # Remove untracked files now, in case they're tracked in the target
403 # revision (it happens!)
404 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
405 if p.returncode != 0:
406 raise VCSException("Git clean failed")
407 if not self.refreshed:
408 # Get latest commits and tags from remote
409 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
410 if p.returncode != 0:
411 raise VCSException("Git fetch failed")
412 p = SilentPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local)
413 if p.returncode != 0:
414 raise VCSException("Git fetch failed")
415 self.refreshed = True
416 # Check out the appropriate revision
417 rev = str(rev if rev else 'origin/master')
418 p = SilentPopen(['git', 'checkout', '-f', rev], cwd=self.local)
419 if p.returncode != 0:
420 raise VCSException("Git checkout failed")
421 # Get rid of any uncontrolled files left behind
422 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
423 if p.returncode != 0:
424 raise VCSException("Git clean failed")
426 def initsubmodules(self):
428 submfile = os.path.join(self.local, '.gitmodules')
429 if not os.path.isfile(submfile):
430 raise VCSException("No git submodules available")
432 # fix submodules not accessible without an account and public key auth
433 with open(submfile, 'r') as f:
434 lines = f.readlines()
435 with open(submfile, 'w') as f:
437 if 'git@github.com' in line:
438 line = line.replace('git@github.com:', 'https://github.com/')
442 ['git', 'reset', '--hard'],
443 ['git', 'clean', '-dffx'],
445 p = SilentPopen(['git', 'submodule', 'foreach', '--recursive'] + cmd, cwd=self.local)
446 if p.returncode != 0:
447 raise VCSException("Git submodule reset failed")
448 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local)
449 if p.returncode != 0:
450 raise VCSException("Git submodule sync failed")
451 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
452 if p.returncode != 0:
453 raise VCSException("Git submodule update failed")
457 p = SilentPopen(['git', 'tag'], cwd=self.local)
458 return p.stdout.splitlines()
460 def latesttags(self, alltags, number):
462 p = SilentPopen(['echo "' + '\n'.join(alltags) + '" | \
463 xargs -I@ git log --format=format:"%at @%n" -1 @ | \
464 sort -n | awk \'{print $2}\''],
465 cwd=self.local, shell=True)
466 return p.stdout.splitlines()[-number:]
469 class vcs_gitsvn(vcs):
474 # Damn git-svn tries to use a graphical password prompt, so we have to
475 # trick it into taking the password from stdin
477 if self.username is None:
479 return ('echo "%s" | DISPLAY="" ' % self.password, ' --username "%s"' % self.username)
481 # If the local directory exists, but is somehow not a git repository, git
482 # will traverse up the directory tree until it finds one that is (i.e.
483 # fdroidserver) and then we'll proceed to destory it! This is called as
486 p = SilentPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
487 result = p.stdout.rstrip()
488 if not result.endswith(self.local):
489 raise VCSException('Repository mismatch')
491 def gotorevisionx(self, rev):
492 if not os.path.exists(self.local):
494 gitsvn_cmd = '%sgit svn clone%s' % self.userargs()
495 if ';' in self.remote:
496 remote_split = self.remote.split(';')
497 for i in remote_split[1:]:
498 if i.startswith('trunk='):
499 gitsvn_cmd += ' -T %s' % i[6:]
500 elif i.startswith('tags='):
501 gitsvn_cmd += ' -t %s' % i[5:]
502 elif i.startswith('branches='):
503 gitsvn_cmd += ' -b %s' % i[9:]
504 p = SilentPopen([gitsvn_cmd + " %s %s" % (remote_split[0], self.local)], shell=True)
505 if p.returncode != 0:
506 raise VCSException("Git clone failed")
508 p = SilentPopen([gitsvn_cmd + " %s %s" % (self.remote, self.local)], shell=True)
509 if p.returncode != 0:
510 raise VCSException("Git clone failed")
514 # Discard any working tree changes
515 p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
516 if p.returncode != 0:
517 raise VCSException("Git reset failed")
518 # Remove untracked files now, in case they're tracked in the target
519 # revision (it happens!)
520 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
521 if p.returncode != 0:
522 raise VCSException("Git clean failed")
523 if not self.refreshed:
524 # Get new commits, branches and tags from repo
525 p = SilentPopen(['%sgit svn fetch %s' % self.userargs()], cwd=self.local, shell=True)
526 if p.returncode != 0:
527 raise VCSException("Git svn fetch failed")
528 p = SilentPopen(['%sgit svn rebase %s' % self.userargs()], cwd=self.local, shell=True)
529 if p.returncode != 0:
530 raise VCSException("Git svn rebase failed")
531 self.refreshed = True
533 rev = str(rev if rev else 'master')
535 nospaces_rev = rev.replace(' ', '%20')
536 # Try finding a svn tag
537 p = SilentPopen(['git', 'checkout', 'tags/' + nospaces_rev], cwd=self.local)
538 if p.returncode != 0:
539 # No tag found, normal svn rev translation
540 # Translate svn rev into git format
541 rev_split = rev.split('/')
542 if len(rev_split) > 1:
543 treeish = rev_split[0]
544 svn_rev = rev_split[1]
547 # if no branch is specified, then assume trunk (ie. 'master'
552 p = SilentPopen(['git', 'svn', 'find-rev', 'r' + svn_rev, treeish], cwd=self.local)
553 git_rev = p.stdout.rstrip()
555 if p.returncode != 0 or not git_rev:
556 # Try a plain git checkout as a last resort
557 p = SilentPopen(['git', 'checkout', rev], cwd=self.local)
558 if p.returncode != 0:
559 raise VCSException("No git treeish found and direct git checkout failed")
561 # Check out the git rev equivalent to the svn rev
562 p = SilentPopen(['git', 'checkout', git_rev], cwd=self.local)
563 if p.returncode != 0:
564 raise VCSException("Git svn checkout failed")
566 # Get rid of any uncontrolled files left behind
567 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
568 if p.returncode != 0:
569 raise VCSException("Git clean failed")
573 return os.listdir(os.path.join(self.local, '.git/svn/refs/remotes/tags'))
577 p = SilentPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local)
578 if p.returncode != 0:
580 return p.stdout.strip()
589 if self.username is None:
590 return ['--non-interactive']
591 return ['--username', self.username,
592 '--password', self.password,
595 def gotorevisionx(self, rev):
596 if not os.path.exists(self.local):
597 p = SilentPopen(['svn', 'checkout', self.remote, self.local] + self.userargs())
598 if p.returncode != 0:
599 raise VCSException("Svn checkout failed")
603 r"svn status | awk '/\?/ {print $2}' | xargs rm -rf"):
604 p = SilentPopen([svncommand], cwd=self.local, shell=True)
605 if p.returncode != 0:
606 raise VCSException("Svn reset ({0}) failed in {1}".format(svncommand, self.local))
607 if not self.refreshed:
608 p = SilentPopen(['svn', 'update'] + self.userargs(), cwd=self.local)
609 if p.returncode != 0:
610 raise VCSException("Svn update failed")
611 self.refreshed = True
613 revargs = list(['-r', rev] if rev else [])
614 p = SilentPopen(['svn', 'update', '--force'] + revargs + self.userargs(), cwd=self.local)
615 if p.returncode != 0:
616 raise VCSException("Svn update failed")
619 p = SilentPopen(['svn', 'info'], cwd=self.local)
620 for line in p.stdout.splitlines():
621 if line and line.startswith('Last Changed Rev: '):
631 def gotorevisionx(self, rev):
632 if not os.path.exists(self.local):
633 p = SilentPopen(['hg', 'clone', self.remote, self.local])
634 if p.returncode != 0:
635 raise VCSException("Hg clone failed")
637 p = SilentPopen(['hg status -uS | xargs rm -rf'], cwd=self.local, shell=True)
638 if p.returncode != 0:
639 raise VCSException("Hg clean failed")
640 if not self.refreshed:
641 p = SilentPopen(['hg', 'pull'], cwd=self.local)
642 if p.returncode != 0:
643 raise VCSException("Hg pull failed")
644 self.refreshed = True
646 rev = str(rev if rev else 'default')
649 p = SilentPopen(['hg', 'update', '-C', rev], cwd=self.local)
650 if p.returncode != 0:
651 raise VCSException("Hg checkout failed")
652 p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
653 # Also delete untracked files, we have to enable purge extension for that:
654 if "'purge' is provided by the following extension" in p.stdout:
655 with open(self.local + "/.hg/hgrc", "a") as myfile:
656 myfile.write("\n[extensions]\nhgext.purge=\n")
657 p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
658 if p.returncode != 0:
659 raise VCSException("HG purge failed")
660 elif p.returncode != 0:
661 raise VCSException("HG purge failed")
664 p = SilentPopen(['hg', 'tags', '-q'], cwd=self.local)
665 return p.stdout.splitlines()[1:]
673 def gotorevisionx(self, rev):
674 if not os.path.exists(self.local):
675 p = SilentPopen(['bzr', 'branch', self.remote, self.local])
676 if p.returncode != 0:
677 raise VCSException("Bzr branch failed")
679 p = SilentPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local)
680 if p.returncode != 0:
681 raise VCSException("Bzr revert failed")
682 if not self.refreshed:
683 p = SilentPopen(['bzr', 'pull'], cwd=self.local)
684 if p.returncode != 0:
685 raise VCSException("Bzr update failed")
686 self.refreshed = True
688 revargs = list(['-r', rev] if rev else [])
689 p = SilentPopen(['bzr', 'revert'] + revargs, cwd=self.local)
690 if p.returncode != 0:
691 raise VCSException("Bzr revert failed")
694 p = SilentPopen(['bzr', 'tags'], cwd=self.local)
695 return [tag.split(' ')[0].strip() for tag in
696 p.stdout.splitlines()]
699 def retrieve_string(app_dir, string, xmlfiles=None):
702 os.path.join(app_dir, 'res'),
703 os.path.join(app_dir, 'src/main'),
708 for res_dir in res_dirs:
709 for r, d, f in os.walk(res_dir):
710 if r.endswith('/values'):
711 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
714 if string.startswith('@string/'):
715 string_search = re.compile(r'.*"' + string[8:] + '".*?>([^<]+?)<.*').search
716 elif string.startswith('&') and string.endswith(';'):
717 string_search = re.compile(r'.*<!ENTITY.*' + string[1:-1] + '.*?"([^"]+?)".*>').search
719 if string_search is not None:
720 for xmlfile in xmlfiles:
721 for line in file(xmlfile):
722 matches = string_search(line)
724 return retrieve_string(app_dir, matches.group(1), xmlfiles)
727 return string.replace("\\'", "'")
730 # Return list of existing files that will be used to find the highest vercode
731 def manifest_paths(app_dir, flavour):
733 possible_manifests = \
734 [os.path.join(app_dir, 'AndroidManifest.xml'),
735 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
736 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
737 os.path.join(app_dir, 'build.gradle')]
740 possible_manifests.append(
741 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
743 return [path for path in possible_manifests if os.path.isfile(path)]
746 # Retrieve the package name. Returns the name, or None if not found.
747 def fetch_real_name(app_dir, flavour):
748 app_search = re.compile(r'.*<application.*').search
749 name_search = re.compile(r'.*android:label="([^"]+)".*').search
751 for f in manifest_paths(app_dir, flavour):
752 if not has_extension(f, 'xml'):
754 logging.debug("fetch_real_name: Checking manifest at " + f)
760 matches = name_search(line)
762 stringname = matches.group(1)
763 logging.debug("fetch_real_name: using string " + stringname)
764 result = retrieve_string(app_dir, stringname)
766 result = result.strip()
771 # Retrieve the version name
772 def version_name(original, app_dir, flavour):
773 for f in manifest_paths(app_dir, flavour):
774 if not has_extension(f, 'xml'):
776 string = retrieve_string(app_dir, original)
782 def get_library_references(root_dir):
784 proppath = os.path.join(root_dir, 'project.properties')
785 if not os.path.isfile(proppath):
787 with open(proppath) as f:
788 for line in f.readlines():
789 if not line.startswith('android.library.reference.'):
791 path = line.split('=')[1].strip()
792 relpath = os.path.join(root_dir, path)
793 if not os.path.isdir(relpath):
795 logging.info("Found subproject at %s" % path)
796 libraries.append(path)
800 def ant_subprojects(root_dir):
801 subprojects = get_library_references(root_dir)
802 for subpath in subprojects:
803 subrelpath = os.path.join(root_dir, subpath)
804 for p in get_library_references(subrelpath):
805 relp = os.path.normpath(os.path.join(subpath, p))
806 if relp not in subprojects:
807 subprojects.insert(0, relp)
811 def remove_debuggable_flags(root_dir):
812 # Remove forced debuggable flags
813 logging.info("Removing debuggable flags")
814 for root, dirs, files in os.walk(root_dir):
815 if 'AndroidManifest.xml' in files:
816 path = os.path.join(root, 'AndroidManifest.xml')
817 p = FDroidPopen(['sed', '-i', 's/android:debuggable="[^"]*"//g', path])
818 if p.returncode != 0:
819 raise BuildException("Failed to remove debuggable flags of %s" % path)
822 # Extract some information from the AndroidManifest.xml at the given path.
823 # Returns (version, vercode, package), any or all of which might be None.
824 # All values returned are strings.
825 def parse_androidmanifests(paths, ignoreversions=None):
828 return (None, None, None)
830 vcsearch = re.compile(r'.*:versionCode="([0-9]+?)".*').search
831 vnsearch = re.compile(r'.*:versionName="([^"]+?)".*').search
832 psearch = re.compile(r'.*package="([^"]+)".*').search
834 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
835 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
836 psearch_g = re.compile(r'.*packageName *=* *["\']([^"]+)["\'].*').search
838 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
846 gradle = has_extension(path, 'gradle')
849 # Remember package name, may be defined separately from version+vercode
850 package = max_package
852 for line in file(path):
855 matches = psearch_g(line)
857 matches = psearch(line)
859 package = matches.group(1)
862 matches = vnsearch_g(line)
864 matches = vnsearch(line)
866 version = matches.group(2 if gradle else 1)
869 matches = vcsearch_g(line)
871 matches = vcsearch(line)
873 vercode = matches.group(1)
875 # Better some package name than nothing
876 if max_package is None:
877 max_package = package
879 if max_vercode is None or (vercode is not None and vercode > max_vercode):
880 if not ignoresearch or not ignoresearch(version):
881 max_version = version
882 max_vercode = vercode
883 max_package = package
885 max_version = "Ignore"
887 if max_version is None:
888 max_version = "Unknown"
890 return (max_version, max_vercode, max_package)
893 class BuildException(Exception):
894 def __init__(self, value, detail=None):
898 def get_wikitext(self):
899 ret = repr(self.value) + "\n"
903 txt = self.detail[-8192:] if len(self.detail) > 8192 else self.detail
911 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
915 class VCSException(Exception):
916 def __init__(self, value):
923 # Get the specified source library.
924 # Returns the path to it. Normally this is the path to be used when referencing
925 # it, which may be a subdirectory of the actual project. If you want the base
926 # directory of the project, pass 'basepath=True'.
927 def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
928 basepath=False, raw=False, prepare=True, preponly=False):
936 name, ref = spec.split('@')
938 number, name = name.split(':', 1)
940 name, subdir = name.split('/', 1)
942 if not name in metadata.srclibs:
943 raise BuildException('srclib ' + name + ' not found.')
945 srclib = metadata.srclibs[name]
947 sdir = os.path.join(srclib_dir, name)
950 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
951 vcs.srclib = (name, number, sdir)
953 vcs.gotorevision(ref)
960 libdir = os.path.join(sdir, subdir)
961 elif srclib["Subdir"]:
962 for subdir in srclib["Subdir"]:
963 libdir_candidate = os.path.join(sdir, subdir)
964 if os.path.exists(libdir_candidate):
965 libdir = libdir_candidate
971 if srclib["Srclibs"]:
973 for lib in srclib["Srclibs"].replace(';', ',').split(','):
975 for t in srclibpaths:
980 raise BuildException('Missing recursive srclib %s for %s' % (
982 place_srclib(libdir, n, s_tuple[2])
985 remove_signing_keys(sdir)
986 remove_debuggable_flags(sdir)
990 if srclib["Prepare"]:
991 cmd = replace_config_vars(srclib["Prepare"])
993 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
994 if p.returncode != 0:
995 raise BuildException("Error running prepare command for srclib %s"
1001 return (name, number, libdir)
1004 # Prepare the source code for a particular build
1005 # 'vcs' - the appropriate vcs object for the application
1006 # 'app' - the application details from the metadata
1007 # 'build' - the build details from the metadata
1008 # 'build_dir' - the path to the build directory, usually
1010 # 'srclib_dir' - the path to the source libraries directory, usually
1012 # 'extlib_dir' - the path to the external libraries directory, usually
1014 # Returns the (root, srclibpaths) where:
1015 # 'root' is the root directory, which may be the same as 'build_dir' or may
1016 # be a subdirectory of it.
1017 # 'srclibpaths' is information on the srclibs being used
1018 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False):
1020 # Optionally, the actual app source can be in a subdirectory
1021 if 'subdir' in build:
1022 root_dir = os.path.join(build_dir, build['subdir'])
1024 root_dir = build_dir
1026 # Get a working copy of the right revision
1027 logging.info("Getting source for revision " + build['commit'])
1028 vcs.gotorevision(build['commit'])
1030 # Initialise submodules if requred
1031 if build['submodules']:
1032 logging.info("Initialising submodules")
1033 vcs.initsubmodules()
1035 # Check that a subdir (if we're using one) exists. This has to happen
1036 # after the checkout, since it might not exist elsewhere
1037 if not os.path.exists(root_dir):
1038 raise BuildException('Missing subdir ' + root_dir)
1040 # Run an init command if one is required
1042 cmd = replace_config_vars(build['init'])
1043 logging.info("Running 'init' commands in %s" % root_dir)
1045 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1046 if p.returncode != 0:
1047 raise BuildException("Error running init command for %s:%s" %
1048 (app['id'], build['version']), p.stdout)
1050 # Apply patches if any
1051 if 'patch' in build:
1052 for patch in build['patch']:
1053 patch = patch.strip()
1054 logging.info("Applying " + patch)
1055 patch_path = os.path.join('metadata', app['id'], patch)
1056 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1057 if p.returncode != 0:
1058 raise BuildException("Failed to apply patch %s" % patch_path)
1060 # Get required source libraries
1062 if 'srclibs' in build:
1063 logging.info("Collecting source libraries")
1064 for lib in build['srclibs']:
1065 srclibpaths.append(getsrclib(lib, srclib_dir, srclibpaths,
1068 for name, number, libpath in srclibpaths:
1069 place_srclib(root_dir, int(number) if number else None, libpath)
1071 basesrclib = vcs.getsrclib()
1072 # If one was used for the main source, add that too.
1074 srclibpaths.append(basesrclib)
1076 # Update the local.properties file
1077 localprops = [os.path.join(build_dir, 'local.properties')]
1078 if 'subdir' in build:
1079 localprops += [os.path.join(root_dir, 'local.properties')]
1080 for path in localprops:
1081 if not os.path.isfile(path):
1083 logging.info("Updating properties file at %s" % path)
1088 # Fix old-fashioned 'sdk-location' by copying
1089 # from sdk.dir, if necessary
1090 if build['oldsdkloc']:
1091 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1092 re.S | re.M).group(1)
1093 props += "sdk-location=%s\n" % sdkloc
1095 props += "sdk.dir=%s\n" % config['sdk_path']
1096 props += "sdk-location=%s\n" % config['sdk_path']
1097 if 'ndk_path' in config:
1099 props += "ndk.dir=%s\n" % config['ndk_path']
1100 props += "ndk-location=%s\n" % config['ndk_path']
1101 # Add java.encoding if necessary
1102 if 'encoding' in build:
1103 props += "java.encoding=%s\n" % build['encoding']
1109 if build['type'] == 'gradle':
1110 flavour = build['gradle'].split('@')[0]
1111 if flavour in ['main', 'yes', '']:
1114 if 'target' in build:
1115 n = build["target"].split('-')[1]
1116 FDroidPopen(['sed', '-i',
1117 's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g',
1120 if '@' in build['gradle']:
1121 gradle_dir = os.path.join(root_dir, build['gradle'].split('@', 1)[1])
1122 gradle_dir = os.path.normpath(gradle_dir)
1123 FDroidPopen(['sed', '-i',
1124 's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g',
1128 # Remove forced debuggable flags
1129 remove_debuggable_flags(root_dir)
1131 # Insert version code and number into the manifest if necessary
1132 if build['forceversion']:
1133 logging.info("Changing the version name")
1134 for path in manifest_paths(root_dir, flavour):
1135 if not os.path.isfile(path):
1137 if has_extension(path, 'xml'):
1138 p = SilentPopen(['sed', '-i',
1139 's/android:versionName="[^"]*"/android:versionName="'
1140 + build['version'] + '"/g',
1142 if p.returncode != 0:
1143 raise BuildException("Failed to amend manifest")
1144 elif has_extension(path, 'gradle'):
1145 p = SilentPopen(['sed', '-i',
1146 's/versionName *=* *"[^"]*"/versionName = "'
1147 + build['version'] + '"/g',
1149 if p.returncode != 0:
1150 raise BuildException("Failed to amend build.gradle")
1151 if build['forcevercode']:
1152 logging.info("Changing the version code")
1153 for path in manifest_paths(root_dir, flavour):
1154 if not os.path.isfile(path):
1156 if has_extension(path, 'xml'):
1157 p = SilentPopen(['sed', '-i',
1158 's/android:versionCode="[^"]*"/android:versionCode="'
1159 + build['vercode'] + '"/g',
1161 if p.returncode != 0:
1162 raise BuildException("Failed to amend manifest")
1163 elif has_extension(path, 'gradle'):
1164 p = SilentPopen(['sed', '-i',
1165 's/versionCode *=* *[0-9]*/versionCode = '
1166 + build['vercode'] + '/g',
1168 if p.returncode != 0:
1169 raise BuildException("Failed to amend build.gradle")
1171 # Delete unwanted files
1173 for part in getpaths(build_dir, build, 'rm'):
1174 dest = os.path.join(build_dir, part)
1175 logging.info("Removing {0}".format(part))
1176 if os.path.lexists(dest):
1177 if os.path.islink(dest):
1178 SilentPopen(['unlink ' + dest], shell=True)
1180 SilentPopen(['rm -rf ' + dest], shell=True)
1182 logging.info("...but it didn't exist")
1184 remove_signing_keys(build_dir)
1186 # Add required external libraries
1187 if 'extlibs' in build:
1188 logging.info("Collecting prebuilt libraries")
1189 libsdir = os.path.join(root_dir, 'libs')
1190 if not os.path.exists(libsdir):
1192 for lib in build['extlibs']:
1194 logging.info("...installing extlib {0}".format(lib))
1195 libf = os.path.basename(lib)
1196 libsrc = os.path.join(extlib_dir, lib)
1197 if not os.path.exists(libsrc):
1198 raise BuildException("Missing extlib file {0}".format(libsrc))
1199 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1201 # Run a pre-build command if one is required
1202 if 'prebuild' in build:
1203 cmd = replace_config_vars(build['prebuild'])
1205 # Substitute source library paths into prebuild commands
1206 for name, number, libpath in srclibpaths:
1207 libpath = os.path.relpath(libpath, root_dir)
1208 cmd = cmd.replace('$$' + name + '$$', libpath)
1210 logging.info("Running 'prebuild' commands in %s" % root_dir)
1212 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1213 if p.returncode != 0:
1214 raise BuildException("Error running prebuild command for %s:%s" %
1215 (app['id'], build['version']), p.stdout)
1217 updatemode = build.get('update', ['auto'])
1218 # Generate (or update) the ant build file, build.xml...
1219 if updatemode != ['no'] and build['type'] == 'ant':
1220 parms = [os.path.join(config['sdk_path'], 'tools', 'android'), 'update']
1221 lparms = parms + ['lib-project']
1222 parms = parms + ['project']
1224 if 'target' in build and build['target']:
1225 parms += ['-t', build['target']]
1226 lparms += ['-t', build['target']]
1227 if updatemode == ['auto']:
1228 update_dirs = ant_subprojects(root_dir) + ['.']
1230 update_dirs = updatemode
1232 for d in update_dirs:
1233 subdir = os.path.join(root_dir, d)
1235 print("Updating main project")
1236 cmd = parms + ['-p', d]
1238 print("Updating subproject %s" % d)
1239 cmd = lparms + ['-p', d]
1240 p = FDroidPopen(cmd, cwd=root_dir)
1241 # Check to see whether an error was returned without a proper exit
1242 # code (this is the case for the 'no target set or target invalid'
1244 if p.returncode != 0 or p.stdout.startswith("Error: "):
1245 raise BuildException("Failed to update project at %s" % d, p.stdout)
1246 # Clean update dirs via ant
1248 logging.info("Cleaning subproject %s" % d)
1249 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1251 return (root_dir, srclibpaths)
1254 # Split and extend via globbing the paths from a field
1255 def getpaths(build_dir, build, field):
1257 if field not in build:
1259 for p in build[field]:
1261 full_path = os.path.join(build_dir, p)
1262 full_path = os.path.normpath(full_path)
1263 paths += [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1267 # Scan the source code in the given directory (and all subdirectories)
1268 # and return the number of fatal problems encountered
1269 def scan_source(build_dir, root_dir, thisbuild):
1273 # Common known non-free blobs (always lower case):
1275 re.compile(r'flurryagent', re.IGNORECASE),
1276 re.compile(r'paypal.*mpl', re.IGNORECASE),
1277 re.compile(r'libgoogleanalytics', re.IGNORECASE),
1278 re.compile(r'admob.*sdk.*android', re.IGNORECASE),
1279 re.compile(r'googleadview', re.IGNORECASE),
1280 re.compile(r'googleadmobadssdk', re.IGNORECASE),
1281 re.compile(r'google.*play.*services', re.IGNORECASE),
1282 re.compile(r'crittercism', re.IGNORECASE),
1283 re.compile(r'heyzap', re.IGNORECASE),
1284 re.compile(r'jpct.*ae', re.IGNORECASE),
1285 re.compile(r'youtubeandroidplayerapi', re.IGNORECASE),
1286 re.compile(r'bugsense', re.IGNORECASE),
1287 re.compile(r'crashlytics', re.IGNORECASE),
1288 re.compile(r'ouya.*sdk', re.IGNORECASE),
1289 re.compile(r'libspen23', re.IGNORECASE),
1292 scanignore = getpaths(build_dir, thisbuild, 'scanignore')
1293 scandelete = getpaths(build_dir, thisbuild, 'scandelete')
1296 ms = magic.open(magic.MIME_TYPE)
1298 except AttributeError:
1302 for i in scanignore:
1303 if fd.startswith(i):
1308 for i in scandelete:
1309 if fd.startswith(i):
1313 def removeproblem(what, fd, fp):
1314 logging.info('Removing %s at %s' % (what, fd))
1317 def warnproblem(what, fd):
1318 logging.warn('Found %s at %s' % (what, fd))
1320 def handleproblem(what, fd, fp):
1322 removeproblem(what, fd, fp)
1324 logging.error('Found %s at %s' % (what, fd))
1328 def insidedir(path, dirname):
1329 return path.endswith('/%s' % dirname) or '/%s/' % dirname in path
1331 # Iterate through all files in the source code
1332 for r, d, f in os.walk(build_dir):
1334 if any(insidedir(r, d) for d in ('.hg', '.git', '.svn', '.bzr')):
1339 # Path (relative) to the file
1340 fp = os.path.join(r, curfile)
1341 fd = fp[len(build_dir) + 1:]
1343 # Check if this file has been explicitly excluded from scanning
1347 mime = magic.from_file(fp, mime=True) if ms is None else ms.file(fp)
1349 if mime == 'application/x-sharedlib':
1350 count += handleproblem('shared library', fd, fp)
1352 elif mime == 'application/x-archive':
1353 count += handleproblem('static library', fd, fp)
1355 elif mime == 'application/x-executable':
1356 count += handleproblem('binary executable', fd, fp)
1358 elif mime == 'application/x-java-applet':
1359 count += handleproblem('Java compiled class', fd, fp)
1364 'application/java-archive',
1365 'application/octet-stream',
1369 if has_extension(fp, 'apk'):
1370 removeproblem('APK file', fd, fp)
1372 elif has_extension(fp, 'jar'):
1374 if any(suspect.match(curfile) for suspect in usual_suspects):
1375 count += handleproblem('usual supect', fd, fp)
1377 warnproblem('JAR file', fd)
1379 elif has_extension(fp, 'zip'):
1380 warnproblem('ZIP file', fd)
1383 warnproblem('unknown compressed or binary file', fd)
1385 elif has_extension(fp, 'java'):
1386 for line in file(fp):
1387 if 'DexClassLoader' in line:
1388 count += handleproblem('DexClassLoader', fd, fp)
1393 # Presence of a jni directory without buildjni=yes might
1394 # indicate a problem (if it's not a problem, explicitly use
1395 # buildjni=no to bypass this check)
1396 if (os.path.exists(os.path.join(root_dir, 'jni')) and
1397 thisbuild.get('buildjni') is None):
1398 logging.warn('Found jni directory, but buildjni is not enabled')
1407 self.path = os.path.join('stats', 'known_apks.txt')
1409 if os.path.exists(self.path):
1410 for line in file(self.path):
1411 t = line.rstrip().split(' ')
1413 self.apks[t[0]] = (t[1], None)
1415 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1416 self.changed = False
1418 def writeifchanged(self):
1420 if not os.path.exists('stats'):
1422 f = open(self.path, 'w')
1424 for apk, app in self.apks.iteritems():
1426 line = apk + ' ' + appid
1428 line += ' ' + time.strftime('%Y-%m-%d', added)
1430 for line in sorted(lst):
1431 f.write(line + '\n')
1434 # Record an apk (if it's new, otherwise does nothing)
1435 # Returns the date it was added.
1436 def recordapk(self, apk, app):
1437 if not apk in self.apks:
1438 self.apks[apk] = (app, time.gmtime(time.time()))
1440 _, added = self.apks[apk]
1443 # Look up information - given the 'apkname', returns (app id, date added/None).
1444 # Or returns None for an unknown apk.
1445 def getapp(self, apkname):
1446 if apkname in self.apks:
1447 return self.apks[apkname]
1450 # Get the most recent 'num' apps added to the repo, as a list of package ids
1451 # with the most recent first.
1452 def getlatest(self, num):
1454 for apk, app in self.apks.iteritems():
1458 if apps[appid] > added:
1462 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1463 lst = [app for app, _ in sortedapps]
1468 def isApkDebuggable(apkfile, config):
1469 """Returns True if the given apk file is debuggable
1471 :param apkfile: full path to the apk to check"""
1473 p = SilentPopen([os.path.join(config['sdk_path'], 'build-tools',
1474 config['build_tools'], 'aapt'),
1475 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'])
1476 if p.returncode != 0:
1477 logging.critical("Failed to get apk manifest information")
1479 for line in p.stdout.splitlines():
1480 if 'android:debuggable' in line and not line.endswith('0x0'):
1485 class AsynchronousFileReader(threading.Thread):
1487 Helper class to implement asynchronous reading of a file
1488 in a separate thread. Pushes read lines on a queue to
1489 be consumed in another thread.
1492 def __init__(self, fd, queue):
1493 assert isinstance(queue, Queue.Queue)
1494 assert callable(fd.readline)
1495 threading.Thread.__init__(self)
1500 '''The body of the tread: read lines and put them on the queue.'''
1501 for line in iter(self._fd.readline, ''):
1502 self._queue.put(line)
1505 '''Check whether there is no more content to expect.'''
1506 return not self.is_alive() and self._queue.empty()
1514 def SilentPopen(commands, cwd=None, shell=False):
1515 return FDroidPopen(commands, cwd=cwd, shell=shell, output=False)
1518 def FDroidPopen(commands, cwd=None, shell=False, output=True):
1520 Run a command and capture the possibly huge output.
1522 :param commands: command and argument list like in subprocess.Popen
1523 :param cwd: optionally specifies a working directory
1524 :returns: A PopenResult.
1529 cwd = os.path.normpath(cwd)
1530 logging.info("Directory: %s" % cwd)
1531 logging.info("> %s" % ' '.join(commands))
1533 result = PopenResult()
1534 p = subprocess.Popen(commands, cwd=cwd, shell=shell,
1535 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1537 stdout_queue = Queue.Queue()
1538 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1539 stdout_reader.start()
1541 # Check the queue for output (until there is no more to get)
1542 while not stdout_reader.eof():
1543 while not stdout_queue.empty():
1544 line = stdout_queue.get()
1545 if output and options.verbose:
1546 # Output directly to console
1547 sys.stdout.write(line)
1549 result.stdout += line
1554 result.returncode = p.returncode
1558 def remove_signing_keys(build_dir):
1559 comment = re.compile(r'[ ]*//')
1560 signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1562 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1563 re.compile(r'.*android\.signingConfigs\..*'),
1564 re.compile(r'.*variant\.outputFile = .*'),
1565 re.compile(r'.*\.readLine\(.*'),
1567 for root, dirs, files in os.walk(build_dir):
1568 if 'build.gradle' in files:
1569 path = os.path.join(root, 'build.gradle')
1571 with open(path, "r") as o:
1572 lines = o.readlines()
1575 with open(path, "w") as o:
1577 if comment.match(line):
1581 opened += line.count('{')
1582 opened -= line.count('}')
1585 if signing_configs.match(line):
1589 if any(s.match(line) for s in line_matches):
1595 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1598 'project.properties',
1600 'default.properties',
1603 if propfile in files:
1604 path = os.path.join(root, propfile)
1606 with open(path, "r") as o:
1607 lines = o.readlines()
1609 with open(path, "w") as o:
1611 if line.startswith('key.store'):
1613 if line.startswith('key.alias'):
1617 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1620 def replace_config_vars(cmd):
1621 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1622 cmd = cmd.replace('$$NDK$$', config['ndk_path'])
1623 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1627 def place_srclib(root_dir, number, libpath):
1630 relpath = os.path.relpath(libpath, root_dir)
1631 proppath = os.path.join(root_dir, 'project.properties')
1634 if os.path.isfile(proppath):
1635 with open(proppath, "r") as o:
1636 lines = o.readlines()
1638 with open(proppath, "w") as o:
1641 if line.startswith('android.library.reference.%d=' % number):
1642 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1647 o.write('android.library.reference.%d=%s\n' % (number, relpath))