import re
import shutil
import glob
+import requests
import stat
import subprocess
import time
import threading
import magic
import logging
+import hashlib
+import socket
+import xml.etree.ElementTree as XMLElementTree
+
from distutils.version import LooseVersion
+from zipfile import ZipFile
import metadata
+XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
+
config = None
options = None
env = None
+orig_path = None
default_config = {
'sdk_path': "$ANDROID_HOME",
- 'ndk_path': "$ANDROID_NDK",
- 'build_tools': "20.0.0",
+ 'ndk_paths': {
+ 'r9b': None,
+ 'r10e': "$ANDROID_NDK"
+ },
+ 'build_tools': "22.0.1",
'ant': "ant",
'mvn3': "mvn",
'gradle': 'gradle',
'sync_from_local_copy_dir': False,
+ 'make_current_version_link': True,
+ 'current_version_name_source': 'Name',
'update_stats': False,
'stats_ignore': [],
'stats_server': None,
'stats_to_carbon': False,
'repo_maxage': 0,
'build_server_always': False,
- 'keystore': os.path.join("$HOME", '.local', 'share', 'fdroidserver', 'keystore.jks'),
+ 'keystore': 'keystore.jks',
'smartcardoptions': [],
'char_limits': {
- 'Summary': 50,
- 'Description': 1500
+ 'Summary': 80,
+ 'Description': 4000
},
'keyaliases': {},
'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
}
-def fill_config_defaults(config):
+def fill_config_defaults(thisconfig):
for k, v in default_config.items():
- if k not in config:
- config[k] = v
+ if k not in thisconfig:
+ thisconfig[k] = v
# Expand paths (~users and $vars)
- for k in ['sdk_path', 'ndk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
- v = config[k]
- orig = v
- v = os.path.expanduser(v)
- v = os.path.expandvars(v)
- if orig != v:
- config[k] = v
- config[k + '_orig'] = orig
+ def expand_path(path):
+ if path is None:
+ return None
+ orig = path
+ path = os.path.expanduser(path)
+ path = os.path.expandvars(path)
+ if orig == path:
+ return None
+ return path
+
+ for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
+ v = thisconfig[k]
+ exp = expand_path(v)
+ if exp is not None:
+ thisconfig[k] = exp
+ thisconfig[k + '_orig'] = v
+
+ for k in ['ndk_paths']:
+ d = thisconfig[k]
+ for k2 in d.copy():
+ v = d[k2]
+ exp = expand_path(v)
+ if exp is not None:
+ thisconfig[k][k2] = exp
+ thisconfig[k][k2 + '_orig'] = v
def read_config(opts, config_file='config.py'):
The config is read from config_file, which is in the current directory when
any of the repo management commands are used.
"""
- global config, options, env
+ global config, options, env, orig_path
if config is not None:
return config
fill_config_defaults(config)
- if not test_sdk_exists(config):
- sys.exit(3)
-
- if not test_build_tools_exists(config):
- sys.exit(3)
-
- bin_paths = {
- 'aapt': [
- os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
- ],
- 'zipalign': [
- os.path.join(config['sdk_path'], 'tools', 'zipalign'),
- os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'zipalign'),
- ],
- 'android': [
- os.path.join(config['sdk_path'], 'tools', 'android'),
- ],
- 'adb': [
- os.path.join(config['sdk_path'], 'platform-tools', 'adb'),
- ],
- }
-
- for b, paths in bin_paths.items():
- config[b] = None
- for path in paths:
- if os.path.isfile(path):
- config[b] = path
- break
- if config[b] is None:
- logging.warn("Could not find %s in any of the following paths:\n%s" % (
- b, '\n'.join(paths)))
-
# There is no standard, so just set up the most common environment
# variables
env = os.environ
+ orig_path = env['PATH']
for n in ['ANDROID_HOME', 'ANDROID_SDK']:
env[n] = config['sdk_path']
- for n in ['ANDROID_NDK', 'NDK']:
- env[n] = config['ndk_path']
for k in ["keystorepass", "keypass"]:
if k in config:
return config
-def test_sdk_exists(config):
- if config['sdk_path'] == default_config['sdk_path']:
+def get_ndk_path(version):
+ if version is None:
+ version = 'r10e' # latest
+ paths = config['ndk_paths']
+ if version not in paths:
+ return ''
+ return paths[version] or ''
+
+
+def find_sdk_tools_cmd(cmd):
+ '''find a working path to a tool from the Android SDK'''
+
+ tooldirs = []
+ if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
+ # try to find a working path to this command, in all the recent possible paths
+ if 'build_tools' in config:
+ build_tools = os.path.join(config['sdk_path'], 'build-tools')
+ # if 'build_tools' was manually set and exists, check only that one
+ configed_build_tools = os.path.join(build_tools, config['build_tools'])
+ if os.path.exists(configed_build_tools):
+ tooldirs.append(configed_build_tools)
+ else:
+ # no configed version, so hunt known paths for it
+ for f in sorted(os.listdir(build_tools), reverse=True):
+ if os.path.isdir(os.path.join(build_tools, f)):
+ tooldirs.append(os.path.join(build_tools, f))
+ tooldirs.append(build_tools)
+ sdk_tools = os.path.join(config['sdk_path'], 'tools')
+ if os.path.exists(sdk_tools):
+ tooldirs.append(sdk_tools)
+ sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
+ if os.path.exists(sdk_platform_tools):
+ tooldirs.append(sdk_platform_tools)
+ tooldirs.append('/usr/bin')
+ for d in tooldirs:
+ if os.path.isfile(os.path.join(d, cmd)):
+ return os.path.join(d, cmd)
+ # did not find the command, exit with error message
+ ensure_build_tools_exists(config)
+
+
+def test_sdk_exists(thisconfig):
+ if 'sdk_path' not in thisconfig:
+ if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
+ return True
+ else:
+ logging.error("'sdk_path' not set in config.py!")
+ return False
+ if thisconfig['sdk_path'] == default_config['sdk_path']:
logging.error('No Android SDK found!')
logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
logging.error('\texport ANDROID_HOME=/opt/android-sdk')
return False
- if not os.path.exists(config['sdk_path']):
- logging.critical('Android SDK path "' + config['sdk_path'] + '" does not exist!')
+ if not os.path.exists(thisconfig['sdk_path']):
+ logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
return False
- if not os.path.isdir(config['sdk_path']):
- logging.critical('Android SDK path "' + config['sdk_path'] + '" is not a directory!')
+ if not os.path.isdir(thisconfig['sdk_path']):
+ logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
return False
for d in ['build-tools', 'platform-tools', 'tools']:
- if not os.path.isdir(os.path.join(config['sdk_path'], d)):
+ if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
- config['sdk_path'], d))
+ thisconfig['sdk_path'], d))
return False
return True
-def test_build_tools_exists(config):
- if not test_sdk_exists(config):
- return False
- build_tools = os.path.join(config['sdk_path'], 'build-tools')
- versioned_build_tools = os.path.join(build_tools, config['build_tools'])
+def ensure_build_tools_exists(thisconfig):
+ if not test_sdk_exists(thisconfig):
+ sys.exit(3)
+ build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
+ versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
if not os.path.isdir(versioned_build_tools):
logging.critical('Android Build Tools path "'
+ versioned_build_tools + '" does not exist!')
- return False
- return True
+ sys.exit(3)
def write_password_file(pwtype, password=None):
class vcs:
+
def __init__(self, remote, local):
# svn, git-svn and bzr may require auth
self.username = None
if self.repotype() in ('git-svn', 'bzr'):
if '@' in remote:
+ if self.repotype == 'git-svn':
+ raise VCSException("Authentication is not supported for git-svn")
self.username, remote = remote.split('@')
if ':' not in self.username:
raise VCSException("Password required with username")
writeback = False
else:
deleterepo = True
- logging.info(
- "Repository details for %s changed - deleting" % (
- self.local))
+ logging.info("Repository details for %s changed - deleting" % (
+ self.local))
else:
deleterepo = True
logging.info("Repository details for %s missing - deleting" % (
# Get a list of all known tags
def gettags(self):
- raise VCSException('gettags not supported for this vcs type')
-
- # Get a list of latest number tags
- def latesttags(self, number):
+ if not self._gettags:
+ raise VCSException('gettags not supported for this vcs type')
+ rtags = []
+ for tag in self._gettags():
+ if re.match('[-A-Za-z0-9_. ]+$', tag):
+ rtags.append(tag)
+ return rtags
+
+ def latesttags(self, tags, number):
+ """Get the most recent tags in a given list.
+
+ :param tags: a list of tags
+ :param number: the number to return
+ :returns: A list containing the most recent tags in the provided
+ list, up to the maximum number given.
+ """
raise VCSException('latesttags not supported for this vcs type')
# Get current commit reference (hash, revision, etc)
# fdroidserver) and then we'll proceed to destroy it! This is called as
# a safety check.
def checkrepo(self):
- p = SilentPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
+ p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
result = p.output.rstrip()
if not result.endswith(self.local):
raise VCSException('Repository mismatch')
else:
self.checkrepo()
# Discard any working tree changes
- p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
+ p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
+ 'git', 'reset', '--hard'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git reset failed", p.output)
# Remove untracked files now, in case they're tracked in the target
# revision (it happens!)
- p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
+ p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
+ 'git', 'clean', '-dffx'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git clean failed", p.output)
if not self.refreshed:
p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
if p.returncode != 0:
raise VCSException("Git fetch failed", p.output)
- p = SilentPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local)
+ p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git fetch failed", p.output)
# Recreate origin/HEAD as git clone would do it, in case it disappeared
- p = SilentPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local)
+ p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
if p.returncode != 0:
lines = p.output.splitlines()
if 'Multiple remote HEAD branches' not in lines[0]:
raise VCSException("Git remote set-head failed", p.output)
branch = lines[1].split(' ')[-1]
- p2 = SilentPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local)
+ p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
if p2.returncode != 0:
raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
self.refreshed = True
# origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
# a github repo. Most of the time this is the same as origin/master.
rev = rev or 'origin/HEAD'
- p = SilentPopen(['git', 'checkout', '-f', rev], cwd=self.local)
+ p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git checkout of '%s' failed" % rev, p.output)
# Get rid of any uncontrolled files left behind
- p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
+ p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git clean failed", p.output)
line = line.replace('git@github.com:', 'https://github.com/')
f.write(line)
- for cmd in [
- ['git', 'reset', '--hard'],
- ['git', 'clean', '-dffx'],
- ]:
- p = SilentPopen(['git', 'submodule', 'foreach', '--recursive'] + cmd, cwd=self.local)
- if p.returncode != 0:
- raise VCSException("Git submodule reset failed", p.output)
- p = SilentPopen(['git', 'submodule', 'sync'], cwd=self.local)
+ p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git submodule sync failed", p.output)
p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
if p.returncode != 0:
raise VCSException("Git submodule update failed", p.output)
- def gettags(self):
+ def _gettags(self):
self.checkrepo()
- p = SilentPopen(['git', 'tag'], cwd=self.local)
+ p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
return p.output.splitlines()
- def latesttags(self, alltags, number):
+ def latesttags(self, tags, number):
self.checkrepo()
- p = SilentPopen(['echo "' + '\n'.join(alltags) + '" | '
- + 'xargs -I@ git log --format=format:"%at @%n" -1 @ | '
- + 'sort -n | awk \'{print $2}\''],
- cwd=self.local, shell=True)
- return p.output.splitlines()[-number:]
+ tl = []
+ for tag in tags:
+ p = FDroidPopen(
+ ['git', 'show', '--format=format:%ct', '-s', tag],
+ cwd=self.local, output=False)
+ # Timestamp is on the last line. For a normal tag, it's the only
+ # line, but for annotated tags, the rest of the info precedes it.
+ ts = int(p.output.splitlines()[-1])
+ tl.append((ts, tag))
+ latest = []
+ for _, t in sorted(tl)[-number:]:
+ latest.append(t)
+ return latest
class vcs_gitsvn(vcs):
def repotype(self):
return 'git-svn'
- # Damn git-svn tries to use a graphical password prompt, so we have to
- # trick it into taking the password from stdin
- def userargs(self):
- if self.username is None:
- return ('', '')
- return ('echo "%s" | DISPLAY="" ' % self.password, ' --username "%s"' % self.username)
-
# If the local directory exists, but is somehow not a git repository, git
# will traverse up the directory tree until it finds one that is (i.e.
# fdroidserver) and then we'll proceed to destory it! This is called as
# a safety check.
def checkrepo(self):
- p = SilentPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
+ p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
result = p.output.rstrip()
if not result.endswith(self.local):
raise VCSException('Repository mismatch')
def gotorevisionx(self, rev):
if not os.path.exists(self.local):
# Brand new checkout
- gitsvn_cmd = '%sgit svn clone%s' % self.userargs()
+ gitsvn_args = ['git', 'svn', 'clone']
if ';' in self.remote:
remote_split = self.remote.split(';')
for i in remote_split[1:]:
if i.startswith('trunk='):
- gitsvn_cmd += ' -T %s' % i[6:]
+ gitsvn_args.extend(['-T', i[6:]])
elif i.startswith('tags='):
- gitsvn_cmd += ' -t %s' % i[5:]
+ gitsvn_args.extend(['-t', i[5:]])
elif i.startswith('branches='):
- gitsvn_cmd += ' -b %s' % i[9:]
- p = SilentPopen([gitsvn_cmd + " %s %s" % (remote_split[0], self.local)], shell=True)
+ gitsvn_args.extend(['-b', i[9:]])
+ gitsvn_args.extend([remote_split[0], self.local])
+ p = FDroidPopen(gitsvn_args, output=False)
if p.returncode != 0:
self.clone_failed = True
raise VCSException("Git svn clone failed", p.output)
else:
- p = SilentPopen([gitsvn_cmd + " %s %s" % (self.remote, self.local)], shell=True)
+ gitsvn_args.extend([self.remote, self.local])
+ p = FDroidPopen(gitsvn_args, output=False)
if p.returncode != 0:
self.clone_failed = True
raise VCSException("Git svn clone failed", p.output)
else:
self.checkrepo()
# Discard any working tree changes
- p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
+ p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git reset failed", p.output)
# Remove untracked files now, in case they're tracked in the target
# revision (it happens!)
- p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
+ p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git clean failed", p.output)
if not self.refreshed:
# Get new commits, branches and tags from repo
- p = SilentPopen(['%sgit svn fetch %s' % self.userargs()], cwd=self.local, shell=True)
+ p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git svn fetch failed")
- p = SilentPopen(['%sgit svn rebase %s' % self.userargs()], cwd=self.local, shell=True)
+ p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git svn rebase failed", p.output)
self.refreshed = True
nospaces_rev = rev.replace(' ', '%20')
# Try finding a svn tag
for treeish in ['origin/', '']:
- p = SilentPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev],
- cwd=self.local)
+ p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
if p.returncode == 0:
break
if p.returncode != 0:
svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
- p = SilentPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish],
- cwd=self.local)
+ p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
git_rev = p.output.rstrip()
if p.returncode == 0 and git_rev:
if p.returncode != 0 or not git_rev:
# Try a plain git checkout as a last resort
- p = SilentPopen(['git', 'checkout', rev], cwd=self.local)
+ p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
else:
# Check out the git rev equivalent to the svn rev
- p = SilentPopen(['git', 'checkout', git_rev], cwd=self.local)
+ p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git checkout of '%s' failed" % rev, p.output)
# Get rid of any uncontrolled files left behind
- p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
+ p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git clean failed", p.output)
- def gettags(self):
+ def _gettags(self):
self.checkrepo()
for treeish in ['origin/', '']:
d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
def getref(self):
self.checkrepo()
- p = SilentPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local)
+ p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
if p.returncode != 0:
return None
return p.output.strip()
def gotorevisionx(self, rev):
if not os.path.exists(self.local):
- p = SilentPopen(['hg', 'clone', self.remote, self.local])
+ p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
if p.returncode != 0:
self.clone_failed = True
raise VCSException("Hg clone failed", p.output)
else:
- p = SilentPopen(['hg status -uS | xargs rm -rf'], cwd=self.local, shell=True)
+ p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
if p.returncode != 0:
- raise VCSException("Hg clean failed", p.output)
+ raise VCSException("Hg status failed", p.output)
+ for line in p.output.splitlines():
+ if not line.startswith('? '):
+ raise VCSException("Unexpected output from hg status -uS: " + line)
+ FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
if not self.refreshed:
- p = SilentPopen(['hg', 'pull'], cwd=self.local)
+ p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Hg pull failed", p.output)
self.refreshed = True
rev = rev or 'default'
if not rev:
return
- p = SilentPopen(['hg', 'update', '-C', rev], cwd=self.local)
+ p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
- p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
+ p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
# Also delete untracked files, we have to enable purge extension for that:
if "'purge' is provided by the following extension" in p.output:
with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
myfile.write("\n[extensions]\nhgext.purge=\n")
- p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
+ p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("HG purge failed", p.output)
elif p.returncode != 0:
raise VCSException("HG purge failed", p.output)
- def gettags(self):
- p = SilentPopen(['hg', 'tags', '-q'], cwd=self.local)
+ def _gettags(self):
+ p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
return p.output.splitlines()[1:]
def gotorevisionx(self, rev):
if not os.path.exists(self.local):
- p = SilentPopen(['bzr', 'branch', self.remote, self.local])
+ p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
if p.returncode != 0:
self.clone_failed = True
raise VCSException("Bzr branch failed", p.output)
else:
- p = SilentPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local)
+ p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Bzr revert failed", p.output)
if not self.refreshed:
- p = SilentPopen(['bzr', 'pull'], cwd=self.local)
+ p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Bzr update failed", p.output)
self.refreshed = True
revargs = list(['-r', rev] if rev else [])
- p = SilentPopen(['bzr', 'revert'] + revargs, cwd=self.local)
+ p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
- def gettags(self):
- p = SilentPopen(['bzr', 'tags'], cwd=self.local)
+ def _gettags(self):
+ p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
return [tag.split(' ')[0].strip() for tag in
p.output.splitlines()]
-def retrieve_string(app_dir, string, xmlfiles=None):
+def unescape_string(string):
+ if string[0] == '"' and string[-1] == '"':
+ return string[1:-1]
- res_dirs = [
- os.path.join(app_dir, 'res'),
- os.path.join(app_dir, 'src', 'main'),
- ]
+ return string.replace("\\'", "'")
+
+
+def retrieve_string(app_dir, string, xmlfiles=None):
if xmlfiles is None:
xmlfiles = []
- for res_dir in res_dirs:
+ for res_dir in [
+ os.path.join(app_dir, 'res'),
+ os.path.join(app_dir, 'src', 'main', 'res'),
+ ]:
for r, d, f in os.walk(res_dir):
if os.path.basename(r) == 'values':
xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
- string_search = None
- if string.startswith('@string/'):
- string_search = re.compile(r'.*name="' + string[8:] + '".*?>"?([^<]+?)"?<.*').search
- elif string.startswith('&') and string.endswith(';'):
- string_search = re.compile(r'.*<!ENTITY.*' + string[1:-1] + '.*?"([^"]+?)".*>').search
-
- if string_search is not None:
- for xmlfile in xmlfiles:
- for line in file(xmlfile):
- matches = string_search(line)
- if matches:
- return retrieve_string(app_dir, matches.group(1), xmlfiles)
- return None
+ if not string.startswith('@string/'):
+ return unescape_string(string)
+
+ name = string[len('@string/'):]
+
+ for path in xmlfiles:
+ if not os.path.isfile(path):
+ continue
+ xml = parse_xml(path)
+ element = xml.find('string[@name="' + name + '"]')
+ if element is not None and element.text is not None:
+ return retrieve_string(app_dir, element.text.encode('utf-8'), xmlfiles)
+
+ return ''
- return string.replace("\\'", "'")
+
+def retrieve_string_singleline(app_dir, string, xmlfiles=None):
+ return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
# Return list of existing files that will be used to find the highest vercode
# Retrieve the package name. Returns the name, or None if not found.
def fetch_real_name(app_dir, flavours):
- app_search = re.compile(r'.*<application.*').search
- name_search = re.compile(r'.*android:label="([^"]+)".*').search
- app_found = False
- for f in manifest_paths(app_dir, flavours):
- if not has_extension(f, 'xml'):
+ for path in manifest_paths(app_dir, flavours):
+ if not has_extension(path, 'xml') or not os.path.isfile(path):
continue
- logging.debug("fetch_real_name: Checking manifest at " + f)
- for line in file(f):
- if not app_found:
- if app_search(line):
- app_found = True
- if app_found:
- matches = name_search(line)
- if matches:
- stringname = matches.group(1)
- logging.debug("fetch_real_name: using string " + stringname)
- result = retrieve_string(app_dir, stringname)
- if result:
- result = result.strip()
- return result
- return None
-
-
-# Retrieve the version name
-def version_name(original, app_dir, flavours):
- for f in manifest_paths(app_dir, flavours):
- if not has_extension(f, 'xml'):
+ logging.debug("fetch_real_name: Checking manifest at " + path)
+ xml = parse_xml(path)
+ app = xml.find('application')
+ if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
continue
- string = retrieve_string(app_dir, original)
- if string:
- return string
- return original
+ label = app.attrib["{http://schemas.android.com/apk/res/android}label"].encode('utf-8')
+ result = retrieve_string_singleline(app_dir, label)
+ if result:
+ result = result.strip()
+ return result
+ return None
def get_library_references(root_dir):
proppath = os.path.join(root_dir, 'project.properties')
if not os.path.isfile(proppath):
return libraries
- with open(proppath) as f:
- for line in f.readlines():
- if not line.startswith('android.library.reference.'):
- continue
- path = line.split('=')[1].strip()
- relpath = os.path.join(root_dir, path)
- if not os.path.isdir(relpath):
- continue
- logging.debug("Found subproject at %s" % path)
- libraries.append(path)
+ for line in file(proppath):
+ if not line.startswith('android.library.reference.'):
+ continue
+ path = line.split('=')[1].strip()
+ relpath = os.path.join(root_dir, path)
+ if not os.path.isdir(relpath):
+ continue
+ logging.debug("Found subproject at %s" % path)
+ libraries.append(path)
return libraries
for root, dirs, files in os.walk(root_dir):
if 'AndroidManifest.xml' in files:
path = os.path.join(root, 'AndroidManifest.xml')
- p = SilentPopen(['sed', '-i', 's/android:debuggable="[^"]*"//g', path])
+ p = FDroidPopen(['sed', '-i', 's/android:debuggable="[^"]*"//g', path], output=False)
if p.returncode != 0:
raise BuildException("Failed to remove debuggable flags of %s" % path)
if not paths:
return (None, None, None)
- vcsearch = re.compile(r'.*:versionCode="([0-9]+?)".*').search
- vnsearch = re.compile(r'.*:versionName="([^"]+?)".*').search
- psearch = re.compile(r'.*package="([^"]+)".*').search
-
vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
psearch_g = re.compile(r'.*packageName *=* *["\']([^"]+)["\'].*').search
for path in paths:
+ if not os.path.isfile(path):
+ continue
+
+ logging.debug("Parsing manifest at {0}".format(path))
gradle = has_extension(path, 'gradle')
version = None
vercode = None
# Remember package name, may be defined separately from version+vercode
package = max_package
- for line in file(path):
- if not package:
- if gradle:
+ if gradle:
+ for line in file(path):
+ if not package:
matches = psearch_g(line)
- else:
- matches = psearch(line)
- if matches:
- package = matches.group(1)
- if not version:
- if gradle:
+ if matches:
+ package = matches.group(1)
+ if not version:
matches = vnsearch_g(line)
- else:
- matches = vnsearch(line)
- if matches:
- version = matches.group(2 if gradle else 1)
- if not vercode:
- if gradle:
+ if matches:
+ version = matches.group(2)
+ if not vercode:
matches = vcsearch_g(line)
- else:
- matches = vcsearch(line)
- if matches:
- vercode = matches.group(1)
+ if matches:
+ vercode = matches.group(1)
+ else:
+ xml = parse_xml(path)
+ if "package" in xml.attrib:
+ package = xml.attrib["package"].encode('utf-8')
+ if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
+ version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"].encode('utf-8')
+ base_dir = os.path.dirname(path)
+ version = retrieve_string_singleline(base_dir, version)
+ if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
+ a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"].encode('utf-8')
+ if string_is_integer(a):
+ vercode = a
+
+ logging.debug("..got package={0}, version={1}, vercode={2}"
+ .format(package, version, vercode))
# Always grab the package name and version name in case they are not
# together with the highest version code
if max_version is None:
max_version = "Unknown"
+ if max_package and not is_valid_package_name(max_package):
+ raise FDroidException("Invalid package name {0}".format(max_package))
+
return (max_version, max_vercode, max_package)
+def is_valid_package_name(name):
+ return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
+
+
class FDroidException(Exception):
+
def __init__(self, value, detail=None):
self.value = value
self.detail = detail
# Returns the path to it. Normally this is the path to be used when referencing
# it, which may be a subdirectory of the actual project. If you want the base
# directory of the project, pass 'basepath=True'.
-def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
- basepath=False, raw=False, prepare=True, preponly=False):
+def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
+ raw=False, prepare=True, preponly=False):
number = None
subdir = None
if libdir is None:
libdir = sdir
- if srclib["Srclibs"]:
- n = 1
- for lib in srclib["Srclibs"].replace(';', ',').split(','):
- s_tuple = None
- for t in srclibpaths:
- if t[0] == lib:
- s_tuple = t
- break
- if s_tuple is None:
- raise VCSException('Missing recursive srclib %s for %s' % (
- lib, name))
- place_srclib(libdir, n, s_tuple[2])
- n += 1
-
remove_signing_keys(sdir)
remove_debuggable_flags(sdir)
if prepare:
if srclib["Prepare"]:
- cmd = replace_config_vars(srclib["Prepare"])
+ cmd = replace_config_vars(srclib["Prepare"], None)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
if p.returncode != 0:
logging.info("Getting source for revision " + build['commit'])
vcs.gotorevision(build['commit'])
- # Initialise submodules if requred
+ # Initialise submodules if required
if build['submodules']:
logging.info("Initialising submodules")
vcs.initsubmodules()
# Run an init command if one is required
if build['init']:
- cmd = replace_config_vars(build['init'])
+ cmd = replace_config_vars(build['init'], build)
logging.info("Running 'init' commands in %s" % root_dir)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if build['srclibs']:
logging.info("Collecting source libraries")
for lib in build['srclibs']:
- srclibpaths.append(getsrclib(lib, srclib_dir, srclibpaths,
- preponly=onserver))
+ srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver))
for name, number, libpath in srclibpaths:
place_srclib(root_dir, int(number) if number else None, libpath)
if build['subdir']:
localprops += [os.path.join(root_dir, 'local.properties')]
for path in localprops:
- if not os.path.isfile(path):
- continue
- logging.info("Updating properties file at %s" % path)
- f = open(path, 'r')
- props = f.read()
- f.close()
- props += '\n'
+ props = ""
+ if os.path.isfile(path):
+ logging.info("Updating local.properties file at %s" % path)
+ f = open(path, 'r')
+ props += f.read()
+ f.close()
+ props += '\n'
+ else:
+ logging.info("Creating local.properties file at %s" % path)
# Fix old-fashioned 'sdk-location' by copying
# from sdk.dir, if necessary
if build['oldsdkloc']:
else:
props += "sdk.dir=%s\n" % config['sdk_path']
props += "sdk-location=%s\n" % config['sdk_path']
- if 'ndk_path' in config:
+ if build['ndk_path']:
# Add ndk location
- props += "ndk.dir=%s\n" % config['ndk_path']
- props += "ndk-location=%s\n" % config['ndk_path']
+ props += "ndk.dir=%s\n" % build['ndk_path']
+ props += "ndk-location=%s\n" % build['ndk_path']
# Add java.encoding if necessary
if build['encoding']:
props += "java.encoding=%s\n" % build['encoding']
if build['type'] == 'gradle':
flavours = build['gradle']
- version_regex = re.compile(r".*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
+ version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
gradlepluginver = None
- gradle_files = [os.path.join(root_dir, 'build.gradle')]
+ gradle_dirs = [root_dir]
# Parent dir build.gradle
parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
if parent_dir.startswith(build_dir):
- gradle_files.append(os.path.join(parent_dir, 'build.gradle'))
+ gradle_dirs.append(parent_dir)
- for path in gradle_files:
+ for dir_path in gradle_dirs:
if gradlepluginver:
break
- if not os.path.isfile(path):
+ if not os.path.isdir(dir_path):
continue
- with open(path) as f:
- for line in f:
+ for filename in os.listdir(dir_path):
+ if not filename.endswith('.gradle'):
+ continue
+ path = os.path.join(dir_path, filename)
+ if not os.path.isfile(path):
+ continue
+ for line in file(path):
match = version_regex.match(line)
if match:
gradlepluginver = match.group(1)
if build['target']:
n = build["target"].split('-')[1]
- SilentPopen(['sed', '-i',
+ FDroidPopen(['sed', '-i',
's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g',
- 'build.gradle'],
- cwd=root_dir)
+ 'build.gradle'], cwd=root_dir, output=False)
# Remove forced debuggable flags
remove_debuggable_flags(root_dir)
if not os.path.isfile(path):
continue
if has_extension(path, 'xml'):
- p = SilentPopen(['sed', '-i',
- 's/android:versionName="[^"]*"/android:versionName="'
- + build['version'] + '"/g',
- path])
+ p = FDroidPopen(['sed', '-i',
+ 's/android:versionName="[^"]*"/android:versionName="' + build['version'] + '"/g',
+ path], output=False)
if p.returncode != 0:
raise BuildException("Failed to amend manifest")
elif has_extension(path, 'gradle'):
- p = SilentPopen(['sed', '-i',
- 's/versionName *=* *"[^"]*"/versionName = "'
- + build['version'] + '"/g',
- path])
+ p = FDroidPopen(['sed', '-i',
+ 's/versionName *=* *.*/versionName = "' + build['version'] + '"/g',
+ path], output=False)
if p.returncode != 0:
raise BuildException("Failed to amend build.gradle")
if build['forcevercode']:
if not os.path.isfile(path):
continue
if has_extension(path, 'xml'):
- p = SilentPopen(['sed', '-i',
- 's/android:versionCode="[^"]*"/android:versionCode="'
- + build['vercode'] + '"/g',
- path])
+ p = FDroidPopen(['sed', '-i',
+ 's/android:versionCode="[^"]*"/android:versionCode="' + build['vercode'] + '"/g',
+ path], output=False)
if p.returncode != 0:
raise BuildException("Failed to amend manifest")
elif has_extension(path, 'gradle'):
- p = SilentPopen(['sed', '-i',
- 's/versionCode *=* *[0-9]*/versionCode = '
- + build['vercode'] + '/g',
- path])
+ p = FDroidPopen(['sed', '-i',
+ 's/versionCode *=* *[0-9]*/versionCode = ' + build['vercode'] + '/g',
+ path], output=False)
if p.returncode != 0:
raise BuildException("Failed to amend build.gradle")
logging.info("Removing {0}".format(part))
if os.path.lexists(dest):
if os.path.islink(dest):
- SilentPopen(['unlink ' + dest], shell=True)
+ FDroidPopen(['unlink', dest], output=False)
else:
- SilentPopen(['rm -rf ' + dest], shell=True)
+ FDroidPopen(['rm', '-rf', dest], output=False)
else:
logging.info("...but it didn't exist")
if build['prebuild']:
logging.info("Running 'prebuild' commands in %s" % root_dir)
- cmd = replace_config_vars(build['prebuild'])
+ cmd = replace_config_vars(build['prebuild'], build)
# Substitute source library paths into prebuild commands
for name, number, libpath in srclibpaths:
# Generate (or update) the ant build file, build.xml...
if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
- parms = [config['android'], 'update', 'lib-project']
- lparms = [config['android'], 'update', 'project']
+ parms = ['android', 'update', 'lib-project']
+ lparms = ['android', 'update', 'project']
if build['target']:
parms += ['-t', build['target']]
else:
logging.debug("Updating subproject %s" % d)
cmd = lparms + ['-p', d]
- p = FDroidPopen(cmd, cwd=root_dir)
+ p = SdkToolsPopen(cmd, cwd=root_dir)
# Check to see whether an error was returned without a proper exit
# code (this is the case for the 'no target set or target invalid'
# error)
# Common known non-free blobs (always lower case):
usual_suspects = [
- re.compile(r'flurryagent', re.IGNORECASE),
- re.compile(r'paypal.*mpl', re.IGNORECASE),
- re.compile(r'google.*analytics', re.IGNORECASE),
- re.compile(r'admob.*sdk.*android', re.IGNORECASE),
- re.compile(r'google.*ad.*view', re.IGNORECASE),
- re.compile(r'google.*admob', re.IGNORECASE),
- re.compile(r'google.*play.*services', re.IGNORECASE),
- re.compile(r'crittercism', re.IGNORECASE),
- re.compile(r'heyzap', re.IGNORECASE),
- re.compile(r'jpct.*ae', re.IGNORECASE),
- re.compile(r'youtube.*android.*player.*api', re.IGNORECASE),
- re.compile(r'bugsense', re.IGNORECASE),
- re.compile(r'crashlytics', re.IGNORECASE),
- re.compile(r'ouya.*sdk', re.IGNORECASE),
- re.compile(r'libspen23', re.IGNORECASE),
- ]
+ re.compile(r'.*flurryagent', re.IGNORECASE),
+ re.compile(r'.*paypal.*mpl', re.IGNORECASE),
+ re.compile(r'.*google.*analytics', re.IGNORECASE),
+ re.compile(r'.*admob.*sdk.*android', re.IGNORECASE),
+ re.compile(r'.*google.*ad.*view', re.IGNORECASE),
+ re.compile(r'.*google.*admob', re.IGNORECASE),
+ re.compile(r'.*google.*play.*services', re.IGNORECASE),
+ re.compile(r'.*crittercism', re.IGNORECASE),
+ re.compile(r'.*heyzap', re.IGNORECASE),
+ re.compile(r'.*jpct.*ae', re.IGNORECASE),
+ re.compile(r'.*youtube.*android.*player.*api', re.IGNORECASE),
+ re.compile(r'.*bugsense', re.IGNORECASE),
+ re.compile(r'.*crashlytics', re.IGNORECASE),
+ re.compile(r'.*ouya.*sdk', re.IGNORECASE),
+ re.compile(r'.*libspen23', re.IGNORECASE),
+ ]
scanignore = getpaths(build_dir, thisbuild, 'scanignore')
scandelete = getpaths(build_dir, thisbuild, 'scandelete')
+ scanignore_worked = set()
+ scandelete_worked = set()
+
try:
ms = magic.open(magic.MIME_TYPE)
ms.load()
ms = None
def toignore(fd):
- for i in scanignore:
- if fd.startswith(i):
+ for p in scanignore:
+ if fd.startswith(p):
+ scanignore_worked.add(p)
return True
return False
def todelete(fd):
- for i in scandelete:
- if fd.startswith(i):
+ for p in scandelete:
+ if fd.startswith(p):
+ scandelete_worked.add(p)
return True
return False
+ def ignoreproblem(what, fd, fp):
+ logging.info('Ignoring %s at %s' % (what, fd))
+ return 0
+
def removeproblem(what, fd, fp):
logging.info('Removing %s at %s' % (what, fd))
os.remove(fp)
+ return 0
def warnproblem(what, fd):
logging.warn('Found %s at %s' % (what, fd))
def handleproblem(what, fd, fp):
if toignore(fd):
- logging.info('Ignoring %s at %s' % (what, fd))
- elif todelete(fd):
- removeproblem(what, fd, fp)
- else:
- logging.error('Found %s at %s' % (what, fd))
- return True
- return False
+ return ignoreproblem(what, fd, fp)
+ if todelete(fd):
+ return removeproblem(what, fd, fp)
+ logging.error('Found %s at %s' % (what, fd))
+ return 1
# Iterate through all files in the source code
for r, d, f in os.walk(build_dir, topdown=True):
'application/zip',
'application/java-archive',
'application/octet-stream',
- 'binary',
- ):
+ 'binary', ):
if has_extension(fp, 'apk'):
removeproblem('APK file', fd, fp)
warnproblem('unknown compressed or binary file', fd)
elif has_extension(fp, 'java'):
+ if not os.path.isfile(fp):
+ continue
for line in file(fp):
if 'DexClassLoader' in line:
count += handleproblem('DexClassLoader', fd, fp)
break
+
+ elif has_extension(fp, 'gradle'):
+ if not os.path.isfile(fp):
+ continue
+ for i, line in enumerate(file(fp)):
+ if any(suspect.match(line) for suspect in usual_suspects):
+ count += handleproblem('usual suspect at line %d' % i, fd, fp)
+ break
if ms is not None:
ms.close()
+ for p in scanignore:
+ if p not in scanignore_worked:
+ logging.error('Unused scanignore path: %s' % p)
+ count += 1
+
+ for p in scandelete:
+ if p not in scandelete_worked:
+ logging.error('Unused scandelete path: %s' % p)
+ count += 1
+
# Presence of a jni directory without buildjni=yes might
# indicate a problem (if it's not a problem, explicitly use
# buildjni=no to bypass this check)
def __init__(self):
self.path = os.path.join('stats', 'known_apks.txt')
self.apks = {}
- if os.path.exists(self.path):
+ if os.path.isfile(self.path):
for line in file(self.path):
t = line.rstrip().split(' ')
if len(t) == 2:
:param apkfile: full path to the apk to check"""
- p = SilentPopen([os.path.join(config['sdk_path'], 'build-tools',
- config['build_tools'], 'aapt'),
- 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'])
+ p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
+ output=False)
if p.returncode != 0:
logging.critical("Failed to get apk manifest information")
sys.exit(1)
class AsynchronousFileReader(threading.Thread):
+
'''
Helper class to implement asynchronous reading of a file
in a separate thread. Pushes read lines on a queue to
output = ''
-def SilentPopen(commands, cwd=None, shell=False):
- return FDroidPopen(commands, cwd=cwd, shell=shell, output=False)
+def SdkToolsPopen(commands, cwd=None, output=True):
+ cmd = commands[0]
+ if cmd not in config:
+ config[cmd] = find_sdk_tools_cmd(commands[0])
+ return FDroidPopen([config[cmd]] + commands[1:],
+ cwd=cwd, output=output)
-def FDroidPopen(commands, cwd=None, shell=False, output=True):
+def FDroidPopen(commands, cwd=None, output=True):
"""
Run a command and capture the possibly huge output.
result = PopenResult()
p = None
try:
- p = subprocess.Popen(commands, cwd=cwd, shell=shell, env=env,
+ p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
except OSError, e:
- raise BuildException("OSError while trying to execute " + ' '.join(commands) + ': ' + str(e))
+ raise BuildException("OSError while trying to execute " +
+ ' '.join(commands) + ': ' + str(e))
stdout_queue = Queue.Queue()
stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
re.compile(r'^[\t ]*signingConfig [^ ]*$'),
re.compile(r'.*android\.signingConfigs\.[^{]*$'),
re.compile(r'.*variant\.outputFile = .*'),
+ re.compile(r'.*output\.outputFile = .*'),
re.compile(r'.*\.readLine\(.*'),
- ]
+ ]
for root, dirs, files in os.walk(build_dir):
if 'build.gradle' in files:
path = os.path.join(root, 'build.gradle')
changed = False
opened = 0
+ i = 0
with open(path, "w") as o:
- for line in lines:
+ while i < len(lines):
+ line = lines[i]
+ i += 1
+ while line.endswith('\\\n'):
+ line = line.rstrip('\\\n') + lines[i]
+ i += 1
+
if comment.match(line):
continue
'project.properties',
'build.properties',
'default.properties',
- 'ant.properties',
- ]:
+ 'ant.properties', ]:
if propfile in files:
path = os.path.join(root, propfile)
logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
-def replace_config_vars(cmd):
+def reset_env_path():
+ global env, orig_path
+ env['PATH'] = orig_path
+
+
+def add_to_env_path(path):
+ global env
+ paths = env['PATH'].split(os.pathsep)
+ if path in paths:
+ return
+ paths.append(path)
+ env['PATH'] = os.pathsep.join(paths)
+
+
+def replace_config_vars(cmd, build):
+ global env
cmd = cmd.replace('$$SDK$$', config['sdk_path'])
- cmd = cmd.replace('$$NDK$$', config['ndk_path'])
+ # env['ANDROID_NDK'] is set in build_local right before prepare_source
+ cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
cmd = cmd.replace('$$MVN3$$', config['mvn3'])
+ if build is not None:
+ cmd = cmd.replace('$$COMMIT$$', build['commit'])
+ cmd = cmd.replace('$$VERSION$$', build['version'])
+ cmd = cmd.replace('$$VERCODE$$', build['vercode'])
return cmd
o.write(line)
if not placed:
o.write('android.library.reference.%d=%s\n' % (number, relpath))
+
+
+def verify_apks(signed_apk, unsigned_apk, tmp_dir):
+ """Verify that two apks are the same
+
+ One of the inputs is signed, the other is unsigned. The signature metadata
+ is transferred from the signed to the unsigned apk, and then jarsigner is
+ used to verify that the signature from the signed apk is also varlid for
+ the unsigned one.
+ :param signed_apk: Path to a signed apk file
+ :param unsigned_apk: Path to an unsigned apk file expected to match it
+ :param tmp_dir: Path to directory for temporary files
+ :returns: None if the verification is successful, otherwise a string
+ describing what went wrong.
+ """
+ sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
+ with ZipFile(signed_apk) as signed_apk_as_zip:
+ meta_inf_files = ['META-INF/MANIFEST.MF']
+ for f in signed_apk_as_zip.namelist():
+ if sigfile.match(f):
+ meta_inf_files.append(f)
+ if len(meta_inf_files) < 3:
+ return "Signature files missing from {0}".format(signed_apk)
+ signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
+ with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
+ for meta_inf_file in meta_inf_files:
+ unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
+
+ if subprocess.call(['jarsigner', '-verify', unsigned_apk]) != 0:
+ logging.info("...NOT verified - {0}".format(signed_apk))
+ return compare_apks(signed_apk, unsigned_apk, tmp_dir)
+ logging.info("...successfully verified")
+ return None
+
+
+def compare_apks(apk1, apk2, tmp_dir):
+ """Compare two apks
+
+ Returns None if the apk content is the same (apart from the signing key),
+ otherwise a string describing what's different, or what went wrong when
+ trying to do the comparison.
+ """
+
+ badchars = re.compile('''[/ :;'"]''')
+ apk1dir = os.path.join(tmp_dir, badchars.sub('_', apk1[0:-4])) # trim .apk
+ apk2dir = os.path.join(tmp_dir, badchars.sub('_', apk2[0:-4])) # trim .apk
+ for d in [apk1dir, apk2dir]:
+ if os.path.exists(d):
+ shutil.rmtree(d)
+ os.mkdir(d)
+ os.mkdir(os.path.join(d, 'jar-xf'))
+
+ if subprocess.call(['jar', 'xf',
+ os.path.abspath(apk1)],
+ cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
+ return("Failed to unpack " + apk1)
+ if subprocess.call(['jar', 'xf',
+ os.path.abspath(apk2)],
+ cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
+ return("Failed to unpack " + apk2)
+
+ # try to find apktool in the path, if it hasn't been manually configed
+ if 'apktool' not in config:
+ tmp = find_command('apktool')
+ if tmp is not None:
+ config['apktool'] = tmp
+ if 'apktool' in config:
+ if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
+ cwd=apk1dir) != 0:
+ return("Failed to unpack " + apk1)
+ if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
+ cwd=apk2dir) != 0:
+ return("Failed to unpack " + apk2)
+
+ p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
+ lines = p.output.splitlines()
+ if len(lines) != 1 or 'META-INF' not in lines[0]:
+ meld = find_command('meld')
+ if meld is not None:
+ p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
+ return("Unexpected diff output - " + p.output)
+
+ # since everything verifies, delete the comparison to keep cruft down
+ shutil.rmtree(apk1dir)
+ shutil.rmtree(apk2dir)
+
+ # If we get here, it seems like they're the same!
+ return None
+
+
+def find_command(command):
+ '''find the full path of a command, or None if it can't be found in the PATH'''
+
+ def is_exe(fpath):
+ return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+ fpath, fname = os.path.split(command)
+ if fpath:
+ if is_exe(command):
+ return command
+ else:
+ for path in os.environ["PATH"].split(os.pathsep):
+ path = path.strip('"')
+ exe_file = os.path.join(path, command)
+ if is_exe(exe_file):
+ return exe_file
+
+ return None
+
+
+def genpassword():
+ '''generate a random password for when generating keys'''
+ h = hashlib.sha256()
+ h.update(os.urandom(16)) # salt
+ h.update(bytes(socket.getfqdn()))
+ return h.digest().encode('base64').strip()
+
+
+def genkeystore(localconfig):
+ '''Generate a new key with random passwords and add it to new keystore'''
+ logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
+ keystoredir = os.path.dirname(localconfig['keystore'])
+ if keystoredir is None or keystoredir == '':
+ keystoredir = os.path.join(os.getcwd(), keystoredir)
+ if not os.path.exists(keystoredir):
+ os.makedirs(keystoredir, mode=0o700)
+
+ write_password_file("keystorepass", localconfig['keystorepass'])
+ write_password_file("keypass", localconfig['keypass'])
+ p = FDroidPopen(['keytool', '-genkey',
+ '-keystore', localconfig['keystore'],
+ '-alias', localconfig['repo_keyalias'],
+ '-keyalg', 'RSA', '-keysize', '4096',
+ '-sigalg', 'SHA256withRSA',
+ '-validity', '10000',
+ '-storepass:file', config['keystorepassfile'],
+ '-keypass:file', config['keypassfile'],
+ '-dname', localconfig['keydname']])
+ # TODO keypass should be sent via stdin
+ os.chmod(localconfig['keystore'], 0o0600)
+ if p.returncode != 0:
+ raise BuildException("Failed to generate key", p.output)
+ # now show the lovely key that was just generated
+ p = FDroidPopen(['keytool', '-list', '-v',
+ '-keystore', localconfig['keystore'],
+ '-alias', localconfig['repo_keyalias'],
+ '-storepass:file', config['keystorepassfile']])
+ logging.info(p.output.strip() + '\n\n')
+
+
+def write_to_config(thisconfig, key, value=None):
+ '''write a key/value to the local config.py'''
+ if value is None:
+ origkey = key + '_orig'
+ value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
+ with open('config.py', 'r') as f:
+ data = f.read()
+ pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
+ repl = '\n' + key + ' = "' + value + '"'
+ data = re.sub(pattern, repl, data)
+ # if this key is not in the file, append it
+ if not re.match('\s*' + key + '\s*=\s*"', data):
+ data += repl
+ # make sure the file ends with a carraige return
+ if not re.match('\n$', data):
+ data += '\n'
+ with open('config.py', 'w') as f:
+ f.writelines(data)
+
+
+def parse_xml(path):
+ return XMLElementTree.parse(path).getroot()
+
+
+def string_is_integer(string):
+ try:
+ int(string)
+ return True
+ except ValueError:
+ return False
+
+
+def download_file(url, local_filename=None, dldir='tmp'):
+ filename = url.split('/')[-1]
+ if local_filename is None:
+ local_filename = os.path.join(dldir, filename)
+ # the stream=True parameter keeps memory usage low
+ r = requests.get(url, stream=True)
+ with open(local_filename, 'wb') as f:
+ for chunk in r.iter_content(chunk_size=1024):
+ if chunk: # filter out keep-alive new chunks
+ f.write(chunk)
+ f.flush()
+ return local_filename
~ianmdlvl / fdroidserver.git / blobdiff