# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-import os, sys, re
+import os
+import sys
+import re
import shutil
+import glob
import stat
import subprocess
import time
import threading
import magic
import logging
+from distutils.version import LooseVersion
import metadata
config = None
options = None
+
+def get_default_config():
+ return {
+ 'sdk_path': os.getenv("ANDROID_HOME"),
+ 'ndk_path': os.getenv("ANDROID_NDK"),
+ 'build_tools': "19.1.0",
+ 'ant': "ant",
+ 'mvn3': "mvn",
+ 'gradle': 'gradle',
+ 'archive_older': 0,
+ 'update_stats': False,
+ 'stats_to_carbon': False,
+ 'repo_maxage': 0,
+ 'build_server_always': False,
+ 'keystore': os.path.join(os.getenv("HOME"), '.local', 'share', 'fdroidserver', 'keystore.jks'),
+ 'smartcardoptions': [],
+ 'char_limits': {
+ 'Summary': 50,
+ 'Description': 1500
+ },
+ 'keyaliases': {},
+ }
+
+
def read_config(opts, config_file='config.py'):
"""Read the repository config
config = {}
- logging.info("Reading %s" % config_file)
+ logging.debug("Reading %s" % config_file)
execfile(config_file, config)
- defconfig = {
- 'sdk_path': "$ANDROID_HOME",
- 'ndk_path': "$ANDROID_NDK",
- 'build_tools': "19.0.3",
- 'ant': "ant",
- 'mvn3': "mvn",
- 'gradle': 'gradle',
- 'archive_older': 0,
- 'update_stats': False,
- 'stats_to_carbon': False,
- 'repo_maxage': 0,
- 'build_server_always': False,
- 'char_limits': {
- 'Summary' : 50,
- 'Description' : 1500
- }
- }
+ # smartcardoptions must be a list since its command line args for Popen
+ if 'smartcardoptions' in config:
+ config['smartcardoptions'] = config['smartcardoptions'].split(' ')
+ elif 'keystore' in config and config['keystore'] == 'NONE':
+ # keystore='NONE' means use smartcard, these are required defaults
+ config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
+ 'SunPKCS11-OpenSC', '-providerClass',
+ 'sun.security.pkcs11.SunPKCS11',
+ '-providerArg', 'opensc-fdroid.cfg']
+
+ if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
+ st = os.stat(config_file)
+ if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
+ logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
+
+ defconfig = get_default_config()
for k, v in defconfig.items():
if k not in config:
config[k] = v
v = os.path.expanduser(v)
config[k] = os.path.expandvars(v)
- if not config['sdk_path']:
- logging.critical("$ANDROID_HOME is not set!")
- sys.exit(3)
- if not os.path.isdir(config['sdk_path']):
- logging.critical("$ANDROID_HOME points to a non-existing directory!")
+ if not test_sdk_exists(config):
sys.exit(3)
- if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
- st = os.stat(config_file)
- if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
- logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
+ for k in ["keystorepass", "keypass"]:
+ if k in config:
+ write_password_file(k)
+
+ # since this is used with rsync, where trailing slashes have meaning,
+ # ensure there is always a trailing slash
+ if 'serverwebroot' in config:
+ if config['serverwebroot'][-1] != '/':
+ config['serverwebroot'] += '/'
+ config['serverwebroot'] = config['serverwebroot'].replace('//', '/')
return config
+
+def test_sdk_exists(c):
+ if c['sdk_path'] is None:
+ # c['sdk_path'] is set to the value of ANDROID_HOME by default
+ logging.critical('No Android SDK found! ANDROID_HOME is not set and sdk_path is not in config.py!')
+ logging.info('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
+ logging.info('\texport ANDROID_HOME=/opt/android-sdk')
+ return False
+ if not os.path.exists(c['sdk_path']):
+ logging.critical('Android SDK path "' + c['sdk_path'] + '" does not exist!')
+ return False
+ if not os.path.isdir(c['sdk_path']):
+ logging.critical('Android SDK path "' + c['sdk_path'] + '" is not a directory!')
+ return False
+ if not os.path.isdir(os.path.join(c['sdk_path'], 'build-tools')):
+ logging.critical('Android SDK path "' + c['sdk_path'] + '" does not contain "build-tools/"!')
+ return False
+ if not os.path.isdir(os.path.join(c['sdk_path'], 'build-tools', c['build_tools'])):
+ logging.critical('Configured build-tools version "' + c['build_tools'] + '" not found in the SDK!')
+ return False
+ return True
+
+
+def test_build_tools_exists(c):
+ if not test_sdk_exists(c):
+ return False
+ build_tools = os.path.join(c['sdk_path'], 'build-tools')
+ versioned_build_tools = os.path.join(build_tools, c['build_tools'])
+ if not os.path.isdir(versioned_build_tools):
+ logging.critical('Android Build Tools path "'
+ + versioned_build_tools + '" does not exist!')
+ return False
+ if not os.path.exists(os.path.join(c['sdk_path'], 'build-tools', c['build_tools'], 'aapt')):
+ logging.critical('Android Build Tools "'
+ + versioned_build_tools
+ + '" does not contain "aapt"!')
+ return False
+ return True
+
+
+def write_password_file(pwtype, password=None):
+ '''
+ writes out passwords to a protected file instead of passing passwords as
+ command line argments
+ '''
+ filename = '.fdroid.' + pwtype + '.txt'
+ fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
+ if password is None:
+ os.write(fd, config[pwtype])
+ else:
+ os.write(fd, password)
+ os.close(fd)
+ config[pwtype + 'file'] = filename
+
+
# Given the arguments in the form of multiple appid:[vc] strings, this returns
# a dictionary with the set of vercodes specified for each package.
def read_pkg_args(args, allow_vercodes=False):
return vercodes
+
# On top of what read_pkg_args does, this returns the whole app metadata, but
# limiting the builds list to the builds matching the vercodes specified.
def read_app_args(args, allapps, allow_vercodes=False):
return apps
+
def has_extension(filename, extension):
name, ext = os.path.splitext(filename)
ext = ext.lower()[1:]
apk_regex = None
+
def apknameinfo(filename):
global apk_regex
filename = os.path.basename(filename)
raise Exception("Invalid apk name: %s" % filename)
return result
+
def getapkname(app, build):
return "%s_%s.apk" % (app['id'], build['vercode'])
+
def getsrcname(app, build):
return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
+
def getappname(app):
if app['Name']:
return app['Name']
return app['Auto Name']
return app['id']
+
def getcvname(app):
return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
+
def getvcs(vcstype, remote, local):
if vcstype == 'git':
return vcs_git(remote, local)
return getsrclib(remote, 'build/srclib', raw=True)
raise VCSException("Invalid vcs type " + vcstype)
+
def getsrclibvcs(name):
- srclib_path = os.path.join('srclibs', name + ".txt")
- if not os.path.exists(srclib_path):
+ if name not in metadata.srclibs:
raise VCSException("Missing srclib " + name)
- return metadata.parse_srclib(srclib_path)['Repo Type']
+ return metadata.srclibs[name]['Repo Type']
+
class vcs:
def __init__(self, remote, local):
self.refreshed = False
self.srclib = None
+ def repotype(self):
+ return None
+
# Take the local repository to a clean version of the given revision, which
# is specificed in the VCS's native format. Beforehand, the repository can
# be dirty, or even non-existent. If the repository does already exist
# and remote that directory was created from, allowing us to drop it
# automatically if either of those things changes.
fdpath = os.path.join(self.local, '..',
- '.fdroidvcs-' + os.path.basename(self.local))
+ '.fdroidvcs-' + os.path.basename(self.local))
cdata = self.repotype() + ' ' + self.remote
writeback = True
deleterepo = False
def gettags(self):
raise VCSException('gettags not supported for this vcs type')
+ # Get a list of latest number tags
+ def latesttags(self, number):
+ raise VCSException('latesttags not supported for this vcs type')
+
# Get current commit reference (hash, revision, etc)
def getref(self):
raise VCSException('getref not supported for this vcs type')
def getsrclib(self):
return self.srclib
+
class vcs_git(vcs):
def repotype(self):
def gotorevisionx(self, rev):
if not os.path.exists(self.local):
# Brand new checkout
- p = SilentPopen(['git', 'clone', self.remote, self.local])
+ p = FDroidPopen(['git', 'clone', self.remote, self.local])
if p.returncode != 0:
raise VCSException("Git clone failed")
self.checkrepo()
raise VCSException("Git clean failed")
if not self.refreshed:
# Get latest commits and tags from remote
- p = SilentPopen(['git', 'fetch', 'origin'], cwd=self.local)
+ p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
if p.returncode != 0:
raise VCSException("Git fetch failed")
p = SilentPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local)
def initsubmodules(self):
self.checkrepo()
+ submfile = os.path.join(self.local, '.gitmodules')
+ if not os.path.isfile(submfile):
+ raise VCSException("No git submodules available")
+
+ # fix submodules not accessible without an account and public key auth
+ with open(submfile, 'r') as f:
+ lines = f.readlines()
+ with open(submfile, 'w') as f:
+ for line in lines:
+ if 'git@github.com' in line:
+ line = line.replace('git@github.com:', 'https://github.com/')
+ f.write(line)
+
for cmd in [
['git', 'reset', '--hard'],
['git', 'clean', '-dffx'],
p = SilentPopen(['git', 'submodule', 'foreach', '--recursive'] + cmd, cwd=self.local)
if p.returncode != 0:
raise VCSException("Git submodule reset failed")
- p = SilentPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
+ p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local)
+ if p.returncode != 0:
+ raise VCSException("Git submodule sync failed")
+ p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
if p.returncode != 0:
raise VCSException("Git submodule update failed")
p = SilentPopen(['git', 'tag'], cwd=self.local)
return p.stdout.splitlines()
+ def latesttags(self, alltags, number):
+ self.checkrepo()
+ p = SilentPopen(['echo "' + '\n'.join(alltags) + '" | \
+ xargs -I@ git log --format=format:"%at @%n" -1 @ | \
+ sort -n | awk \'{print $2}\''],
+ cwd=self.local, shell=True)
+ return p.stdout.splitlines()[-number:]
+
class vcs_gitsvn(vcs):
svn_rev = rev_split[1]
else:
- # if no branch is specified, then assume trunk (ie. 'master'
+ # if no branch is specified, then assume trunk (ie. 'master'
# branch):
treeish = 'master'
svn_rev = rev
return None
return p.stdout.strip()
+
class vcs_svn(vcs):
def repotype(self):
return line[18:]
return None
+
class vcs_hg(vcs):
def repotype(self):
p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
# Also delete untracked files, we have to enable purge extension for that:
if "'purge' is provided by the following extension" in p.stdout:
- with open(self.local+"/.hg/hgrc", "a") as myfile:
+ with open(self.local + "/.hg/hgrc", "a") as myfile:
myfile.write("\n[extensions]\nhgext.purge=\n")
p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
if p.returncode != 0:
return [tag.split(' ')[0].strip() for tag in
p.stdout.splitlines()]
+
def retrieve_string(app_dir, string, xmlfiles=None):
- res_dir = os.path.join(app_dir, 'res')
+ res_dirs = [
+ os.path.join(app_dir, 'res'),
+ os.path.join(app_dir, 'src/main'),
+ ]
if xmlfiles is None:
xmlfiles = []
- for r,d,f in os.walk(res_dir):
- if r.endswith('/values'):
- xmlfiles += [os.path.join(r,x) for x in f if x.endswith('.xml')]
+ for res_dir in res_dirs:
+ for r, d, f in os.walk(res_dir):
+ if r.endswith('/values'):
+ xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
string_search = None
if string.startswith('@string/'):
- string_search = re.compile(r'.*"'+string[8:]+'".*?>([^<]+?)<.*').search
+ string_search = re.compile(r'.*name="' + string[8:] + '".*?>([^<]+?)<.*').search
elif string.startswith('&') and string.endswith(';'):
- string_search = re.compile(r'.*<!ENTITY.*'+string[1:-1]+'.*?"([^"]+?)".*>').search
+ string_search = re.compile(r'.*<!ENTITY.*' + string[1:-1] + '.*?"([^"]+?)".*>').search
if string_search is not None:
for xmlfile in xmlfiles:
return retrieve_string(app_dir, matches.group(1), xmlfiles)
return None
- return string.replace("\\'","'")
+ return string.replace("\\'", "'")
+
# Return list of existing files that will be used to find the highest vercode
def manifest_paths(app_dir, flavour):
- possible_manifests = [ os.path.join(app_dir, 'AndroidManifest.xml'),
- os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
- os.path.join(app_dir, 'build.gradle') ]
+ possible_manifests = \
+ [os.path.join(app_dir, 'AndroidManifest.xml'),
+ os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
+ os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
+ os.path.join(app_dir, 'build.gradle')]
if flavour:
possible_manifests.append(
- os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
+ os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
return [path for path in possible_manifests if os.path.isfile(path)]
+
# Retrieve the package name. Returns the name, or None if not found.
def fetch_real_name(app_dir, flavour):
app_search = re.compile(r'.*<application.*').search
return result
return None
+
# Retrieve the version name
def version_name(original, app_dir, flavour):
for f in manifest_paths(app_dir, flavour):
return string
return original
+
def get_library_references(root_dir):
libraries = []
proppath = os.path.join(root_dir, 'project.properties')
libraries.append(path)
return libraries
+
def ant_subprojects(root_dir):
subprojects = get_library_references(root_dir)
for subpath in subprojects:
subrelpath = os.path.join(root_dir, subpath)
for p in get_library_references(subrelpath):
- relp = os.path.normpath(os.path.join(subpath,p))
+ relp = os.path.normpath(os.path.join(subpath, p))
if relp not in subprojects:
subprojects.insert(0, relp)
return subprojects
+
def remove_debuggable_flags(root_dir):
# Remove forced debuggable flags
logging.info("Removing debuggable flags")
for root, dirs, files in os.walk(root_dir):
if 'AndroidManifest.xml' in files:
path = os.path.join(root, 'AndroidManifest.xml')
- p = FDroidPopen(['sed','-i', 's/android:debuggable="[^"]*"//g', path])
+ p = FDroidPopen(['sed', '-i', 's/android:debuggable="[^"]*"//g', path])
if p.returncode != 0:
raise BuildException("Failed to remove debuggable flags of %s" % path)
+
# Extract some information from the AndroidManifest.xml at the given path.
# Returns (version, vercode, package), any or all of which might be None.
# All values returned are strings.
-def parse_androidmanifests(paths):
+def parse_androidmanifests(paths, ignoreversions=None):
if not paths:
return (None, None, None)
vnsearch = re.compile(r'.*:versionName="([^"]+?)".*').search
psearch = re.compile(r'.*package="([^"]+)".*').search
- vcsearch_g = re.compile(r'.*versionCode[ ]*[=]*[ ]*["\']*([0-9]+)["\']*').search
- vnsearch_g = re.compile(r'.*versionName[ ]*[=]*[ ]*(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
- psearch_g = re.compile(r'.*packageName[ ]*[=]*[ ]*["\']([^"]+)["\'].*').search
+ vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
+ vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
+ psearch_g = re.compile(r'.*packageName *=* *["\']([^"]+)["\'].*').search
+
+ ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
max_version = None
max_vercode = None
if matches:
vercode = matches.group(1)
- # Better some package name than nothing
- if max_package is None:
+ # Always grab the package name and version name in case they are not
+ # together with the highest version code
+ if max_package is None and package is not None:
max_package = package
+ if max_version is None and version is not None:
+ max_version = version
if max_vercode is None or (vercode is not None and vercode > max_vercode):
- max_version = version
- max_vercode = vercode
- max_package = package
+ if not ignoresearch or not ignoresearch(version):
+ if version is not None:
+ max_version = version
+ if vercode is not None:
+ max_vercode = vercode
+ if package is not None:
+ max_package = package
+ else:
+ max_version = "Ignore"
if max_version is None:
max_version = "Unknown"
return (max_version, max_vercode, max_package)
+
class BuildException(Exception):
- def __init__(self, value, detail = None):
+ def __init__(self, value, detail=None):
self.value = value
self.detail = detail
return ret
def __str__(self):
- ret = repr(self.value)
+ ret = self.value
if self.detail:
ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
return ret
+
class VCSException(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
- return repr(self.value)
+ return self.value
+
# Get the specified source library.
# Returns the path to it. Normally this is the path to be used when referencing
# it, which may be a subdirectory of the actual project. If you want the base
# directory of the project, pass 'basepath=True'.
def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
- basepath=False, raw=False, prepare=True, preponly=False):
+ basepath=False, raw=False, prepare=True, preponly=False):
number = None
subdir = None
if ':' in name:
number, name = name.split(':', 1)
if '/' in name:
- name, subdir = name.split('/',1)
+ name, subdir = name.split('/', 1)
- srclib_path = os.path.join('srclibs', name + ".txt")
-
- if not os.path.exists(srclib_path):
+ if name not in metadata.srclibs:
raise BuildException('srclib ' + name + ' not found.')
- srclib = metadata.parse_srclib(srclib_path)
+ srclib = metadata.srclibs[name]
sdir = os.path.join(srclib_dir, name)
libdir = sdir
if srclib["Srclibs"]:
- for n,lib in enumerate(srclib["Srclibs"].replace(';',',').split(',')):
+ n = 1
+ for lib in srclib["Srclibs"].replace(';', ',').split(','):
s_tuple = None
for t in srclibpaths:
if t[0] == lib:
raise BuildException('Missing recursive srclib %s for %s' % (
lib, name))
place_srclib(libdir, n, s_tuple[2])
+ n += 1
remove_signing_keys(sdir)
remove_debuggable_flags(sdir)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
if p.returncode != 0:
raise BuildException("Error running prepare command for srclib %s"
- % name, p.stdout)
+ % name, p.stdout)
if basepath:
libdir = sdir
def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False):
# Optionally, the actual app source can be in a subdirectory
- if 'subdir' in build:
+ if build['subdir']:
root_dir = os.path.join(build_dir, build['subdir'])
else:
root_dir = build_dir
raise BuildException('Missing subdir ' + root_dir)
# Run an init command if one is required
- if 'init' in build:
+ if build['init']:
cmd = replace_config_vars(build['init'])
logging.info("Running 'init' commands in %s" % root_dir)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if p.returncode != 0:
raise BuildException("Error running init command for %s:%s" %
- (app['id'], build['version']), p.stdout)
+ (app['id'], build['version']), p.stdout)
# Apply patches if any
- if 'patch' in build:
+ if build['patch']:
+ logging.info("Applying patches")
for patch in build['patch']:
patch = patch.strip()
logging.info("Applying " + patch)
# Get required source libraries
srclibpaths = []
- if 'srclibs' in build:
+ if build['srclibs']:
logging.info("Collecting source libraries")
for lib in build['srclibs']:
srclibpaths.append(getsrclib(lib, srclib_dir, srclibpaths,
- preponly=onserver))
+ preponly=onserver))
for name, number, libpath in srclibpaths:
place_srclib(root_dir, int(number) if number else None, libpath)
srclibpaths.append(basesrclib)
# Update the local.properties file
- localprops = [ os.path.join(build_dir, 'local.properties') ]
- if 'subdir' in build:
- localprops += [ os.path.join(root_dir, 'local.properties') ]
+ localprops = [os.path.join(build_dir, 'local.properties')]
+ if build['subdir']:
+ localprops += [os.path.join(root_dir, 'local.properties')]
for path in localprops:
if not os.path.isfile(path):
continue
# from sdk.dir, if necessary
if build['oldsdkloc']:
sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
- re.S|re.M).group(1)
+ re.S | re.M).group(1)
props += "sdk-location=%s\n" % sdkloc
else:
props += "sdk.dir=%s\n" % config['sdk_path']
props += "ndk.dir=%s\n" % config['ndk_path']
props += "ndk-location=%s\n" % config['ndk_path']
# Add java.encoding if necessary
- if 'encoding' in build:
+ if build['encoding']:
props += "java.encoding=%s\n" % build['encoding']
f = open(path, 'w')
f.write(props)
if flavour in ['main', 'yes', '']:
flavour = None
- if 'target' in build:
+ version_regex = re.compile(r".*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
+ gradlepluginver = None
+
+ gradle_files = [os.path.join(root_dir, 'build.gradle')]
+
+ # Parent dir build.gradle
+ parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
+ if parent_dir.startswith(build_dir):
+ gradle_files.append(os.path.join(parent_dir, 'build.gradle'))
+
+ # Gradle execution dir build.gradle
+ if '@' in build['gradle']:
+ gradle_file = os.path.join(root_dir, build['gradle'].split('@', 1)[1], 'build.gradle')
+ gradle_file = os.path.normpath(gradle_file)
+ if gradle_file not in gradle_files:
+ gradle_files.append(gradle_file)
+
+ for path in gradle_files:
+ if gradlepluginver:
+ break
+ if not os.path.isfile(path):
+ continue
+ with open(path) as f:
+ for line in f:
+ match = version_regex.match(line)
+ if match:
+ gradlepluginver = match.group(1)
+ break
+
+ if gradlepluginver:
+ build['gradlepluginver'] = LooseVersion(gradlepluginver)
+ else:
+ logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
+ build['gradlepluginver'] = LooseVersion('0.11')
+
+ if build['target']:
n = build["target"].split('-')[1]
FDroidPopen(['sed', '-i',
- 's@compileSdkVersion[ ]*[0-9]*@compileSdkVersion '+n+'@g',
- 'build.gradle'], cwd=root_dir)
+ 's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g',
+ 'build.gradle'],
+ cwd=root_dir)
if '@' in build['gradle']:
- gradle_dir = os.path.join(root_dir, build['gradle'].split('@',1)[1])
+ gradle_dir = os.path.join(root_dir, build['gradle'].split('@', 1)[1])
gradle_dir = os.path.normpath(gradle_dir)
FDroidPopen(['sed', '-i',
- 's@compileSdkVersion[ ]*[0-9]*@compileSdkVersion '+n+'@g',
- 'build.gradle'], cwd=gradle_dir)
+ 's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g',
+ 'build.gradle'],
+ cwd=gradle_dir)
# Remove forced debuggable flags
remove_debuggable_flags(root_dir)
continue
if has_extension(path, 'xml'):
p = SilentPopen(['sed', '-i',
- 's/android:versionName="[^"]*"/android:versionName="' + build['version'] + '"/g',
- path])
+ 's/android:versionName="[^"]*"/android:versionName="'
+ + build['version'] + '"/g',
+ path])
if p.returncode != 0:
raise BuildException("Failed to amend manifest")
elif has_extension(path, 'gradle'):
p = SilentPopen(['sed', '-i',
- 's/versionName[ ]*=[ ]*"[^"]*"/versionName = "' + build['version'] + '"/g',
- path])
+ 's/versionName *=* *"[^"]*"/versionName = "'
+ + build['version'] + '"/g',
+ path])
if p.returncode != 0:
raise BuildException("Failed to amend build.gradle")
if build['forcevercode']:
continue
if has_extension(path, 'xml'):
p = SilentPopen(['sed', '-i',
- 's/android:versionCode="[^"]*"/android:versionCode="' + build['vercode'] + '"/g',
- path])
+ 's/android:versionCode="[^"]*"/android:versionCode="'
+ + build['vercode'] + '"/g',
+ path])
if p.returncode != 0:
raise BuildException("Failed to amend manifest")
elif has_extension(path, 'gradle'):
p = SilentPopen(['sed', '-i',
- 's/versionCode[ ]*=[ ]*[0-9]*/versionCode = ' + build['vercode'] + '/g',
- path])
+ 's/versionCode *=* *[0-9]*/versionCode = '
+ + build['vercode'] + '/g',
+ path])
if p.returncode != 0:
raise BuildException("Failed to amend build.gradle")
# Delete unwanted files
- if 'rm' in build:
- for part in build['rm']:
+ if build['rm']:
+ logging.info("Removing specified files")
+ for part in getpaths(build_dir, build, 'rm'):
dest = os.path.join(build_dir, part)
logging.info("Removing {0}".format(part))
if os.path.lexists(dest):
remove_signing_keys(build_dir)
# Add required external libraries
- if 'extlibs' in build:
+ if build['extlibs']:
logging.info("Collecting prebuilt libraries")
libsdir = os.path.join(root_dir, 'libs')
if not os.path.exists(libsdir):
shutil.copyfile(libsrc, os.path.join(libsdir, libf))
# Run a pre-build command if one is required
- if 'prebuild' in build:
+ if build['prebuild']:
+ logging.info("Running 'prebuild' commands in %s" % root_dir)
+
cmd = replace_config_vars(build['prebuild'])
# Substitute source library paths into prebuild commands
libpath = os.path.relpath(libpath, root_dir)
cmd = cmd.replace('$$' + name + '$$', libpath)
- logging.info("Running 'prebuild' commands in %s" % root_dir)
-
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if p.returncode != 0:
raise BuildException("Error running prebuild command for %s:%s" %
- (app['id'], build['version']), p.stdout)
+ (app['id'], build['version']), p.stdout)
- updatemode = build.get('update', ['auto'])
# Generate (or update) the ant build file, build.xml...
- if updatemode != ['no'] and build['type'] == 'ant':
+ if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
parms = [os.path.join(config['sdk_path'], 'tools', 'android'), 'update']
lparms = parms + ['lib-project']
parms = parms + ['project']
- if 'target' in build and build['target']:
+ if build['target']:
parms += ['-t', build['target']]
lparms += ['-t', build['target']]
- if updatemode == ['auto']:
+ if build['update'] == ['auto']:
update_dirs = ant_subprojects(root_dir) + ['.']
else:
- update_dirs = updatemode
+ update_dirs = build['update']
for d in update_dirs:
subdir = os.path.join(root_dir, d)
return (root_dir, srclibpaths)
+
+# Split and extend via globbing the paths from a field
+def getpaths(build_dir, build, field):
+ paths = []
+ for p in build[field]:
+ p = p.strip()
+ full_path = os.path.join(build_dir, p)
+ full_path = os.path.normpath(full_path)
+ paths += [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
+ return paths
+
+
# Scan the source code in the given directory (and all subdirectories)
-# and return a list of potential problems.
+# and return the number of fatal problems encountered
def scan_source(build_dir, root_dir, thisbuild):
- problems = []
+ count = 0
# Common known non-free blobs (always lower case):
- usual_suspects = ['flurryagent',
- 'paypal_mpl',
- 'libgoogleanalytics',
- 'admob-sdk-android',
- 'googleadview',
- 'googleadmobadssdk',
- 'google-play-services',
- 'crittercism',
- 'heyzap',
- 'jpct-ae',
- 'youtubeandroidplayerapi',
- 'bugsense',
- 'crashlytics',
- 'ouya-sdk']
-
- def getpaths(field):
- paths = []
- if field not in thisbuild:
- return paths
- for p in thisbuild[field]:
- p = p.strip()
- if p == '.':
- p = '/'
- elif p.startswith('./'):
- p = p[1:]
- elif not p.startswith('/'):
- p = '/' + p;
- if p not in paths:
- paths.append(p)
- return paths
-
- scanignore = getpaths('scanignore')
- scandelete = getpaths('scandelete')
+ usual_suspects = [
+ re.compile(r'flurryagent', re.IGNORECASE),
+ re.compile(r'paypal.*mpl', re.IGNORECASE),
+ re.compile(r'libgoogleanalytics', re.IGNORECASE),
+ re.compile(r'admob.*sdk.*android', re.IGNORECASE),
+ re.compile(r'googleadview', re.IGNORECASE),
+ re.compile(r'googleadmobadssdk', re.IGNORECASE),
+ re.compile(r'google.*play.*services', re.IGNORECASE),
+ re.compile(r'crittercism', re.IGNORECASE),
+ re.compile(r'heyzap', re.IGNORECASE),
+ re.compile(r'jpct.*ae', re.IGNORECASE),
+ re.compile(r'youtubeandroidplayerapi', re.IGNORECASE),
+ re.compile(r'bugsense', re.IGNORECASE),
+ re.compile(r'crashlytics', re.IGNORECASE),
+ re.compile(r'ouya.*sdk', re.IGNORECASE),
+ re.compile(r'libspen23', re.IGNORECASE),
+ ]
+
+ scanignore = getpaths(build_dir, thisbuild, 'scanignore')
+ scandelete = getpaths(build_dir, thisbuild, 'scandelete')
try:
ms = magic.open(magic.MIME_TYPE)
logging.info('Removing %s at %s' % (what, fd))
os.remove(fp)
+ def warnproblem(what, fd):
+ logging.warn('Found %s at %s' % (what, fd))
+
def handleproblem(what, fd, fp):
if todelete(fd):
removeproblem(what, fd, fp)
else:
- problems.append('Found %s at %s' % (what, fd))
-
- def warnproblem(what, fd, fp):
- logging.warn('Found %s at %s' % (what, fd))
+ logging.error('Found %s at %s' % (what, fd))
+ return True
+ return False
def insidedir(path, dirname):
return path.endswith('/%s' % dirname) or '/%s/' % dirname in path
# Iterate through all files in the source code
- for r,d,f in os.walk(build_dir):
+ for r, d, f in os.walk(build_dir):
- if any(insidedir(r, igndir) for igndir in ('.hg', '.git', '.svn')):
+ if any(insidedir(r, d) for d in ('.hg', '.git', '.svn', '.bzr')):
continue
for curfile in f:
# Path (relative) to the file
fp = os.path.join(r, curfile)
- fd = fp[len(build_dir):]
+ fd = fp[len(build_dir) + 1:]
# Check if this file has been explicitly excluded from scanning
if toignore(fd):
continue
- for suspect in usual_suspects:
- if suspect in curfile.lower():
- handleproblem('usual supect', fd, fp)
-
mime = magic.from_file(fp, mime=True) if ms is None else ms.file(fp)
+
if mime == 'application/x-sharedlib':
- handleproblem('shared library', fd, fp)
+ count += handleproblem('shared library', fd, fp)
+
elif mime == 'application/x-archive':
- handleproblem('static library', fd, fp)
+ count += handleproblem('static library', fd, fp)
+
elif mime == 'application/x-executable':
- handleproblem('binary executable', fd, fp)
+ count += handleproblem('binary executable', fd, fp)
+
elif mime == 'application/x-java-applet':
- handleproblem('Java compiled class', fd, fp)
- elif mime == 'application/jar' and has_extension(fp, 'apk'):
- removeproblem('APK file', fd, fp)
- elif has_extension(fp, 'jar') and mime in [
+ count += handleproblem('Java compiled class', fd, fp)
+
+ elif mime in (
+ 'application/jar',
'application/zip',
'application/java-archive',
+ 'application/octet-stream',
'binary',
- ]:
- warnproblem('JAR file', fd, fp)
- elif mime == 'application/zip':
- warnproblem('ZIP file', fd, fp)
+ ):
+
+ if has_extension(fp, 'apk'):
+ removeproblem('APK file', fd, fp)
+
+ elif has_extension(fp, 'jar'):
+
+ if any(suspect.match(curfile) for suspect in usual_suspects):
+ count += handleproblem('usual supect', fd, fp)
+ else:
+ warnproblem('JAR file', fd)
+
+ elif has_extension(fp, 'zip'):
+ warnproblem('ZIP file', fd)
+
+ else:
+ warnproblem('unknown compressed or binary file', fd)
elif has_extension(fp, 'java'):
for line in file(fp):
if 'DexClassLoader' in line:
- handleproblem('DexClassLoader', fd, fp)
+ count += handleproblem('DexClassLoader', fd, fp)
break
if ms is not None:
ms.close()
# indicate a problem (if it's not a problem, explicitly use
# buildjni=no to bypass this check)
if (os.path.exists(os.path.join(root_dir, 'jni')) and
- thisbuild.get('buildjni') is None):
- msg = 'Found jni directory, but buildjni is not enabled'
- problems.append(msg)
+ not thisbuild['buildjni']):
+ logging.error('Found jni directory, but buildjni is not enabled')
+ count += 1
- return problems
+ return count
class KnownApks:
self.path = os.path.join('stats', 'known_apks.txt')
self.apks = {}
if os.path.exists(self.path):
- for line in file( self.path):
+ for line in file(self.path):
t = line.rstrip().split(' ')
if len(t) == 2:
self.apks[t[0]] = (t[1], None)
# Record an apk (if it's new, otherwise does nothing)
# Returns the date it was added.
def recordapk(self, apk, app):
- if not apk in self.apks:
+ if apk not in self.apks:
self.apks[apk] = (app, time.gmtime(time.time()))
self.changed = True
_, added = self.apks[apk]
else:
apps[appid] = added
sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
- lst = [app for app,added in sortedapps]
+ lst = [app for app, _ in sortedapps]
lst.reverse()
return lst
+
def isApkDebuggable(apkfile, config):
"""Returns True if the given apk file is debuggable
:param apkfile: full path to the apk to check"""
- p = SilentPopen([os.path.join(config['sdk_path'],
- 'build-tools', config['build_tools'], 'aapt'),
- 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'])
+ p = SilentPopen([os.path.join(config['sdk_path'], 'build-tools',
+ config['build_tools'], 'aapt'),
+ 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'])
if p.returncode != 0:
logging.critical("Failed to get apk manifest information")
sys.exit(1)
'''Check whether there is no more content to expect.'''
return not self.is_alive() and self._queue.empty()
+
class PopenResult:
returncode = None
stdout = ''
-def SilentPopen(commands, cwd=None, shell=False):
- """
- Run a command silently and capture the output.
- :param commands: command and argument list like in subprocess.Popen
- :param cwd: optionally specifies a working directory
- :returns: A Popen object.
- """
-
- if cwd:
- cwd = os.path.normpath(cwd)
- logging.debug("Directory: %s" % cwd)
- logging.debug("> %s" % ' '.join(commands))
-
- result = PopenResult()
- p = subprocess.Popen(commands, cwd=cwd, shell=shell,
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+def SilentPopen(commands, cwd=None, shell=False):
+ return FDroidPopen(commands, cwd=cwd, shell=shell, output=False)
- result.stdout = p.communicate()[0]
- result.returncode = p.returncode
- return result
-def FDroidPopen(commands, cwd=None, shell=False):
+def FDroidPopen(commands, cwd=None, shell=False, output=True):
"""
Run a command and capture the possibly huge output.
:returns: A PopenResult.
"""
- if cwd:
- cwd = os.path.normpath(cwd)
- logging.info("Directory: %s" % cwd)
- logging.info("> %s" % ' '.join(commands))
+ if output:
+ if cwd:
+ cwd = os.path.normpath(cwd)
+ logging.info("Directory: %s" % cwd)
+ logging.info("> %s" % ' '.join(commands))
result = PopenResult()
p = subprocess.Popen(commands, cwd=cwd, shell=shell,
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
stdout_queue = Queue.Queue()
stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
while not stdout_reader.eof():
while not stdout_queue.empty():
line = stdout_queue.get()
- if options.verbose:
+ if output:
# Output directly to console
sys.stdout.write(line)
sys.stdout.flush()
result.returncode = p.returncode
return result
+
def remove_signing_keys(build_dir):
comment = re.compile(r'[ ]*//')
signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
line_matches = [
- re.compile(r'^[\t ]*signingConfig [^ ]*$'),
- re.compile(r'android.signingConfigs.'),
- re.compile(r'variant.outputFile = '),
- re.compile(r'.readLine\('),
- ]
+ re.compile(r'^[\t ]*signingConfig [^ ]*$'),
+ re.compile(r'.*android\.signingConfigs\.[^{]*$'),
+ re.compile(r'.*variant\.outputFile = .*'),
+ re.compile(r'.*\.readLine\(.*'),
+ ]
for root, dirs, files in os.walk(build_dir):
if 'build.gradle' in files:
path = os.path.join(root, 'build.gradle')
with open(path, "r") as o:
lines = o.readlines()
+ changed = False
+
opened = 0
with open(path, "w") as o:
for line in lines:
continue
if signing_configs.match(line):
+ changed = True
opened += 1
continue
if any(s.match(line) for s in line_matches):
+ changed = True
continue
if opened == 0:
o.write(line)
- logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
+ if changed:
+ logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
for propfile in [
'project.properties',
with open(path, "r") as o:
lines = o.readlines()
+ changed = False
+
with open(path, "w") as o:
for line in lines:
- if line.startswith('key.store'):
- continue
- if line.startswith('key.alias'):
+ if any(line.startswith(s) for s in ('key.store', 'key.alias')):
+ changed = True
continue
+
o.write(line)
- logging.info("Cleaned %s of keysigning configs at %s" % (propfile,path))
+ if changed:
+ logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
+
def replace_config_vars(cmd):
cmd = cmd.replace('$$SDK$$', config['sdk_path'])
cmd = cmd.replace('$$MVN3$$', config['mvn3'])
return cmd
+
def place_srclib(root_dir, number, libpath):
if not number:
return
placed = False
for line in lines:
if line.startswith('android.library.reference.%d=' % number):
- o.write('android.library.reference.%d=%s\n' % (number,relpath))
+ o.write('android.library.reference.%d=%s\n' % (number, relpath))
placed = True
else:
o.write(line)
if not placed:
- o.write('android.library.reference.%d=%s\n' % (number,relpath))
-
+ o.write('android.library.reference.%d=%s\n' % (number, relpath))