import re
import shutil
import glob
+import requests
import stat
import subprocess
import time
import threading
import magic
import logging
+import hashlib
+import socket
+import xml.etree.ElementTree as XMLElementTree
+
from distutils.version import LooseVersion
from zipfile import ZipFile
import metadata
+XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
+
config = None
options = None
env = None
'sdk_path': "$ANDROID_HOME",
'ndk_paths': {
'r9b': None,
- 'r10d': "$ANDROID_NDK"
+ 'r10e': "$ANDROID_NDK"
},
- 'build_tools': "22.0.0",
+ 'build_tools': "22.0.1",
'ant': "ant",
'mvn3': "mvn",
'gradle': 'gradle',
'stats_to_carbon': False,
'repo_maxage': 0,
'build_server_always': False,
- 'keystore': os.path.join("$HOME", '.local', 'share', 'fdroidserver', 'keystore.jks'),
+ 'keystore': 'keystore.jks',
'smartcardoptions': [],
'char_limits': {
- 'Summary': 50,
- 'Description': 1500
+ 'Summary': 80,
+ 'Description': 4000
},
'keyaliases': {},
'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
def get_ndk_path(version):
if version is None:
- version = 'r10d' # latest
+ version = 'r10e' # latest
paths = config['ndk_paths']
if version not in paths:
return ''
else:
self.checkrepo()
# Discard any working tree changes
- p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
+ p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
+ 'git', 'reset', '--hard'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git reset failed", p.output)
# Remove untracked files now, in case they're tracked in the target
# revision (it happens!)
- p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
+ p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
+ 'git', 'clean', '-dffx'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git clean failed", p.output)
if not self.refreshed:
line = line.replace('git@github.com:', 'https://github.com/')
f.write(line)
- for cmd in [
- ['git', 'reset', '--hard'],
- ['git', 'clean', '-dffx'],
- ]:
- p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive'] + cmd, cwd=self.local, output=False)
- if p.returncode != 0:
- raise VCSException("Git submodule reset failed", p.output)
p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git submodule sync failed", p.output)
p.output.splitlines()]
-def retrieve_string(app_dir, string, xmlfiles=None):
+def unescape_string(string):
+ if string[0] == '"' and string[-1] == '"':
+ return string[1:-1]
+
+ return string.replace("\\'", "'")
- res_dirs = [
- os.path.join(app_dir, 'res'),
- os.path.join(app_dir, 'src', 'main'),
- ]
+
+def retrieve_string(app_dir, string, xmlfiles=None):
if xmlfiles is None:
xmlfiles = []
- for res_dir in res_dirs:
+ for res_dir in [
+ os.path.join(app_dir, 'res'),
+ os.path.join(app_dir, 'src', 'main', 'res'),
+ ]:
for r, d, f in os.walk(res_dir):
if os.path.basename(r) == 'values':
xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
- string_search = None
- if string.startswith('@string/'):
- string_search = re.compile(r'.*name="' + string[8:] + '".*?>"?([^<]+?)"?<.*').search
- elif string.startswith('&') and string.endswith(';'):
- string_search = re.compile(r'.*<!ENTITY.*' + string[1:-1] + '.*?"([^"]+?)".*>').search
-
- if string_search is not None:
- for xmlfile in xmlfiles:
- for line in file(xmlfile):
- matches = string_search(line)
- if matches:
- return retrieve_string(app_dir, matches.group(1), xmlfiles)
- return None
+ if not string.startswith('@string/'):
+ return unescape_string(string)
- return string.replace("\\'", "'")
+ name = string[len('@string/'):]
+
+ for path in xmlfiles:
+ if not os.path.isfile(path):
+ continue
+ xml = parse_xml(path)
+ element = xml.find('string[@name="' + name + '"]')
+ if element is not None and element.text is not None:
+ return retrieve_string(app_dir, element.text.encode('utf-8'), xmlfiles)
+
+ return ''
+
+
+def retrieve_string_singleline(app_dir, string, xmlfiles=None):
+ return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
# Return list of existing files that will be used to find the highest vercode
# Retrieve the package name. Returns the name, or None if not found.
def fetch_real_name(app_dir, flavours):
- app_search = re.compile(r'.*<application.*').search
- name_search = re.compile(r'.*android:label="([^"]+)".*').search
- app_found = False
- for f in manifest_paths(app_dir, flavours):
- if not has_extension(f, 'xml'):
+ for path in manifest_paths(app_dir, flavours):
+ if not has_extension(path, 'xml') or not os.path.isfile(path):
continue
- logging.debug("fetch_real_name: Checking manifest at " + f)
- for line in file(f):
- if not app_found:
- if app_search(line):
- app_found = True
- if app_found:
- matches = name_search(line)
- if matches:
- stringname = matches.group(1)
- logging.debug("fetch_real_name: using string " + stringname)
- result = retrieve_string(app_dir, stringname)
- if result:
- result = result.strip()
- return result
- return None
-
-
-# Retrieve the version name
-def version_name(original, app_dir, flavours):
- for f in manifest_paths(app_dir, flavours):
- if not has_extension(f, 'xml'):
+ logging.debug("fetch_real_name: Checking manifest at " + path)
+ xml = parse_xml(path)
+ app = xml.find('application')
+ if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
continue
- string = retrieve_string(app_dir, original)
- if string:
- return string
- return original
+ label = app.attrib["{http://schemas.android.com/apk/res/android}label"].encode('utf-8')
+ result = retrieve_string_singleline(app_dir, label)
+ if result:
+ result = result.strip()
+ return result
+ return None
def get_library_references(root_dir):
proppath = os.path.join(root_dir, 'project.properties')
if not os.path.isfile(proppath):
return libraries
- with open(proppath) as f:
- for line in f.readlines():
- if not line.startswith('android.library.reference.'):
- continue
- path = line.split('=')[1].strip()
- relpath = os.path.join(root_dir, path)
- if not os.path.isdir(relpath):
- continue
- logging.debug("Found subproject at %s" % path)
- libraries.append(path)
+ for line in file(proppath):
+ if not line.startswith('android.library.reference.'):
+ continue
+ path = line.split('=')[1].strip()
+ relpath = os.path.join(root_dir, path)
+ if not os.path.isdir(relpath):
+ continue
+ logging.debug("Found subproject at %s" % path)
+ libraries.append(path)
return libraries
if not paths:
return (None, None, None)
- vcsearch = re.compile(r'.*:versionCode="([0-9]+?)".*').search
- vnsearch = re.compile(r'.*:versionName="([^"]+?)".*').search
- psearch = re.compile(r'.*package="([^"]+)".*').search
-
vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
psearch_g = re.compile(r'.*packageName *=* *["\']([^"]+)["\'].*').search
for path in paths:
+ if not os.path.isfile(path):
+ continue
+
logging.debug("Parsing manifest at {0}".format(path))
gradle = has_extension(path, 'gradle')
version = None
# Remember package name, may be defined separately from version+vercode
package = max_package
- for line in file(path):
- if not package:
- if gradle:
+ if gradle:
+ for line in file(path):
+ if not package:
matches = psearch_g(line)
- else:
- matches = psearch(line)
- if matches:
- package = matches.group(1)
- if not version:
- if gradle:
+ if matches:
+ package = matches.group(1)
+ if not version:
matches = vnsearch_g(line)
- else:
- matches = vnsearch(line)
- if matches:
- version = matches.group(2 if gradle else 1)
- if not vercode:
- if gradle:
+ if matches:
+ version = matches.group(2)
+ if not vercode:
matches = vcsearch_g(line)
- else:
- matches = vcsearch(line)
- if matches:
- vercode = matches.group(1)
+ if matches:
+ vercode = matches.group(1)
+ else:
+ xml = parse_xml(path)
+ if "package" in xml.attrib:
+ package = xml.attrib["package"].encode('utf-8')
+ if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
+ version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"].encode('utf-8')
+ base_dir = os.path.dirname(path)
+ version = retrieve_string_singleline(base_dir, version)
+ if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
+ a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"].encode('utf-8')
+ if string_is_integer(a):
+ vercode = a
logging.debug("..got package={0}, version={1}, vercode={2}"
.format(package, version, vercode))
# Returns the path to it. Normally this is the path to be used when referencing
# it, which may be a subdirectory of the actual project. If you want the base
# directory of the project, pass 'basepath=True'.
-def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
- basepath=False, raw=False, prepare=True, preponly=False):
+def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
+ raw=False, prepare=True, preponly=False):
number = None
subdir = None
if libdir is None:
libdir = sdir
- if srclib["Srclibs"]:
- n = 1
- for lib in srclib["Srclibs"].replace(';', ',').split(','):
- s_tuple = None
- for t in srclibpaths:
- if t[0] == lib:
- s_tuple = t
- break
- if s_tuple is None:
- raise VCSException('Missing recursive srclib %s for %s' % (
- lib, name))
- place_srclib(libdir, n, s_tuple[2])
- n += 1
-
remove_signing_keys(sdir)
remove_debuggable_flags(sdir)
if prepare:
if srclib["Prepare"]:
- cmd = replace_config_vars(srclib["Prepare"])
+ cmd = replace_config_vars(srclib["Prepare"], None)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
if p.returncode != 0:
logging.info("Getting source for revision " + build['commit'])
vcs.gotorevision(build['commit'])
- # Initialise submodules if requred
+ # Initialise submodules if required
if build['submodules']:
logging.info("Initialising submodules")
vcs.initsubmodules()
# Run an init command if one is required
if build['init']:
- cmd = replace_config_vars(build['init'])
+ cmd = replace_config_vars(build['init'], build)
logging.info("Running 'init' commands in %s" % root_dir)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if build['srclibs']:
logging.info("Collecting source libraries")
for lib in build['srclibs']:
- srclibpaths.append(getsrclib(lib, srclib_dir, build, srclibpaths,
- preponly=onserver))
+ srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver))
for name, number, libpath in srclibpaths:
place_srclib(root_dir, int(number) if number else None, libpath)
if build['type'] == 'gradle':
flavours = build['gradle']
- version_regex = re.compile(r".*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
+ version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
gradlepluginver = None
- gradle_files = [os.path.join(root_dir, 'build.gradle')]
+ gradle_dirs = [root_dir]
# Parent dir build.gradle
parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
if parent_dir.startswith(build_dir):
- gradle_files.append(os.path.join(parent_dir, 'build.gradle'))
+ gradle_dirs.append(parent_dir)
- for path in gradle_files:
+ for dir_path in gradle_dirs:
if gradlepluginver:
break
- if not os.path.isfile(path):
+ if not os.path.isdir(dir_path):
continue
- with open(path) as f:
- for line in f:
+ for filename in os.listdir(dir_path):
+ if not filename.endswith('.gradle'):
+ continue
+ path = os.path.join(dir_path, filename)
+ if not os.path.isfile(path):
+ continue
+ for line in file(path):
match = version_regex.match(line)
if match:
gradlepluginver = match.group(1)
if build['prebuild']:
logging.info("Running 'prebuild' commands in %s" % root_dir)
- cmd = replace_config_vars(build['prebuild'])
+ cmd = replace_config_vars(build['prebuild'], build)
# Substitute source library paths into prebuild commands
for name, number, libpath in srclibpaths:
# Common known non-free blobs (always lower case):
usual_suspects = [
- re.compile(r'flurryagent', re.IGNORECASE),
- re.compile(r'paypal.*mpl', re.IGNORECASE),
- re.compile(r'google.*analytics', re.IGNORECASE),
- re.compile(r'admob.*sdk.*android', re.IGNORECASE),
- re.compile(r'google.*ad.*view', re.IGNORECASE),
- re.compile(r'google.*admob', re.IGNORECASE),
- re.compile(r'google.*play.*services', re.IGNORECASE),
- re.compile(r'crittercism', re.IGNORECASE),
- re.compile(r'heyzap', re.IGNORECASE),
- re.compile(r'jpct.*ae', re.IGNORECASE),
- re.compile(r'youtube.*android.*player.*api', re.IGNORECASE),
- re.compile(r'bugsense', re.IGNORECASE),
- re.compile(r'crashlytics', re.IGNORECASE),
- re.compile(r'ouya.*sdk', re.IGNORECASE),
- re.compile(r'libspen23', re.IGNORECASE),
+ re.compile(r'.*flurryagent', re.IGNORECASE),
+ re.compile(r'.*paypal.*mpl', re.IGNORECASE),
+ re.compile(r'.*google.*analytics', re.IGNORECASE),
+ re.compile(r'.*admob.*sdk.*android', re.IGNORECASE),
+ re.compile(r'.*google.*ad.*view', re.IGNORECASE),
+ re.compile(r'.*google.*admob', re.IGNORECASE),
+ re.compile(r'.*google.*play.*services', re.IGNORECASE),
+ re.compile(r'.*crittercism', re.IGNORECASE),
+ re.compile(r'.*heyzap', re.IGNORECASE),
+ re.compile(r'.*jpct.*ae', re.IGNORECASE),
+ re.compile(r'.*youtube.*android.*player.*api', re.IGNORECASE),
+ re.compile(r'.*bugsense', re.IGNORECASE),
+ re.compile(r'.*crashlytics', re.IGNORECASE),
+ re.compile(r'.*ouya.*sdk', re.IGNORECASE),
+ re.compile(r'.*libspen23', re.IGNORECASE),
]
scanignore = getpaths(build_dir, thisbuild, 'scanignore')
warnproblem('unknown compressed or binary file', fd)
elif has_extension(fp, 'java'):
+ if not os.path.isfile(fp):
+ continue
for line in file(fp):
if 'DexClassLoader' in line:
count += handleproblem('DexClassLoader', fd, fp)
break
+
+ elif has_extension(fp, 'gradle'):
+ if not os.path.isfile(fp):
+ continue
+ for i, line in enumerate(file(fp)):
+ if any(suspect.match(line) for suspect in usual_suspects):
+ count += handleproblem('usual suspect at line %d' % i, fd, fp)
+ break
if ms is not None:
ms.close()
def __init__(self):
self.path = os.path.join('stats', 'known_apks.txt')
self.apks = {}
- if os.path.exists(self.path):
+ if os.path.isfile(self.path):
for line in file(self.path):
t = line.rstrip().split(' ')
if len(t) == 2:
env['PATH'] = os.pathsep.join(paths)
-def replace_config_vars(cmd):
+def replace_config_vars(cmd, build):
global env
cmd = cmd.replace('$$SDK$$', config['sdk_path'])
# env['ANDROID_NDK'] is set in build_local right before prepare_source
cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
cmd = cmd.replace('$$MVN3$$', config['mvn3'])
+ if build is not None:
+ cmd = cmd.replace('$$COMMIT$$', build['commit'])
+ cmd = cmd.replace('$$VERSION$$', build['version'])
+ cmd = cmd.replace('$$VERCODE$$', build['vercode'])
return cmd
return exe_file
return None
+
+
+def genpassword():
+ '''generate a random password for when generating keys'''
+ h = hashlib.sha256()
+ h.update(os.urandom(16)) # salt
+ h.update(bytes(socket.getfqdn()))
+ return h.digest().encode('base64').strip()
+
+
+def genkeystore(localconfig):
+ '''Generate a new key with random passwords and add it to new keystore'''
+ logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
+ keystoredir = os.path.dirname(localconfig['keystore'])
+ if keystoredir is None or keystoredir == '':
+ keystoredir = os.path.join(os.getcwd(), keystoredir)
+ if not os.path.exists(keystoredir):
+ os.makedirs(keystoredir, mode=0o700)
+
+ write_password_file("keystorepass", localconfig['keystorepass'])
+ write_password_file("keypass", localconfig['keypass'])
+ p = FDroidPopen(['keytool', '-genkey',
+ '-keystore', localconfig['keystore'],
+ '-alias', localconfig['repo_keyalias'],
+ '-keyalg', 'RSA', '-keysize', '4096',
+ '-sigalg', 'SHA256withRSA',
+ '-validity', '10000',
+ '-storepass:file', config['keystorepassfile'],
+ '-keypass:file', config['keypassfile'],
+ '-dname', localconfig['keydname']])
+ # TODO keypass should be sent via stdin
+ os.chmod(localconfig['keystore'], 0o0600)
+ if p.returncode != 0:
+ raise BuildException("Failed to generate key", p.output)
+ # now show the lovely key that was just generated
+ p = FDroidPopen(['keytool', '-list', '-v',
+ '-keystore', localconfig['keystore'],
+ '-alias', localconfig['repo_keyalias'],
+ '-storepass:file', config['keystorepassfile']])
+ logging.info(p.output.strip() + '\n\n')
+
+
+def write_to_config(thisconfig, key, value=None):
+ '''write a key/value to the local config.py'''
+ if value is None:
+ origkey = key + '_orig'
+ value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
+ with open('config.py', 'r') as f:
+ data = f.read()
+ pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
+ repl = '\n' + key + ' = "' + value + '"'
+ data = re.sub(pattern, repl, data)
+ # if this key is not in the file, append it
+ if not re.match('\s*' + key + '\s*=\s*"', data):
+ data += repl
+ # make sure the file ends with a carraige return
+ if not re.match('\n$', data):
+ data += '\n'
+ with open('config.py', 'w') as f:
+ f.writelines(data)
+
+
+def parse_xml(path):
+ return XMLElementTree.parse(path).getroot()
+
+
+def string_is_integer(string):
+ try:
+ int(string)
+ return True
+ except ValueError:
+ return False
+
+
+def download_file(url, local_filename=None, dldir='tmp'):
+ filename = url.split('/')[-1]
+ if local_filename is None:
+ local_filename = os.path.join(dldir, filename)
+ # the stream=True parameter keeps memory usage low
+ r = requests.get(url, stream=True)
+ with open(local_filename, 'wb') as f:
+ for chunk in r.iter_content(chunk_size=1024):
+ if chunk: # filter out keep-alive new chunks
+ f.write(chunk)
+ f.flush()
+ return local_filename
~ianmdlvl / fdroidserver.git / blobdiff