import re
import shutil
import glob
+import requests
import stat
import subprocess
import time
import operator
import Queue
import threading
-import magic
import logging
import hashlib
import socket
'sdk_path': "$ANDROID_HOME",
'ndk_paths': {
'r9b': None,
- 'r10d': "$ANDROID_NDK"
+ 'r10e': "$ANDROID_NDK"
},
- 'build_tools': "22.0.0",
+ 'build_tools': "22.0.1",
'ant': "ant",
'mvn3': "mvn",
'gradle': 'gradle',
'keystore': 'keystore.jks',
'smartcardoptions': [],
'char_limits': {
- 'Summary': 50,
- 'Description': 1500
+ 'Summary': 80,
+ 'Description': 4000
},
'keyaliases': {},
'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
thisconfig[k][k2 + '_orig'] = v
+def regsub_file(pattern, repl, path):
+ with open(path, 'r') as f:
+ text = f.read()
+ text = re.sub(pattern, repl, text)
+ with open(path, 'w') as f:
+ f.write(text)
+
+
def read_config(opts, config_file='config.py'):
"""Read the repository config
def get_ndk_path(version):
if version is None:
- version = 'r10d' # latest
+ version = 'r10e' # latest
paths = config['ndk_paths']
if version not in paths:
return ''
# lifetime of the vcs object.
# None is acceptable for 'rev' if you know you are cloning a clean copy of
# the repo - otherwise it must specify a valid revision.
- def gotorevision(self, rev):
+ def gotorevision(self, rev, refresh=True):
if self.clone_failed:
raise VCSException("Downloading the repository already failed once, not trying again.")
shutil.rmtree(self.local)
exc = None
+ if not refresh:
+ self.refreshed = True
try:
self.gotorevisionx(rev)
else:
self.checkrepo()
# Discard any working tree changes
- p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
+ p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
+ 'git', 'reset', '--hard'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git reset failed", p.output)
# Remove untracked files now, in case they're tracked in the target
# revision (it happens!)
- p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
+ p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
+ 'git', 'clean', '-dffx'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git clean failed", p.output)
if not self.refreshed:
line = line.replace('git@github.com:', 'https://github.com/')
f.write(line)
- for cmd in [
- ['git', 'reset', '--hard'],
- ['git', 'clean', '-dffx'],
- ]:
- p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive'] + cmd, cwd=self.local, output=False)
- if p.returncode != 0:
- raise VCSException("Git submodule reset failed", p.output)
p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git submodule sync failed", p.output)
p.output.splitlines()]
+def unescape_string(string):
+ if string[0] == '"' and string[-1] == '"':
+ return string[1:-1]
+
+ return string.replace("\\'", "'")
+
+
def retrieve_string(app_dir, string, xmlfiles=None):
if xmlfiles is None:
xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
if not string.startswith('@string/'):
- return string.replace("\\'", "'")
+ return unescape_string(string)
name = string[len('@string/'):]
continue
xml = parse_xml(path)
element = xml.find('string[@name="' + name + '"]')
- if element is not None:
+ if element is not None and element.text is not None:
return retrieve_string(app_dir, element.text.encode('utf-8'), xmlfiles)
return ''
+def retrieve_string_singleline(app_dir, string, xmlfiles=None):
+ return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
+
+
# Return list of existing files that will be used to find the highest vercode
def manifest_paths(app_dir, flavours):
if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
continue
label = app.attrib["{http://schemas.android.com/apk/res/android}label"].encode('utf-8')
- result = retrieve_string(app_dir, label)
+ result = retrieve_string_singleline(app_dir, label)
if result:
result = result.strip()
return result
return None
-# Retrieve the version name
-def version_name(original, app_dir, flavours):
- for path in manifest_paths(app_dir, flavours):
- if not has_extension(path, 'xml'):
- continue
- string = retrieve_string(app_dir, original)
- if string:
- return string
- return original
-
-
def get_library_references(root_dir):
libraries = []
proppath = os.path.join(root_dir, 'project.properties')
logging.debug("Removing debuggable flags from %s" % root_dir)
for root, dirs, files in os.walk(root_dir):
if 'AndroidManifest.xml' in files:
- path = os.path.join(root, 'AndroidManifest.xml')
- p = FDroidPopen(['sed', '-i', 's/android:debuggable="[^"]*"//g', path], output=False)
- if p.returncode != 0:
- raise BuildException("Failed to remove debuggable flags of %s" % path)
+ regsub_file(r'android:debuggable="[^"]*"',
+ '',
+ os.path.join(root, 'AndroidManifest.xml'))
# Extract some information from the AndroidManifest.xml at the given path.
package = xml.attrib["package"].encode('utf-8')
if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"].encode('utf-8')
+ base_dir = os.path.dirname(path)
+ version = retrieve_string_singleline(base_dir, version)
if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
- vercode = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"].encode('utf-8')
+ a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"].encode('utf-8')
+ if string_is_integer(a):
+ vercode = a
logging.debug("..got package={0}, version={1}, vercode={2}"
.format(package, version, vercode))
# it, which may be a subdirectory of the actual project. If you want the base
# directory of the project, pass 'basepath=True'.
def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
- raw=False, prepare=True, preponly=False):
+ raw=False, prepare=True, preponly=False, refresh=True):
number = None
subdir = None
vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
vcs.srclib = (name, number, sdir)
if ref:
- vcs.gotorevision(ref)
+ vcs.gotorevision(ref, refresh)
if raw:
return vcs
# 'root' is the root directory, which may be the same as 'build_dir' or may
# be a subdirectory of it.
# 'srclibpaths' is information on the srclibs being used
-def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False):
+def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
# Optionally, the actual app source can be in a subdirectory
if build['subdir']:
# Get a working copy of the right revision
logging.info("Getting source for revision " + build['commit'])
- vcs.gotorevision(build['commit'])
+ vcs.gotorevision(build['commit'], refresh)
- # Initialise submodules if requred
+ # Initialise submodules if required
if build['submodules']:
logging.info("Initialising submodules")
vcs.initsubmodules()
if build['srclibs']:
logging.info("Collecting source libraries")
for lib in build['srclibs']:
- srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver))
+ srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
for name, number, libpath in srclibpaths:
place_srclib(root_dir, int(number) if number else None, libpath)
if build['target']:
n = build["target"].split('-')[1]
- FDroidPopen(['sed', '-i',
- 's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g',
- 'build.gradle'], cwd=root_dir, output=False)
+ regsub_file(r'compileSdkVersion[ =]+[0-9]+',
+ r'compileSdkVersion %s' % n,
+ os.path.join(root_dir, 'build.gradle'))
# Remove forced debuggable flags
remove_debuggable_flags(root_dir)
if not os.path.isfile(path):
continue
if has_extension(path, 'xml'):
- p = FDroidPopen(['sed', '-i',
- 's/android:versionName="[^"]*"/android:versionName="' + build['version'] + '"/g',
- path], output=False)
- if p.returncode != 0:
- raise BuildException("Failed to amend manifest")
+ regsub_file(r'android:versionName="[^"]*"',
+ r'android:versionName="%s"' % build['version'],
+ path)
elif has_extension(path, 'gradle'):
- p = FDroidPopen(['sed', '-i',
- 's/versionName *=* *.*/versionName = "' + build['version'] + '"/g',
- path], output=False)
- if p.returncode != 0:
- raise BuildException("Failed to amend build.gradle")
+ regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
+ r"""\1versionName '%s'""" % build['version'],
+ path)
+
if build['forcevercode']:
logging.info("Changing the version code")
for path in manifest_paths(root_dir, flavours):
if not os.path.isfile(path):
continue
if has_extension(path, 'xml'):
- p = FDroidPopen(['sed', '-i',
- 's/android:versionCode="[^"]*"/android:versionCode="' + build['vercode'] + '"/g',
- path], output=False)
- if p.returncode != 0:
- raise BuildException("Failed to amend manifest")
+ regsub_file(r'android:versionCode="[^"]*"',
+ r'android:versionCode="%s"' % build['vercode'],
+ path)
elif has_extension(path, 'gradle'):
- p = FDroidPopen(['sed', '-i',
- 's/versionCode *=* *[0-9]*/versionCode = ' + build['vercode'] + '/g',
- path], output=False)
- if p.returncode != 0:
- raise BuildException("Failed to amend build.gradle")
+ regsub_file(r'versionCode[ =]+[0-9]+',
+ r'versionCode %s' % build['vercode'],
+ path)
# Delete unwanted files
if build['rm']:
return paths
+def get_mime_type(path):
+ '''
+ There are two incompatible versions of the 'magic' module, one
+ that comes as part of libmagic, which is what Debian includes as
+ python-magic, then another called python-magic that is a separate
+ project that wraps libmagic. The second is 'magic' on pypi, so
+ both need to be supported. Then on platforms where libmagic is
+ not easily included, e.g. OSX and Windows, fallback to the
+ built-in 'mimetypes' module so this will work without
+ libmagic. Hence this function with the following hacks:
+ '''
+
+ try:
+ import magic
+ ms = None
+ try:
+ ms = magic.open(magic.MIME_TYPE)
+ ms.load()
+ return magic.from_file(path, mime=True)
+ except AttributeError:
+ return ms.file(path)
+ if ms is not None:
+ ms.close()
+ except UnicodeError:
+ logging.warn('Found malformed magic number at %s' % path)
+ except ImportError:
+ import mimetypes
+ mimetypes.init()
+ return mimetypes.guess_type(path, strict=False)
+
+
# Scan the source code in the given directory (and all subdirectories)
# and return the number of fatal problems encountered
def scan_source(build_dir, root_dir, thisbuild):
# Common known non-free blobs (always lower case):
usual_suspects = [
- re.compile(r'flurryagent', re.IGNORECASE),
- re.compile(r'paypal.*mpl', re.IGNORECASE),
- re.compile(r'google.*analytics', re.IGNORECASE),
- re.compile(r'admob.*sdk.*android', re.IGNORECASE),
- re.compile(r'google.*ad.*view', re.IGNORECASE),
- re.compile(r'google.*admob', re.IGNORECASE),
- re.compile(r'google.*play.*services', re.IGNORECASE),
- re.compile(r'crittercism', re.IGNORECASE),
- re.compile(r'heyzap', re.IGNORECASE),
- re.compile(r'jpct.*ae', re.IGNORECASE),
- re.compile(r'youtube.*android.*player.*api', re.IGNORECASE),
- re.compile(r'bugsense', re.IGNORECASE),
- re.compile(r'crashlytics', re.IGNORECASE),
- re.compile(r'ouya.*sdk', re.IGNORECASE),
- re.compile(r'libspen23', re.IGNORECASE),
+ re.compile(r'.*flurryagent', re.IGNORECASE),
+ re.compile(r'.*paypal.*mpl', re.IGNORECASE),
+ re.compile(r'.*google.*analytics', re.IGNORECASE),
+ re.compile(r'.*admob.*sdk.*android', re.IGNORECASE),
+ re.compile(r'.*google.*ad.*view', re.IGNORECASE),
+ re.compile(r'.*google.*admob', re.IGNORECASE),
+ re.compile(r'.*google.*play.*services', re.IGNORECASE),
+ re.compile(r'.*crittercism', re.IGNORECASE),
+ re.compile(r'.*heyzap', re.IGNORECASE),
+ re.compile(r'.*jpct.*ae', re.IGNORECASE),
+ re.compile(r'.*youtube.*android.*player.*api', re.IGNORECASE),
+ re.compile(r'.*bugsense', re.IGNORECASE),
+ re.compile(r'.*crashlytics', re.IGNORECASE),
+ re.compile(r'.*ouya.*sdk', re.IGNORECASE),
+ re.compile(r'.*libspen23', re.IGNORECASE),
]
scanignore = getpaths(build_dir, thisbuild, 'scanignore')
scanignore_worked = set()
scandelete_worked = set()
- try:
- ms = magic.open(magic.MIME_TYPE)
- ms.load()
- except AttributeError:
- ms = None
-
def toignore(fd):
for p in scanignore:
if fd.startswith(p):
fp = os.path.join(r, curfile)
fd = fp[len(build_dir) + 1:]
- try:
- mime = magic.from_file(fp, mime=True) if ms is None else ms.file(fp)
- except UnicodeError:
- warnproblem('malformed magic number', fd)
+ mime = get_mime_type(fp)
if mime == 'application/x-sharedlib':
count += handleproblem('shared library', fd, fp)
elif mime == 'application/x-archive':
count += handleproblem('static library', fd, fp)
- elif mime == 'application/x-executable':
+ elif mime == 'application/x-executable' or mime == 'application/x-mach-binary':
count += handleproblem('binary executable', fd, fp)
elif mime == 'application/x-java-applet':
else:
warnproblem('unknown compressed or binary file', fd)
- elif has_extension(fp, 'java') and os.path.isfile(fp):
+ elif has_extension(fp, 'java'):
if not os.path.isfile(fp):
continue
for line in file(fp):
if 'DexClassLoader' in line:
count += handleproblem('DexClassLoader', fd, fp)
break
- if ms is not None:
- ms.close()
+
+ elif has_extension(fp, 'gradle'):
+ if not os.path.isfile(fp):
+ continue
+ for i, line in enumerate(file(fp)):
+ if any(suspect.match(line) for suspect in usual_suspects):
+ count += handleproblem('usual suspect at line %d' % i, fd, fp)
+ break
for p in scanignore:
if p not in scanignore_worked:
def parse_xml(path):
return XMLElementTree.parse(path).getroot()
+
+
+def string_is_integer(string):
+ try:
+ int(string)
+ return True
+ except ValueError:
+ return False
+
+
+def download_file(url, local_filename=None, dldir='tmp'):
+ filename = url.split('/')[-1]
+ if local_filename is None:
+ local_filename = os.path.join(dldir, filename)
+ # the stream=True parameter keeps memory usage low
+ r = requests.get(url, stream=True)
+ with open(local_filename, 'wb') as f:
+ for chunk in r.iter_content(chunk_size=1024):
+ if chunk: # filter out keep-alive new chunks
+ f.write(chunk)
+ f.flush()
+ return local_filename
~ianmdlvl / fdroidserver.git / blobdiff