# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# common.py is imported by all modules, so do not import third-party
+# libraries here as they will become a requirement for all commands.
+
import os
import sys
import re
import subprocess
import time
import operator
-import Queue
-import threading
-import magic
import logging
import hashlib
import socket
import xml.etree.ElementTree as XMLElementTree
-from distutils.version import LooseVersion
+try:
+ # Python 2
+ from Queue import Queue
+except ImportError:
+ # Python 3
+ from queue import Queue
+
from zipfile import ZipFile
import metadata
+from fdroidserver.asynchronousfilereader import AsynchronousFileReader
+
XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
'sdk_path': "$ANDROID_HOME",
'ndk_paths': {
'r9b': None,
- 'r10d': "$ANDROID_NDK"
+ 'r10e': "$ANDROID_NDK",
+ },
+ 'build_tools': "23.0.2",
+ 'java_paths': {
+ '1.7': "/usr/lib/jvm/java-7-openjdk",
+ '1.8': None,
},
- 'build_tools': "22.0.0",
'ant': "ant",
'mvn3': "mvn",
'gradle': 'gradle',
+ 'accepted_formats': ['txt', 'yaml'],
'sync_from_local_copy_dir': False,
+ 'per_app_repos': False,
'make_current_version_link': True,
'current_version_name_source': 'Name',
'update_stats': False,
'keystore': 'keystore.jks',
'smartcardoptions': [],
'char_limits': {
- 'Summary': 50,
- 'Description': 1500
+ 'Summary': 80,
+ 'Description': 4000,
},
'keyaliases': {},
'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
}
+def setup_global_opts(parser):
+ parser.add_argument("-v", "--verbose", action="store_true", default=False,
+ help="Spew out even more information than normal")
+ parser.add_argument("-q", "--quiet", action="store_true", default=False,
+ help="Restrict output to warnings and errors")
+
+
def fill_config_defaults(thisconfig):
for k, v in default_config.items():
if k not in thisconfig:
thisconfig[k] = exp
thisconfig[k + '_orig'] = v
- for k in ['ndk_paths']:
+ for k in ['ndk_paths', 'java_paths']:
d = thisconfig[k]
for k2 in d.copy():
v = d[k2]
thisconfig[k][k2 + '_orig'] = v
+def regsub_file(pattern, repl, path):
+ with open(path, 'r') as f:
+ text = f.read()
+ text = re.sub(pattern, repl, text)
+ with open(path, 'w') as f:
+ f.write(text)
+
+
def read_config(opts, config_file='config.py'):
"""Read the repository config
for n in ['ANDROID_HOME', 'ANDROID_SDK']:
env[n] = config['sdk_path']
+ for v in ['7', '8']:
+ cpath = config['java_paths']['1.%s' % v]
+ if cpath:
+ env['JAVA%s_HOME' % v] = cpath
+
for k in ["keystorepass", "keypass"]:
if k in config:
write_password_file(k)
return config
-def get_ndk_path(version):
- if version is None:
- version = 'r10d' # latest
- paths = config['ndk_paths']
- if version not in paths:
- return ''
- return paths[version] or ''
-
-
def find_sdk_tools_cmd(cmd):
'''find a working path to a tool from the Android SDK'''
command line argments
'''
filename = '.fdroid.' + pwtype + '.txt'
- fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
+ fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
if password is None:
os.write(fd, config[pwtype])
else:
vc = vercodes[appid]
if not vc:
continue
- app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
- if len(app['builds']) != len(vercodes[appid]):
+ app.builds = [b for b in app.builds if b.vercode in vc]
+ if len(app.builds) != len(vercodes[appid]):
error = True
- allvcs = [b['vercode'] for b in app['builds']]
+ allvcs = [b.vercode for b in app.builds]
for v in vercodes[appid]:
if v not in allvcs:
logging.critical("No such vercode %s for app %s" % (v, appid))
return apps
-def has_extension(filename, extension):
- name, ext = os.path.splitext(filename)
- ext = ext.lower()[1:]
- return ext == extension
+def get_extension(filename):
+ base, ext = os.path.splitext(filename)
+ if not ext:
+ return base, ''
+ return base, ext.lower()[1:]
+
+
+def has_extension(filename, ext):
+ _, f_ext = get_extension(filename)
+ return ext == f_ext
-apk_regex = None
+
+apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
def clean_description(description):
def apknameinfo(filename):
- global apk_regex
filename = os.path.basename(filename)
- if apk_regex is None:
- apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
m = apk_regex.match(filename)
try:
result = (m.group(1), m.group(2))
def getapkname(app, build):
- return "%s_%s.apk" % (app['id'], build['vercode'])
+ return "%s_%s.apk" % (app.id, build.vercode)
def getsrcname(app, build):
- return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
+ return "%s_%s_src.tar.gz" % (app.id, build.vercode)
def getappname(app):
- if app['Name']:
- return app['Name']
- if app['Auto Name']:
- return app['Auto Name']
- return app['id']
+ if app.Name:
+ return app.Name
+ if app.AutoName:
+ return app.AutoName
+ return app.id
def getcvname(app):
- return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
+ return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
def getvcs(vcstype, remote, local):
# lifetime of the vcs object.
# None is acceptable for 'rev' if you know you are cloning a clean copy of
# the repo - otherwise it must specify a valid revision.
- def gotorevision(self, rev):
+ def gotorevision(self, rev, refresh=True):
if self.clone_failed:
raise VCSException("Downloading the repository already failed once, not trying again.")
shutil.rmtree(self.local)
exc = None
+ if not refresh:
+ self.refreshed = True
try:
self.gotorevisionx(rev)
- except FDroidException, e:
+ except FDroidException as e:
exc = e
# If necessary, write the .fdroidvcs file.
raise VCSException('gettags not supported for this vcs type')
rtags = []
for tag in self._gettags():
- if re.match('[-A-Za-z0-9_. ]+$', tag):
+ if re.match('[-A-Za-z0-9_. /]+$', tag):
rtags.append(tag)
return rtags
else:
self.checkrepo()
# Discard any working tree changes
- p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
+ p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
+ 'git', 'reset', '--hard'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git reset failed", p.output)
# Remove untracked files now, in case they're tracked in the target
# revision (it happens!)
- p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
+ p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
+ 'git', 'clean', '-dffx'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git clean failed", p.output)
if not self.refreshed:
for line in lines:
if 'git@github.com' in line:
line = line.replace('git@github.com:', 'https://github.com/')
+ if 'git@gitlab.com' in line:
+ line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
f.write(line)
- for cmd in [
- ['git', 'reset', '--hard'],
- ['git', 'clean', '-dffx'],
- ]:
- p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive'] + cmd, cwd=self.local, output=False)
- if p.returncode != 0:
- raise VCSException("Git submodule reset failed", p.output)
p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException("Git submodule sync failed", p.output)
p.output.splitlines()]
+def unescape_string(string):
+ if len(string) < 2:
+ return string
+ if string[0] == '"' and string[-1] == '"':
+ return string[1:-1]
+
+ return string.replace("\\'", "'")
+
+
def retrieve_string(app_dir, string, xmlfiles=None):
+ if not string.startswith('@string/'):
+ return unescape_string(string)
+
if xmlfiles is None:
xmlfiles = []
for res_dir in [
if os.path.basename(r) == 'values':
xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
- if not string.startswith('@string/'):
- return string.replace("\\'", "'")
-
name = string[len('@string/'):]
+ def element_content(element):
+ if element.text is None:
+ return ""
+ s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
+ return s.strip()
+
for path in xmlfiles:
if not os.path.isfile(path):
continue
xml = parse_xml(path)
- element = xml.find('string[@name="'+name+'"]')
+ element = xml.find('string[@name="' + name + '"]')
if element is not None:
- return retrieve_string(app_dir, element.text, xmlfiles)
+ content = element_content(element)
+ return retrieve_string(app_dir, content, xmlfiles)
return ''
+def retrieve_string_singleline(app_dir, string, xmlfiles=None):
+ return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
+
+
# Return list of existing files that will be used to find the highest vercode
def manifest_paths(app_dir, flavours):
logging.debug("fetch_real_name: Checking manifest at " + path)
xml = parse_xml(path)
app = xml.find('application')
- label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
- result = retrieve_string(app_dir, label)
+ if app is None:
+ continue
+ if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
+ continue
+ label = app.attrib["{http://schemas.android.com/apk/res/android}label"].encode('utf-8')
+ result = retrieve_string_singleline(app_dir, label)
if result:
result = result.strip()
return result
return None
-# Retrieve the version name
-def version_name(original, app_dir, flavours):
- for path in manifest_paths(app_dir, flavours):
- if not has_extension(path, 'xml'):
- continue
- string = retrieve_string(app_dir, original)
- if string:
- return string
- return original
-
-
def get_library_references(root_dir):
libraries = []
proppath = os.path.join(root_dir, 'project.properties')
logging.debug("Removing debuggable flags from %s" % root_dir)
for root, dirs, files in os.walk(root_dir):
if 'AndroidManifest.xml' in files:
- path = os.path.join(root, 'AndroidManifest.xml')
- p = FDroidPopen(['sed', '-i', 's/android:debuggable="[^"]*"//g', path], output=False)
- if p.returncode != 0:
- raise BuildException("Failed to remove debuggable flags of %s" % path)
+ regsub_file(r'android:debuggable="[^"]*"',
+ '',
+ os.path.join(root, 'AndroidManifest.xml'))
+
+
+vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
+vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
+psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
+
+
+def app_matches_packagename(app, package):
+ if not package:
+ return False
+ appid = app.UpdateCheckName or app.id
+ if appid is None or appid == "Ignore":
+ return True
+ return appid == package
# Extract some information from the AndroidManifest.xml at the given path.
# Returns (version, vercode, package), any or all of which might be None.
# All values returned are strings.
-def parse_androidmanifests(paths, ignoreversions=None):
+def parse_androidmanifests(paths, app):
+
+ ignoreversions = app.UpdateCheckIgnore
+ ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
if not paths:
return (None, None, None)
- vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
- vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
- psearch_g = re.compile(r'.*packageName *=* *["\']([^"]+)["\'].*').search
-
- ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
-
max_version = None
max_vercode = None
max_package = None
gradle = has_extension(path, 'gradle')
version = None
vercode = None
- # Remember package name, may be defined separately from version+vercode
- package = max_package
+ package = None
if gradle:
for line in file(path):
+ if gradle_comment.match(line):
+ continue
+ # Grab first occurence of each to avoid running into
+ # alternative flavours and builds.
if not package:
matches = psearch_g(line)
if matches:
- package = matches.group(1)
+ s = matches.group(2)
+ if app_matches_packagename(app, s):
+ package = s
if not version:
matches = vnsearch_g(line)
if matches:
if matches:
vercode = matches.group(1)
else:
- xml = parse_xml(path)
- if "package" in xml.attrib:
- package = xml.attrib["package"]
- if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
- version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
- if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
- vercode = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
+ try:
+ xml = parse_xml(path)
+ if "package" in xml.attrib:
+ s = xml.attrib["package"].encode('utf-8')
+ if app_matches_packagename(app, s):
+ package = s
+ if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
+ version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"].encode('utf-8')
+ base_dir = os.path.dirname(path)
+ version = retrieve_string_singleline(base_dir, version)
+ if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
+ a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"].encode('utf-8')
+ if string_is_integer(a):
+ vercode = a
+ except Exception:
+ logging.warning("Problem with xml at {0}".format(path))
+
+ # Remember package name, may be defined separately from version+vercode
+ if package is None:
+ package = max_package
logging.debug("..got package={0}, version={1}, vercode={2}"
.format(package, version, vercode))
self.value = value
self.detail = detail
+ def shortened_detail(self):
+ if len(self.detail) < 16000:
+ return self.detail
+ return '[...]\n' + self.detail[-16000:]
+
def get_wikitext(self):
ret = repr(self.value) + "\n"
if self.detail:
ret += "=detail=\n"
- ret += "<pre>\n"
- txt = self.detail[-8192:] if len(self.detail) > 8192 else self.detail
- ret += str(txt)
- ret += "</pre>\n"
+ ret += "<pre>\n" + self.shortened_detail() + "</pre>\n"
return ret
def __str__(self):
# it, which may be a subdirectory of the actual project. If you want the base
# directory of the project, pass 'basepath=True'.
def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
- raw=False, prepare=True, preponly=False):
+ raw=False, prepare=True, preponly=False, refresh=True):
number = None
subdir = None
vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
vcs.srclib = (name, number, sdir)
if ref:
- vcs.gotorevision(ref)
+ vcs.gotorevision(ref, refresh)
if raw:
return vcs
return (name, number, libdir)
+gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
+
# Prepare the source code for a particular build
# 'vcs' - the appropriate vcs object for the application
# 'root' is the root directory, which may be the same as 'build_dir' or may
# be a subdirectory of it.
# 'srclibpaths' is information on the srclibs being used
-def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False):
+def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
# Optionally, the actual app source can be in a subdirectory
- if build['subdir']:
- root_dir = os.path.join(build_dir, build['subdir'])
+ if build.subdir:
+ root_dir = os.path.join(build_dir, build.subdir)
else:
root_dir = build_dir
# Get a working copy of the right revision
- logging.info("Getting source for revision " + build['commit'])
- vcs.gotorevision(build['commit'])
+ logging.info("Getting source for revision " + build.commit)
+ vcs.gotorevision(build.commit, refresh)
- # Initialise submodules if requred
- if build['submodules']:
+ # Initialise submodules if required
+ if build.submodules:
logging.info("Initialising submodules")
vcs.initsubmodules()
raise BuildException('Missing subdir ' + root_dir)
# Run an init command if one is required
- if build['init']:
- cmd = replace_config_vars(build['init'], build)
+ if build.init:
+ cmd = replace_config_vars(build.init, build)
logging.info("Running 'init' commands in %s" % root_dir)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if p.returncode != 0:
raise BuildException("Error running init command for %s:%s" %
- (app['id'], build['version']), p.output)
+ (app.id, build.version), p.output)
# Apply patches if any
- if build['patch']:
+ if build.patch:
logging.info("Applying patches")
- for patch in build['patch']:
+ for patch in build.patch:
patch = patch.strip()
logging.info("Applying " + patch)
- patch_path = os.path.join('metadata', app['id'], patch)
+ patch_path = os.path.join('metadata', app.id, patch)
p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
if p.returncode != 0:
raise BuildException("Failed to apply patch %s" % patch_path)
# Get required source libraries
srclibpaths = []
- if build['srclibs']:
+ if build.srclibs:
logging.info("Collecting source libraries")
- for lib in build['srclibs']:
- srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver))
+ for lib in build.srclibs:
+ srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
for name, number, libpath in srclibpaths:
place_srclib(root_dir, int(number) if number else None, libpath)
# Update the local.properties file
localprops = [os.path.join(build_dir, 'local.properties')]
- if build['subdir']:
- localprops += [os.path.join(root_dir, 'local.properties')]
+ if build.subdir:
+ parts = build.subdir.split(os.sep)
+ cur = build_dir
+ for d in parts:
+ cur = os.path.join(cur, d)
+ localprops += [os.path.join(cur, 'local.properties')]
for path in localprops:
props = ""
if os.path.isfile(path):
logging.info("Updating local.properties file at %s" % path)
- f = open(path, 'r')
- props += f.read()
- f.close()
+ with open(path, 'r') as f:
+ props += f.read()
props += '\n'
else:
logging.info("Creating local.properties file at %s" % path)
# Fix old-fashioned 'sdk-location' by copying
# from sdk.dir, if necessary
- if build['oldsdkloc']:
+ if build.oldsdkloc:
sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
re.S | re.M).group(1)
props += "sdk-location=%s\n" % sdkloc
else:
props += "sdk.dir=%s\n" % config['sdk_path']
props += "sdk-location=%s\n" % config['sdk_path']
- if build['ndk_path']:
+ ndk_path = build.ndk_path()
+ if ndk_path:
# Add ndk location
- props += "ndk.dir=%s\n" % build['ndk_path']
- props += "ndk-location=%s\n" % build['ndk_path']
+ props += "ndk.dir=%s\n" % ndk_path
+ props += "ndk-location=%s\n" % ndk_path
# Add java.encoding if necessary
- if build['encoding']:
- props += "java.encoding=%s\n" % build['encoding']
- f = open(path, 'w')
- f.write(props)
- f.close()
+ if build.encoding:
+ props += "java.encoding=%s\n" % build.encoding
+ with open(path, 'w') as f:
+ f.write(props)
flavours = []
- if build['type'] == 'gradle':
- flavours = build['gradle']
+ if build.method() == 'gradle':
+ flavours = build.gradle
- version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
- gradlepluginver = None
-
- gradle_dirs = [root_dir]
-
- # Parent dir build.gradle
- parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
- if parent_dir.startswith(build_dir):
- gradle_dirs.append(parent_dir)
-
- for dir_path in gradle_dirs:
- if gradlepluginver:
- break
- if not os.path.isdir(dir_path):
- continue
- for filename in os.listdir(dir_path):
- if not filename.endswith('.gradle'):
- continue
- path = os.path.join(dir_path, filename)
- if not os.path.isfile(path):
- continue
- for line in file(path):
- match = version_regex.match(line)
- if match:
- gradlepluginver = match.group(1)
- break
-
- if gradlepluginver:
- build['gradlepluginver'] = LooseVersion(gradlepluginver)
- else:
- logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
- build['gradlepluginver'] = LooseVersion('0.11')
-
- if build['target']:
- n = build["target"].split('-')[1]
- FDroidPopen(['sed', '-i',
- 's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g',
- 'build.gradle'], cwd=root_dir, output=False)
+ if build.target:
+ n = build.target.split('-')[1]
+ regsub_file(r'compileSdkVersion[ =]+[0-9]+',
+ r'compileSdkVersion %s' % n,
+ os.path.join(root_dir, 'build.gradle'))
# Remove forced debuggable flags
remove_debuggable_flags(root_dir)
# Insert version code and number into the manifest if necessary
- if build['forceversion']:
+ if build.forceversion:
logging.info("Changing the version name")
for path in manifest_paths(root_dir, flavours):
if not os.path.isfile(path):
continue
if has_extension(path, 'xml'):
- p = FDroidPopen(['sed', '-i',
- 's/android:versionName="[^"]*"/android:versionName="' + build['version'] + '"/g',
- path], output=False)
- if p.returncode != 0:
- raise BuildException("Failed to amend manifest")
+ regsub_file(r'android:versionName="[^"]*"',
+ r'android:versionName="%s"' % build.version,
+ path)
elif has_extension(path, 'gradle'):
- p = FDroidPopen(['sed', '-i',
- 's/versionName *=* *.*/versionName = "' + build['version'] + '"/g',
- path], output=False)
- if p.returncode != 0:
- raise BuildException("Failed to amend build.gradle")
- if build['forcevercode']:
+ regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
+ r"""\1versionName '%s'""" % build.version,
+ path)
+
+ if build.forcevercode:
logging.info("Changing the version code")
for path in manifest_paths(root_dir, flavours):
if not os.path.isfile(path):
continue
if has_extension(path, 'xml'):
- p = FDroidPopen(['sed', '-i',
- 's/android:versionCode="[^"]*"/android:versionCode="' + build['vercode'] + '"/g',
- path], output=False)
- if p.returncode != 0:
- raise BuildException("Failed to amend manifest")
+ regsub_file(r'android:versionCode="[^"]*"',
+ r'android:versionCode="%s"' % build.vercode,
+ path)
elif has_extension(path, 'gradle'):
- p = FDroidPopen(['sed', '-i',
- 's/versionCode *=* *[0-9]*/versionCode = ' + build['vercode'] + '/g',
- path], output=False)
- if p.returncode != 0:
- raise BuildException("Failed to amend build.gradle")
+ regsub_file(r'versionCode[ =]+[0-9]+',
+ r'versionCode %s' % build.vercode,
+ path)
# Delete unwanted files
- if build['rm']:
+ if build.rm:
logging.info("Removing specified files")
- for part in getpaths(build_dir, build, 'rm'):
+ for part in getpaths(build_dir, build.rm):
dest = os.path.join(build_dir, part)
logging.info("Removing {0}".format(part))
if os.path.lexists(dest):
remove_signing_keys(build_dir)
# Add required external libraries
- if build['extlibs']:
+ if build.extlibs:
logging.info("Collecting prebuilt libraries")
libsdir = os.path.join(root_dir, 'libs')
if not os.path.exists(libsdir):
os.mkdir(libsdir)
- for lib in build['extlibs']:
+ for lib in build.extlibs:
lib = lib.strip()
logging.info("...installing extlib {0}".format(lib))
libf = os.path.basename(lib)
shutil.copyfile(libsrc, os.path.join(libsdir, libf))
# Run a pre-build command if one is required
- if build['prebuild']:
+ if build.prebuild:
logging.info("Running 'prebuild' commands in %s" % root_dir)
- cmd = replace_config_vars(build['prebuild'], build)
+ cmd = replace_config_vars(build.prebuild, build)
# Substitute source library paths into prebuild commands
for name, number, libpath in srclibpaths:
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if p.returncode != 0:
raise BuildException("Error running prebuild command for %s:%s" %
- (app['id'], build['version']), p.output)
+ (app.id, build.version), p.output)
# Generate (or update) the ant build file, build.xml...
- if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
+ if build.method() == 'ant' and build.update != ['no']:
parms = ['android', 'update', 'lib-project']
lparms = ['android', 'update', 'project']
- if build['target']:
- parms += ['-t', build['target']]
- lparms += ['-t', build['target']]
- if build['update'] == ['auto']:
- update_dirs = ant_subprojects(root_dir) + ['.']
+ if build.target:
+ parms += ['-t', build.target]
+ lparms += ['-t', build.target]
+ if build.update:
+ update_dirs = build.update
else:
- update_dirs = build['update']
+ update_dirs = ant_subprojects(root_dir) + ['.']
for d in update_dirs:
subdir = os.path.join(root_dir, d)
return (root_dir, srclibpaths)
-# Split and extend via globbing the paths from a field
-def getpaths(build_dir, build, field):
- paths = []
- for p in build[field]:
+# Extend via globbing the paths from a field and return them as a map from
+# original path to resulting paths
+def getpaths_map(build_dir, globpaths):
+ paths = dict()
+ for p in globpaths:
p = p.strip()
full_path = os.path.join(build_dir, p)
full_path = os.path.normpath(full_path)
- paths += [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
+ paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
+ if not paths[p]:
+ raise FDroidException("glob path '%s' did not match any files/dirs" % p)
return paths
-# Scan the source code in the given directory (and all subdirectories)
-# and return the number of fatal problems encountered
-def scan_source(build_dir, root_dir, thisbuild):
-
- count = 0
-
- # Common known non-free blobs (always lower case):
- usual_suspects = [
- re.compile(r'flurryagent', re.IGNORECASE),
- re.compile(r'paypal.*mpl', re.IGNORECASE),
- re.compile(r'google.*analytics', re.IGNORECASE),
- re.compile(r'admob.*sdk.*android', re.IGNORECASE),
- re.compile(r'google.*ad.*view', re.IGNORECASE),
- re.compile(r'google.*admob', re.IGNORECASE),
- re.compile(r'google.*play.*services', re.IGNORECASE),
- re.compile(r'crittercism', re.IGNORECASE),
- re.compile(r'heyzap', re.IGNORECASE),
- re.compile(r'jpct.*ae', re.IGNORECASE),
- re.compile(r'youtube.*android.*player.*api', re.IGNORECASE),
- re.compile(r'bugsense', re.IGNORECASE),
- re.compile(r'crashlytics', re.IGNORECASE),
- re.compile(r'ouya.*sdk', re.IGNORECASE),
- re.compile(r'libspen23', re.IGNORECASE),
- ]
-
- scanignore = getpaths(build_dir, thisbuild, 'scanignore')
- scandelete = getpaths(build_dir, thisbuild, 'scandelete')
-
- scanignore_worked = set()
- scandelete_worked = set()
-
- try:
- ms = magic.open(magic.MIME_TYPE)
- ms.load()
- except AttributeError:
- ms = None
-
- def toignore(fd):
- for p in scanignore:
- if fd.startswith(p):
- scanignore_worked.add(p)
- return True
- return False
-
- def todelete(fd):
- for p in scandelete:
- if fd.startswith(p):
- scandelete_worked.add(p)
- return True
- return False
-
- def ignoreproblem(what, fd, fp):
- logging.info('Ignoring %s at %s' % (what, fd))
- return 0
-
- def removeproblem(what, fd, fp):
- logging.info('Removing %s at %s' % (what, fd))
- os.remove(fp)
- return 0
-
- def warnproblem(what, fd):
- logging.warn('Found %s at %s' % (what, fd))
-
- def handleproblem(what, fd, fp):
- if toignore(fd):
- return ignoreproblem(what, fd, fp)
- if todelete(fd):
- return removeproblem(what, fd, fp)
- logging.error('Found %s at %s' % (what, fd))
- return 1
-
- # Iterate through all files in the source code
- for r, d, f in os.walk(build_dir, topdown=True):
-
- # It's topdown, so checking the basename is enough
- for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
- if ignoredir in d:
- d.remove(ignoredir)
-
- for curfile in f:
-
- # Path (relative) to the file
- fp = os.path.join(r, curfile)
- fd = fp[len(build_dir) + 1:]
-
- try:
- mime = magic.from_file(fp, mime=True) if ms is None else ms.file(fp)
- except UnicodeError:
- warnproblem('malformed magic number', fd)
-
- if mime == 'application/x-sharedlib':
- count += handleproblem('shared library', fd, fp)
-
- elif mime == 'application/x-archive':
- count += handleproblem('static library', fd, fp)
-
- elif mime == 'application/x-executable':
- count += handleproblem('binary executable', fd, fp)
-
- elif mime == 'application/x-java-applet':
- count += handleproblem('Java compiled class', fd, fp)
-
- elif mime in (
- 'application/jar',
- 'application/zip',
- 'application/java-archive',
- 'application/octet-stream',
- 'binary', ):
-
- if has_extension(fp, 'apk'):
- removeproblem('APK file', fd, fp)
-
- elif has_extension(fp, 'jar'):
-
- if any(suspect.match(curfile) for suspect in usual_suspects):
- count += handleproblem('usual supect', fd, fp)
- else:
- warnproblem('JAR file', fd)
-
- elif has_extension(fp, 'zip'):
- warnproblem('ZIP file', fd)
-
- else:
- warnproblem('unknown compressed or binary file', fd)
-
- elif has_extension(fp, 'java') and os.path.isfile(fp):
- if not os.path.isfile(fp):
- continue
- for line in file(fp):
- if 'DexClassLoader' in line:
- count += handleproblem('DexClassLoader', fd, fp)
- break
- if ms is not None:
- ms.close()
-
- for p in scanignore:
- if p not in scanignore_worked:
- logging.error('Unused scanignore path: %s' % p)
- count += 1
-
- for p in scandelete:
- if p not in scandelete_worked:
- logging.error('Unused scandelete path: %s' % p)
- count += 1
+# Extend via globbing the paths from a field and return them as a set
+def getpaths(build_dir, globpaths):
+ paths_map = getpaths_map(build_dir, globpaths)
+ paths = set()
+ for k, v in paths_map.iteritems():
+ for p in v:
+ paths.add(p)
+ return paths
- # Presence of a jni directory without buildjni=yes might
- # indicate a problem (if it's not a problem, explicitly use
- # buildjni=no to bypass this check)
- if (os.path.exists(os.path.join(root_dir, 'jni')) and
- not thisbuild['buildjni']):
- logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
- count += 1
- return count
+def natural_key(s):
+ return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
class KnownApks:
self.changed = False
def writeifchanged(self):
- if self.changed:
- if not os.path.exists('stats'):
- os.mkdir('stats')
- f = open(self.path, 'w')
- lst = []
- for apk, app in self.apks.iteritems():
- appid, added = app
- line = apk + ' ' + appid
- if added:
- line += ' ' + time.strftime('%Y-%m-%d', added)
- lst.append(line)
- for line in sorted(lst):
+ if not self.changed:
+ return
+
+ if not os.path.exists('stats'):
+ os.mkdir('stats')
+
+ lst = []
+ for apk, app in self.apks.iteritems():
+ appid, added = app
+ line = apk + ' ' + appid
+ if added:
+ line += ' ' + time.strftime('%Y-%m-%d', added)
+ lst.append(line)
+
+ with open(self.path, 'w') as f:
+ for line in sorted(lst, key=natural_key):
f.write(line + '\n')
- f.close()
# Record an apk (if it's new, otherwise does nothing)
# Returns the date it was added.
return False
-class AsynchronousFileReader(threading.Thread):
-
- '''
- Helper class to implement asynchronous reading of a file
- in a separate thread. Pushes read lines on a queue to
- be consumed in another thread.
- '''
-
- def __init__(self, fd, queue):
- assert isinstance(queue, Queue.Queue)
- assert callable(fd.readline)
- threading.Thread.__init__(self)
- self._fd = fd
- self._queue = queue
-
- def run(self):
- '''The body of the tread: read lines and put them on the queue.'''
- for line in iter(self._fd.readline, ''):
- self._queue.put(line)
-
- def eof(self):
- '''Check whether there is no more content to expect.'''
- return not self.is_alive() and self._queue.empty()
-
-
class PopenResult:
returncode = None
output = ''
cmd = commands[0]
if cmd not in config:
config[cmd] = find_sdk_tools_cmd(commands[0])
- return FDroidPopen([config[cmd]] + commands[1:],
+ abscmd = config[cmd]
+ if abscmd is None:
+ logging.critical("Could not find '%s' on your system" % cmd)
+ sys.exit(1)
+ return FDroidPopen([abscmd] + commands[1:],
cwd=cwd, output=output)
try:
p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- except OSError, e:
+ except OSError as e:
raise BuildException("OSError while trying to execute " +
' '.join(commands) + ': ' + str(e))
- stdout_queue = Queue.Queue()
+ stdout_queue = Queue()
stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
- stdout_reader.start()
# Check the queue for output (until there is no more to get)
while not stdout_reader.eof():
return result
+gradle_comment = re.compile(r'[ ]*//')
+gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
+gradle_line_matches = [
+ re.compile(r'^[\t ]*signingConfig [^ ]*$'),
+ re.compile(r'.*android\.signingConfigs\.[^{]*$'),
+ re.compile(r'.*\.readLine\(.*'),
+]
+
+
def remove_signing_keys(build_dir):
- comment = re.compile(r'[ ]*//')
- signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
- line_matches = [
- re.compile(r'^[\t ]*signingConfig [^ ]*$'),
- re.compile(r'.*android\.signingConfigs\.[^{]*$'),
- re.compile(r'.*variant\.outputFile = .*'),
- re.compile(r'.*output\.outputFile = .*'),
- re.compile(r'.*\.readLine\(.*'),
- ]
for root, dirs, files in os.walk(build_dir):
if 'build.gradle' in files:
path = os.path.join(root, 'build.gradle')
line = line.rstrip('\\\n') + lines[i]
i += 1
- if comment.match(line):
+ if gradle_comment.match(line):
+ o.write(line)
continue
if opened > 0:
opened -= line.count('}')
continue
- if signing_configs.match(line):
+ if gradle_signing_configs.match(line):
changed = True
opened += 1
continue
- if any(s.match(line) for s in line_matches):
+ if any(s.match(line) for s in gradle_line_matches):
changed = True
continue
cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
cmd = cmd.replace('$$MVN3$$', config['mvn3'])
if build is not None:
- cmd = cmd.replace('$$COMMIT$$', build['commit'])
- cmd = cmd.replace('$$VERSION$$', build['version'])
- cmd = cmd.replace('$$VERCODE$$', build['vercode'])
+ cmd = cmd.replace('$$COMMIT$$', build.commit)
+ cmd = cmd.replace('$$VERSION$$', build.version)
+ cmd = cmd.replace('$$VERCODE$$', build.vercode)
return cmd
if not placed:
o.write('android.library.reference.%d=%s\n' % (number, relpath))
+apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
+
def verify_apks(signed_apk, unsigned_apk, tmp_dir):
"""Verify that two apks are the same
:returns: None if the verification is successful, otherwise a string
describing what went wrong.
"""
- sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
with ZipFile(signed_apk) as signed_apk_as_zip:
meta_inf_files = ['META-INF/MANIFEST.MF']
for f in signed_apk_as_zip.namelist():
- if sigfile.match(f):
+ if apk_sigfile.match(f):
meta_inf_files.append(f)
if len(meta_inf_files) < 3:
return "Signature files missing from {0}".format(signed_apk)
logging.info("...successfully verified")
return None
+apk_badchars = re.compile('''[/ :;'"]''')
+
def compare_apks(apk1, apk2, tmp_dir):
"""Compare two apks
trying to do the comparison.
"""
- badchars = re.compile('''[/ :;'"]''')
- apk1dir = os.path.join(tmp_dir, badchars.sub('_', apk1[0:-4])) # trim .apk
- apk2dir = os.path.join(tmp_dir, badchars.sub('_', apk2[0:-4])) # trim .apk
+ apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
+ apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
for d in [apk1dir, apk2dir]:
if os.path.exists(d):
shutil.rmtree(d)
'-keypass:file', config['keypassfile'],
'-dname', localconfig['keydname']])
# TODO keypass should be sent via stdin
- os.chmod(localconfig['keystore'], 0o0600)
if p.returncode != 0:
raise BuildException("Failed to generate key", p.output)
+ os.chmod(localconfig['keystore'], 0o0600)
# now show the lovely key that was just generated
p = FDroidPopen(['keytool', '-list', '-v',
'-keystore', localconfig['keystore'],
def parse_xml(path):
return XMLElementTree.parse(path).getroot()
+
+
+def string_is_integer(string):
+ try:
+ int(string)
+ return True
+ except ValueError:
+ return False
+
+
+def get_per_app_repos():
+ '''per-app repos are dirs named with the packageName of a single app'''
+
+ # Android packageNames are Java packages, they may contain uppercase or
+ # lowercase letters ('A' through 'Z'), numbers, and underscores
+ # ('_'). However, individual package name parts may only start with
+ # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
+ p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
+
+ repos = []
+ for root, dirs, files in os.walk(os.getcwd()):
+ for d in dirs:
+ print('checking', root, 'for', d)
+ if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
+ # standard parts of an fdroid repo, so never packageNames
+ continue
+ elif p.match(d) \
+ and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
+ repos.append(d)
+ break
+ return repos
~ianmdlvl / fdroidserver.git / blobdiff