-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
#
# common.py - part of the FDroid server tools
# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
# common.py is imported by all modules, so do not import third-party
# libraries here as they will become a requirement for all commands.
+import io
import os
import sys
import re
import subprocess
import time
import operator
-import Queue
import logging
import hashlib
import socket
+import base64
import xml.etree.ElementTree as XMLElementTree
-from distutils.version import LooseVersion
+from queue import Queue
+
from zipfile import ZipFile
-import metadata
-from fdroidserver.asynchronousfilereader import AsynchronousFileReader
+import fdroidserver.metadata
+from .asynchronousfilereader import AsynchronousFileReader
XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
'r9b': None,
'r10e': "$ANDROID_NDK",
},
- 'build_tools': "23.0.2",
- 'java_paths': {
- '1.7': "/usr/lib/jvm/java-7-openjdk",
- '1.8': None,
- },
+ 'build_tools': "24.0.0",
+ 'force_build_tools': False,
+ 'java_paths': None,
'ant': "ant",
'mvn3': "mvn",
'gradle': 'gradle',
- 'accepted_formats': ['txt', 'yaml'],
+ 'accepted_formats': ['txt', 'yml'],
'sync_from_local_copy_dir': False,
'per_app_repos': False,
'make_current_version_link': True,
thisconfig[k] = exp
thisconfig[k + '_orig'] = v
+ # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
+ if thisconfig['java_paths'] is None:
+ thisconfig['java_paths'] = dict()
+ pathlist = []
+ pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
+ pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
+ pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
+ pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
+ if os.getenv('JAVA_HOME') is not None:
+ pathlist += os.getenv('JAVA_HOME')
+ if os.getenv('PROGRAMFILES') is not None:
+ pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
+ for d in sorted(pathlist):
+ if os.path.islink(d):
+ continue
+ j = os.path.basename(d)
+ # the last one found will be the canonical one, so order appropriately
+ for regex in [
+ r'^1\.([6-9])\.0\.jdk$', # OSX
+ r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
+ r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
+ r'^jdk([6-9])-openjdk$', # Arch
+ r'^java-([6-9])-openjdk$', # Arch
+ r'^java-([6-9])-jdk$', # Arch (oracle)
+ r'^java-1\.([6-9])\.0-.*$', # RedHat
+ r'^java-([6-9])-oracle$', # Debian WebUpd8
+ r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
+ r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
+ ]:
+ m = re.match(regex, j)
+ if not m:
+ continue
+ osxhome = os.path.join(d, 'Contents', 'Home')
+ if os.path.exists(osxhome):
+ thisconfig['java_paths'][m.group(1)] = osxhome
+ else:
+ thisconfig['java_paths'][m.group(1)] = d
+
+ for java_version in ('7', '8', '9'):
+ if java_version not in thisconfig['java_paths']:
+ continue
+ java_home = thisconfig['java_paths'][java_version]
+ jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
+ if os.path.exists(jarsigner):
+ thisconfig['jarsigner'] = jarsigner
+ thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
+ break # Java7 is preferred, so quit if found
+
for k in ['ndk_paths', 'java_paths']:
d = thisconfig[k]
for k2 in d.copy():
def regsub_file(pattern, repl, path):
- with open(path, 'r') as f:
+ with open(path, 'rb') as f:
text = f.read()
- text = re.sub(pattern, repl, text)
- with open(path, 'w') as f:
+ text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
+ with open(path, 'wb') as f:
f.write(text)
def read_config(opts, config_file='config.py'):
"""Read the repository config
- The config is read from config_file, which is in the current directory when
- any of the repo management commands are used.
+ The config is read from config_file, which is in the current
+ directory when any of the repo management commands are used. If
+ there is a local metadata file in the git repo, then config.py is
+ not required, just use defaults.
+
"""
- global config, options, env, orig_path
+ global config, options
if config is not None:
return config
- if not os.path.isfile(config_file):
- logging.critical("Missing config file - is this a repo directory?")
- sys.exit(2)
options = opts
config = {}
- logging.debug("Reading %s" % config_file)
- execfile(config_file, config)
+ if os.path.isfile(config_file):
+ logging.debug("Reading %s" % config_file)
+ with io.open(config_file, "rb") as f:
+ code = compile(f.read(), config_file, 'exec')
+ exec(code, None, config)
+ elif len(get_local_metadata_files()) == 0:
+ logging.critical("Missing config file - is this a repo directory?")
+ sys.exit(2)
# smartcardoptions must be a list since its command line args for Popen
if 'smartcardoptions' in config:
fill_config_defaults(config)
- # There is no standard, so just set up the most common environment
- # variables
- env = os.environ
- orig_path = env['PATH']
- for n in ['ANDROID_HOME', 'ANDROID_SDK']:
- env[n] = config['sdk_path']
-
- for v in ['7', '8']:
- cpath = config['java_paths']['1.%s' % v]
- if cpath:
- env['JAVA%s_HOME' % v] = cpath
-
for k in ["keystorepass", "keypass"]:
if k in config:
write_password_file(k)
config[k] = clean_description(config[k])
if 'serverwebroot' in config:
- if isinstance(config['serverwebroot'], basestring):
+ if isinstance(config['serverwebroot'], str):
roots = [config['serverwebroot']]
- elif all(isinstance(item, basestring) for item in config['serverwebroot']):
+ elif all(isinstance(item, str) for item in config['serverwebroot']):
roots = config['serverwebroot']
else:
raise TypeError('only accepts strings, lists, and tuples')
return config
-def get_ndk_path(version):
- if version is None:
- version = 'r10e' # falls back to latest
- paths = config['ndk_paths']
- if version not in paths:
- return ''
- return paths[version] or ''
-
-
def find_sdk_tools_cmd(cmd):
'''find a working path to a tool from the Android SDK'''
command line argments
'''
filename = '.fdroid.' + pwtype + '.txt'
- fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
+ fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
if password is None:
- os.write(fd, config[pwtype])
+ os.write(fd, config[pwtype].encode('utf-8'))
else:
- os.write(fd, password)
+ os.write(fd, password.encode('utf-8'))
os.close(fd)
config[pwtype + 'file'] = filename
+def get_local_metadata_files():
+ '''get any metadata files local to an app's source repo
+
+ This tries to ignore anything that does not count as app metdata,
+ including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
+
+ '''
+ return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
+
+
# Given the arguments in the form of multiple appid:[vc] strings, this returns
# a dictionary with the set of vercodes specified for each package.
def read_pkg_args(args, allow_vercodes=False):
return allapps
apps = {}
- for appid, app in allapps.iteritems():
+ for appid, app in allapps.items():
if appid in vercodes:
apps[appid] = app
raise FDroidException("No packages specified")
error = False
- for appid, app in apps.iteritems():
+ for appid, app in apps.items():
vc = vercodes[appid]
if not vc:
continue
- app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
- if len(app['builds']) != len(vercodes[appid]):
+ app.builds = [b for b in app.builds if b.vercode in vc]
+ if len(app.builds) != len(vercodes[appid]):
error = True
- allvcs = [b['vercode'] for b in app['builds']]
+ allvcs = [b.vercode for b in app.builds]
for v in vercodes[appid]:
if v not in allvcs:
logging.critical("No such vercode %s for app %s" % (v, appid))
return ext == f_ext
-apk_regex = None
+apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
def clean_description(description):
def apknameinfo(filename):
- global apk_regex
filename = os.path.basename(filename)
- if apk_regex is None:
- apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
m = apk_regex.match(filename)
try:
result = (m.group(1), m.group(2))
def getapkname(app, build):
- return "%s_%s.apk" % (app['id'], build['vercode'])
+ return "%s_%s.apk" % (app.id, build.vercode)
def getsrcname(app, build):
- return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
+ return "%s_%s_src.tar.gz" % (app.id, build.vercode)
def getappname(app):
- if app['Name']:
- return app['Name']
- if app['Auto Name']:
- return app['Auto Name']
- return app['id']
+ if app.Name:
+ return app.Name
+ if app.AutoName:
+ return app.AutoName
+ return app.id
def getcvname(app):
- return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
+ return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
def getvcs(vcstype, remote, local):
def getsrclibvcs(name):
- if name not in metadata.srclibs:
+ if name not in fdroidserver.metadata.srclibs:
raise VCSException("Missing srclib " + name)
- return metadata.srclibs[name]['Repo Type']
+ return fdroidserver.metadata.srclibs[name]['Repo Type']
class vcs:
# automatically if either of those things changes.
fdpath = os.path.join(self.local, '..',
'.fdroidvcs-' + os.path.basename(self.local))
+ fdpath = os.path.normpath(fdpath)
cdata = self.repotype() + ' ' + self.remote
writeback = True
deleterepo = False
try:
self.gotorevisionx(rev)
- except FDroidException, e:
+ except FDroidException as e:
exc = e
# If necessary, write the .fdroidvcs file.
if writeback and not self.clone_failed:
- with open(fdpath, 'w') as f:
+ os.makedirs(os.path.dirname(fdpath), exist_ok=True)
+ with open(fdpath, 'w+') as f:
f.write(cdata)
if exc is not None:
rtags.append(tag)
return rtags
- def latesttags(self, tags, number):
- """Get the most recent tags in a given list.
-
- :param tags: a list of tags
- :param number: the number to return
- :returns: A list containing the most recent tags in the provided
- list, up to the maximum number given.
- """
+ # Get a list of all the known tags, sorted from newest to oldest
+ def latesttags(self):
raise VCSException('latesttags not supported for this vcs type')
# Get current commit reference (hash, revision, etc)
p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
return p.output.splitlines()
- def latesttags(self, tags, number):
+ tag_format = re.compile(r'tag: ([^),]*)')
+
+ def latesttags(self):
self.checkrepo()
- tl = []
- for tag in tags:
- p = FDroidPopen(
- ['git', 'show', '--format=format:%ct', '-s', tag],
- cwd=self.local, output=False)
- # Timestamp is on the last line. For a normal tag, it's the only
- # line, but for annotated tags, the rest of the info precedes it.
- ts = int(p.output.splitlines()[-1])
- tl.append((ts, tag))
- latest = []
- for _, t in sorted(tl)[-number:]:
- latest.append(t)
- return latest
+ p = FDroidPopen(['git', 'log', '--tags',
+ '--simplify-by-decoration', '--pretty=format:%d'],
+ cwd=self.local, output=False)
+ tags = []
+ for line in p.output.splitlines():
+ for tag in self.tag_format.findall(line):
+ tags.append(tag)
+ return tags
class vcs_gitsvn(vcs):
if element.text is None:
return ""
s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
- return s.strip()
+ return s.decode('utf-8').strip()
for path in xmlfiles:
if not os.path.isfile(path):
logging.debug("fetch_real_name: Checking manifest at " + path)
xml = parse_xml(path)
app = xml.find('application')
+ if app is None:
+ continue
if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
continue
- label = app.attrib["{http://schemas.android.com/apk/res/android}label"].encode('utf-8')
+ label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
result = retrieve_string_singleline(app_dir, label)
if result:
result = result.strip()
proppath = os.path.join(root_dir, 'project.properties')
if not os.path.isfile(proppath):
return libraries
- for line in file(proppath):
- if not line.startswith('android.library.reference.'):
- continue
- path = line.split('=')[1].strip()
- relpath = os.path.join(root_dir, path)
- if not os.path.isdir(relpath):
- continue
- logging.debug("Found subproject at %s" % path)
- libraries.append(path)
+ with open(proppath, 'r', encoding='iso-8859-1') as f:
+ for line in f:
+ if not line.startswith('android.library.reference.'):
+ continue
+ path = line.split('=')[1].strip()
+ relpath = os.path.join(root_dir, path)
+ if not os.path.isdir(relpath):
+ continue
+ logging.debug("Found subproject at %s" % path)
+ libraries.append(path)
return libraries
def app_matches_packagename(app, package):
if not package:
return False
- appid = app['Update Check Name'] or app['id']
- if appid == "Ignore":
+ appid = app.UpdateCheckName or app.id
+ if appid is None or appid == "Ignore":
return True
return appid == package
# All values returned are strings.
def parse_androidmanifests(paths, app):
- ignoreversions = app['Update Check Ignore']
+ ignoreversions = app.UpdateCheckIgnore
ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
if not paths:
package = None
if gradle:
- for line in file(path):
- if gradle_comment.match(line):
- continue
- # Grab first occurence of each to avoid running into
- # alternative flavours and builds.
- if not package:
- matches = psearch_g(line)
- if matches:
- s = matches.group(2)
- if app_matches_packagename(app, s):
- package = s
- if not version:
- matches = vnsearch_g(line)
- if matches:
- version = matches.group(2)
- if not vercode:
- matches = vcsearch_g(line)
- if matches:
- vercode = matches.group(1)
+ with open(path, 'r') as f:
+ for line in f:
+ if gradle_comment.match(line):
+ continue
+ # Grab first occurence of each to avoid running into
+ # alternative flavours and builds.
+ if not package:
+ matches = psearch_g(line)
+ if matches:
+ s = matches.group(2)
+ if app_matches_packagename(app, s):
+ package = s
+ if not version:
+ matches = vnsearch_g(line)
+ if matches:
+ version = matches.group(2)
+ if not vercode:
+ matches = vcsearch_g(line)
+ if matches:
+ vercode = matches.group(1)
else:
- xml = parse_xml(path)
- if "package" in xml.attrib:
- s = xml.attrib["package"].encode('utf-8')
- if app_matches_packagename(app, s):
- package = s
- if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
- version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"].encode('utf-8')
- base_dir = os.path.dirname(path)
- version = retrieve_string_singleline(base_dir, version)
- if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
- a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"].encode('utf-8')
- if string_is_integer(a):
- vercode = a
+ try:
+ xml = parse_xml(path)
+ if "package" in xml.attrib:
+ s = xml.attrib["package"]
+ if app_matches_packagename(app, s):
+ package = s
+ if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
+ version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
+ base_dir = os.path.dirname(path)
+ version = retrieve_string_singleline(base_dir, version)
+ if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
+ a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
+ if string_is_integer(a):
+ vercode = a
+ except Exception:
+ logging.warning("Problem with xml at {0}".format(path))
# Remember package name, may be defined separately from version+vercode
if package is None:
# it, which may be a subdirectory of the actual project. If you want the base
# directory of the project, pass 'basepath=True'.
def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
- raw=False, prepare=True, preponly=False, refresh=True):
+ raw=False, prepare=True, preponly=False, refresh=True,
+ build=None):
number = None
subdir = None
if '/' in name:
name, subdir = name.split('/', 1)
- if name not in metadata.srclibs:
+ if name not in fdroidserver.metadata.srclibs:
raise VCSException('srclib ' + name + ' not found.')
- srclib = metadata.srclibs[name]
+ srclib = fdroidserver.metadata.srclibs[name]
sdir = os.path.join(srclib_dir, name)
if prepare:
if srclib["Prepare"]:
- cmd = replace_config_vars(srclib["Prepare"], None)
+ cmd = replace_config_vars(srclib["Prepare"], build)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
if p.returncode != 0:
def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
# Optionally, the actual app source can be in a subdirectory
- if build['subdir']:
- root_dir = os.path.join(build_dir, build['subdir'])
+ if build.subdir:
+ root_dir = os.path.join(build_dir, build.subdir)
else:
root_dir = build_dir
# Get a working copy of the right revision
- logging.info("Getting source for revision " + build['commit'])
- vcs.gotorevision(build['commit'], refresh)
+ logging.info("Getting source for revision " + build.commit)
+ vcs.gotorevision(build.commit, refresh)
# Initialise submodules if required
- if build['submodules']:
+ if build.submodules:
logging.info("Initialising submodules")
vcs.initsubmodules()
raise BuildException('Missing subdir ' + root_dir)
# Run an init command if one is required
- if build['init']:
- cmd = replace_config_vars(build['init'], build)
+ if build.init:
+ cmd = replace_config_vars(build.init, build)
logging.info("Running 'init' commands in %s" % root_dir)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if p.returncode != 0:
raise BuildException("Error running init command for %s:%s" %
- (app['id'], build['version']), p.output)
+ (app.id, build.version), p.output)
# Apply patches if any
- if build['patch']:
+ if build.patch:
logging.info("Applying patches")
- for patch in build['patch']:
+ for patch in build.patch:
patch = patch.strip()
logging.info("Applying " + patch)
- patch_path = os.path.join('metadata', app['id'], patch)
+ patch_path = os.path.join('metadata', app.id, patch)
p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
if p.returncode != 0:
raise BuildException("Failed to apply patch %s" % patch_path)
# Get required source libraries
srclibpaths = []
- if build['srclibs']:
+ if build.srclibs:
logging.info("Collecting source libraries")
- for lib in build['srclibs']:
- srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
+ for lib in build.srclibs:
+ srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
+ refresh=refresh, build=build))
for name, number, libpath in srclibpaths:
place_srclib(root_dir, int(number) if number else None, libpath)
# Update the local.properties file
localprops = [os.path.join(build_dir, 'local.properties')]
- if build['subdir']:
- parts = build['subdir'].split(os.sep)
+ if build.subdir:
+ parts = build.subdir.split(os.sep)
cur = build_dir
for d in parts:
cur = os.path.join(cur, d)
props = ""
if os.path.isfile(path):
logging.info("Updating local.properties file at %s" % path)
- with open(path, 'r') as f:
+ with open(path, 'r', encoding='iso-8859-1') as f:
props += f.read()
props += '\n'
else:
logging.info("Creating local.properties file at %s" % path)
# Fix old-fashioned 'sdk-location' by copying
# from sdk.dir, if necessary
- if build['oldsdkloc']:
+ if build.oldsdkloc:
sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
re.S | re.M).group(1)
props += "sdk-location=%s\n" % sdkloc
else:
props += "sdk.dir=%s\n" % config['sdk_path']
props += "sdk-location=%s\n" % config['sdk_path']
- if build['ndk_path']:
+ ndk_path = build.ndk_path()
+ if ndk_path:
# Add ndk location
- props += "ndk.dir=%s\n" % build['ndk_path']
- props += "ndk-location=%s\n" % build['ndk_path']
+ props += "ndk.dir=%s\n" % ndk_path
+ props += "ndk-location=%s\n" % ndk_path
# Add java.encoding if necessary
- if build['encoding']:
- props += "java.encoding=%s\n" % build['encoding']
- with open(path, 'w') as f:
+ if build.encoding:
+ props += "java.encoding=%s\n" % build.encoding
+ with open(path, 'w', encoding='iso-8859-1') as f:
f.write(props)
flavours = []
- if build['type'] == 'gradle':
- flavours = build['gradle']
-
- gradlepluginver = None
+ if build.build_method() == 'gradle':
+ flavours = build.gradle
- gradle_dirs = [root_dir]
-
- # Parent dir build.gradle
- parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
- if parent_dir.startswith(build_dir):
- gradle_dirs.append(parent_dir)
-
- for dir_path in gradle_dirs:
- if gradlepluginver:
- break
- if not os.path.isdir(dir_path):
- continue
- for filename in os.listdir(dir_path):
- if not filename.endswith('.gradle'):
- continue
- path = os.path.join(dir_path, filename)
- if not os.path.isfile(path):
- continue
- for line in file(path):
- match = gradle_version_regex.match(line)
- if match:
- gradlepluginver = match.group(1)
- break
-
- if gradlepluginver:
- build['gradlepluginver'] = LooseVersion(gradlepluginver)
- else:
- logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
- build['gradlepluginver'] = LooseVersion('0.11')
-
- if build['target']:
- n = build["target"].split('-')[1]
+ if build.target:
+ n = build.target.split('-')[1]
regsub_file(r'compileSdkVersion[ =]+[0-9]+',
r'compileSdkVersion %s' % n,
os.path.join(root_dir, 'build.gradle'))
remove_debuggable_flags(root_dir)
# Insert version code and number into the manifest if necessary
- if build['forceversion']:
+ if build.forceversion:
logging.info("Changing the version name")
for path in manifest_paths(root_dir, flavours):
if not os.path.isfile(path):
continue
if has_extension(path, 'xml'):
regsub_file(r'android:versionName="[^"]*"',
- r'android:versionName="%s"' % build['version'],
+ r'android:versionName="%s"' % build.version,
path)
elif has_extension(path, 'gradle'):
regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
- r"""\1versionName '%s'""" % build['version'],
+ r"""\1versionName '%s'""" % build.version,
path)
- if build['forcevercode']:
+ if build.forcevercode:
logging.info("Changing the version code")
for path in manifest_paths(root_dir, flavours):
if not os.path.isfile(path):
continue
if has_extension(path, 'xml'):
regsub_file(r'android:versionCode="[^"]*"',
- r'android:versionCode="%s"' % build['vercode'],
+ r'android:versionCode="%s"' % build.vercode,
path)
elif has_extension(path, 'gradle'):
regsub_file(r'versionCode[ =]+[0-9]+',
- r'versionCode %s' % build['vercode'],
+ r'versionCode %s' % build.vercode,
path)
# Delete unwanted files
- if build['rm']:
+ if build.rm:
logging.info("Removing specified files")
- for part in getpaths(build_dir, build['rm']):
+ for part in getpaths(build_dir, build.rm):
dest = os.path.join(build_dir, part)
logging.info("Removing {0}".format(part))
if os.path.lexists(dest):
remove_signing_keys(build_dir)
# Add required external libraries
- if build['extlibs']:
+ if build.extlibs:
logging.info("Collecting prebuilt libraries")
libsdir = os.path.join(root_dir, 'libs')
if not os.path.exists(libsdir):
os.mkdir(libsdir)
- for lib in build['extlibs']:
+ for lib in build.extlibs:
lib = lib.strip()
logging.info("...installing extlib {0}".format(lib))
libf = os.path.basename(lib)
shutil.copyfile(libsrc, os.path.join(libsdir, libf))
# Run a pre-build command if one is required
- if build['prebuild']:
+ if build.prebuild:
logging.info("Running 'prebuild' commands in %s" % root_dir)
- cmd = replace_config_vars(build['prebuild'], build)
+ cmd = replace_config_vars(build.prebuild, build)
# Substitute source library paths into prebuild commands
for name, number, libpath in srclibpaths:
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
if p.returncode != 0:
raise BuildException("Error running prebuild command for %s:%s" %
- (app['id'], build['version']), p.output)
+ (app.id, build.version), p.output)
# Generate (or update) the ant build file, build.xml...
- if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
+ if build.build_method() == 'ant' and build.update != ['no']:
parms = ['android', 'update', 'lib-project']
lparms = ['android', 'update', 'project']
- if build['target']:
- parms += ['-t', build['target']]
- lparms += ['-t', build['target']]
- if build['update'] == ['auto']:
- update_dirs = ant_subprojects(root_dir) + ['.']
+ if build.target:
+ parms += ['-t', build.target]
+ lparms += ['-t', build.target]
+ if build.update:
+ update_dirs = build.update
else:
- update_dirs = build['update']
+ update_dirs = ant_subprojects(root_dir) + ['.']
for d in update_dirs:
subdir = os.path.join(root_dir, d)
full_path = os.path.join(build_dir, p)
full_path = os.path.normpath(full_path)
paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
+ if not paths[p]:
+ raise FDroidException("glob path '%s' did not match any files/dirs" % p)
return paths
def getpaths(build_dir, globpaths):
paths_map = getpaths_map(build_dir, globpaths)
paths = set()
- for k, v in paths_map.iteritems():
+ for k, v in paths_map.items():
for p in v:
paths.add(p)
return paths
self.path = os.path.join('stats', 'known_apks.txt')
self.apks = {}
if os.path.isfile(self.path):
- for line in file(self.path):
- t = line.rstrip().split(' ')
- if len(t) == 2:
- self.apks[t[0]] = (t[1], None)
- else:
- self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
+ with open(self.path, 'r', encoding='utf8') as f:
+ for line in f:
+ t = line.rstrip().split(' ')
+ if len(t) == 2:
+ self.apks[t[0]] = (t[1], None)
+ else:
+ self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
self.changed = False
def writeifchanged(self):
os.mkdir('stats')
lst = []
- for apk, app in self.apks.iteritems():
+ for apk, app in self.apks.items():
appid, added = app
line = apk + ' ' + appid
if added:
line += ' ' + time.strftime('%Y-%m-%d', added)
lst.append(line)
- with open(self.path, 'w') as f:
+ with open(self.path, 'w', encoding='utf8') as f:
for line in sorted(lst, key=natural_key):
f.write(line + '\n')
# with the most recent first.
def getlatest(self, num):
apps = {}
- for apk, app in self.apks.iteritems():
+ for apk, app in self.apks.items():
appid, added = app
if added:
if appid in apps:
apps[appid] = added
else:
apps[appid] = added
- sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
+ sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
lst = [app for app, _ in sortedapps]
lst.reverse()
return lst
class PopenResult:
- returncode = None
- output = ''
+ def __init__(self):
+ self.returncode = None
+ self.output = None
def SdkToolsPopen(commands, cwd=None, output=True):
cwd=cwd, output=output)
-def FDroidPopen(commands, cwd=None, output=True):
+def FDroidPopenBytes(commands, cwd=None, output=True, stderr_to_stdout=True):
"""
- Run a command and capture the possibly huge output.
+ Run a command and capture the possibly huge output as bytes.
:param commands: command and argument list like in subprocess.Popen
:param cwd: optionally specifies a working directory
"""
global env
+ if env is None:
+ set_FDroidPopen_env()
if cwd:
cwd = os.path.normpath(cwd)
logging.debug("Directory: %s" % cwd)
logging.debug("> %s" % ' '.join(commands))
+ stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
result = PopenResult()
p = None
try:
p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- except OSError, e:
+ stdout=subprocess.PIPE, stderr=stderr_param)
+ except OSError as e:
raise BuildException("OSError while trying to execute " +
' '.join(commands) + ': ' + str(e))
- stdout_queue = Queue.Queue()
+ if not stderr_to_stdout and options.verbose:
+ stderr_queue = Queue()
+ stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
+
+ while not stderr_reader.eof():
+ while not stderr_queue.empty():
+ line = stderr_queue.get()
+ sys.stderr.buffer.write(line)
+ sys.stderr.flush()
+
+ time.sleep(0.1)
+
+ stdout_queue = Queue()
stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
+ buf = io.BytesIO()
# Check the queue for output (until there is no more to get)
while not stdout_reader.eof():
line = stdout_queue.get()
if output and options.verbose:
# Output directly to console
- sys.stderr.write(line)
+ sys.stderr.buffer.write(line)
sys.stderr.flush()
- result.output += line
+ buf.write(line)
time.sleep(0.1)
result.returncode = p.wait()
+ result.output = buf.getvalue()
+ buf.close()
+ return result
+
+
+def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
+ """
+ Run a command and capture the possibly huge output as a str.
+
+ :param commands: command and argument list like in subprocess.Popen
+ :param cwd: optionally specifies a working directory
+ :returns: A PopenResult.
+ """
+ result = FDroidPopenBytes(commands, cwd, output, stderr_to_stdout)
+ result.output = result.output.decode('utf-8')
return result
gradle_line_matches = [
re.compile(r'^[\t ]*signingConfig [^ ]*$'),
re.compile(r'.*android\.signingConfigs\.[^{]*$'),
- re.compile(r'.*variant\.outputFile = .*'),
- re.compile(r'.*output\.outputFile = .*'),
re.compile(r'.*\.readLine\(.*'),
]
if 'build.gradle' in files:
path = os.path.join(root, 'build.gradle')
- with open(path, "r") as o:
+ with open(path, "r", encoding='utf8') as o:
lines = o.readlines()
changed = False
opened = 0
i = 0
- with open(path, "w") as o:
+ with open(path, "w", encoding='utf8') as o:
while i < len(lines):
line = lines[i]
i += 1
if propfile in files:
path = os.path.join(root, propfile)
- with open(path, "r") as o:
+ with open(path, "r", encoding='iso-8859-1') as o:
lines = o.readlines()
changed = False
- with open(path, "w") as o:
+ with open(path, "w", encoding='iso-8859-1') as o:
for line in lines:
if any(line.startswith(s) for s in ('key.store', 'key.alias')):
changed = True
logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
-def reset_env_path():
+def set_FDroidPopen_env(build=None):
+ '''
+ set up the environment variables for the build environment
+
+ There is only a weak standard, the variables used by gradle, so also set
+ up the most commonly used environment variables for SDK and NDK. Also, if
+ there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
+ '''
global env, orig_path
- env['PATH'] = orig_path
+ if env is None:
+ env = os.environ
+ orig_path = env['PATH']
+ for n in ['ANDROID_HOME', 'ANDROID_SDK']:
+ env[n] = config['sdk_path']
+ for k, v in config['java_paths'].items():
+ env['JAVA%s_HOME' % k] = v
+
+ missinglocale = True
+ for k, v in env.items():
+ if k == 'LANG' and v != 'C':
+ missinglocale = False
+ elif k == 'LC_ALL':
+ missinglocale = False
+ if missinglocale:
+ env['LANG'] = 'en_US.UTF-8'
-def add_to_env_path(path):
- global env
- paths = env['PATH'].split(os.pathsep)
- if path in paths:
- return
- paths.append(path)
- env['PATH'] = os.pathsep.join(paths)
+ if build is not None:
+ path = build.ndk_path()
+ paths = orig_path.split(os.pathsep)
+ if path not in paths:
+ paths = [path] + paths
+ env['PATH'] = os.pathsep.join(paths)
+ for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
+ env[n] = build.ndk_path()
def replace_config_vars(cmd, build):
- global env
cmd = cmd.replace('$$SDK$$', config['sdk_path'])
- # env['ANDROID_NDK'] is set in build_local right before prepare_source
- cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
+ cmd = cmd.replace('$$NDK$$', build.ndk_path())
cmd = cmd.replace('$$MVN3$$', config['mvn3'])
if build is not None:
- cmd = cmd.replace('$$COMMIT$$', build['commit'])
- cmd = cmd.replace('$$VERSION$$', build['version'])
- cmd = cmd.replace('$$VERCODE$$', build['vercode'])
+ cmd = cmd.replace('$$COMMIT$$', build.commit)
+ cmd = cmd.replace('$$VERSION$$', build.version)
+ cmd = cmd.replace('$$VERCODE$$', build.vercode)
return cmd
lines = []
if os.path.isfile(proppath):
- with open(proppath, "r") as o:
+ with open(proppath, "r", encoding='iso-8859-1') as o:
lines = o.readlines()
- with open(proppath, "w") as o:
+ with open(proppath, "w", encoding='iso-8859-1') as o:
placed = False
for line in lines:
if line.startswith('android.library.reference.%d=' % number):
if not placed:
o.write('android.library.reference.%d=%s\n' % (number, relpath))
-apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
+apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
def verify_apks(signed_apk, unsigned_apk, tmp_dir):
for meta_inf_file in meta_inf_files:
unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
- if subprocess.call(['jarsigner', '-verify', unsigned_apk]) != 0:
+ if subprocess.call([config['jarsigner'], '-verify', unsigned_apk]) != 0:
logging.info("...NOT verified - {0}".format(signed_apk))
return compare_apks(signed_apk, unsigned_apk, tmp_dir)
logging.info("...successfully verified")
'''generate a random password for when generating keys'''
h = hashlib.sha256()
h.update(os.urandom(16)) # salt
- h.update(bytes(socket.getfqdn()))
- return h.digest().encode('base64').strip()
+ h.update(socket.getfqdn().encode('utf-8'))
+ passwd = base64.b64encode(h.digest()).strip()
+ return passwd.decode('utf-8')
def genkeystore(localconfig):
write_password_file("keystorepass", localconfig['keystorepass'])
write_password_file("keypass", localconfig['keypass'])
- p = FDroidPopen(['keytool', '-genkey',
+ p = FDroidPopen([config['keytool'], '-genkey',
'-keystore', localconfig['keystore'],
'-alias', localconfig['repo_keyalias'],
'-keyalg', 'RSA', '-keysize', '4096',
raise BuildException("Failed to generate key", p.output)
os.chmod(localconfig['keystore'], 0o0600)
# now show the lovely key that was just generated
- p = FDroidPopen(['keytool', '-list', '-v',
+ p = FDroidPopen([config['keytool'], '-list', '-v',
'-keystore', localconfig['keystore'],
'-alias', localconfig['repo_keyalias'],
'-storepass:file', config['keystorepassfile']])
if value is None:
origkey = key + '_orig'
value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
- with open('config.py', 'r') as f:
+ with open('config.py', 'r', encoding='utf8') as f:
data = f.read()
pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
repl = '\n' + key + ' = "' + value + '"'
# make sure the file ends with a carraige return
if not re.match('\n$', data):
data += '\n'
- with open('config.py', 'w') as f:
+ with open('config.py', 'w', encoding='utf8') as f:
f.writelines(data)
repos = []
for root, dirs, files in os.walk(os.getcwd()):
for d in dirs:
- print 'checking', root, 'for', d
+ print('checking', root, 'for', d)
if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
# standard parts of an fdroid repo, so never packageNames
continue
~ianmdlvl / fdroidserver.git / blobdiff