-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
#
# common.py - part of the FDroid server tools
# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
'sdk_path': "$ANDROID_HOME",
'ndk_paths': {
'r9b': None,
- 'r10e': "$ANDROID_NDK",
+ 'r10e': None,
+ 'r12b': "$ANDROID_NDK",
},
- 'build_tools': "23.0.2",
+ 'build_tools': "24.0.0",
+ 'force_build_tools': False,
'java_paths': None,
'ant': "ant",
'mvn3': "mvn",
'gradle': 'gradle',
- 'accepted_formats': ['txt', 'yaml'],
+ 'accepted_formats': ['txt', 'yml'],
'sync_from_local_copy_dir': False,
'per_app_repos': False,
'make_current_version_link': True,
# find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
if thisconfig['java_paths'] is None:
thisconfig['java_paths'] = dict()
- for d in sorted(glob.glob('/usr/lib/jvm/j*[6-9]*')
- + glob.glob('/usr/java/jdk1.[6-9]*')
- + glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
- + glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')):
+ pathlist = []
+ pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
+ pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
+ pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
+ pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
+ if os.getenv('JAVA_HOME') is not None:
+ pathlist += os.getenv('JAVA_HOME')
+ if os.getenv('PROGRAMFILES') is not None:
+ pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
+ for d in sorted(pathlist):
if os.path.islink(d):
continue
j = os.path.basename(d)
for regex in [
r'^1\.([6-9])\.0\.jdk$', # OSX
r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
+ r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
r'^jdk([6-9])-openjdk$', # Arch
r'^java-([6-9])-openjdk$', # Arch
r'^java-([6-9])-jdk$', # Arch (oracle)
def regsub_file(pattern, repl, path):
- with open(path, 'r') as f:
+ with open(path, 'rb') as f:
text = f.read()
- text = re.sub(pattern, repl, text)
- with open(path, 'w') as f:
+ text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
+ with open(path, 'wb') as f:
f.write(text)
def read_config(opts, config_file='config.py'):
"""Read the repository config
- The config is read from config_file, which is in the current directory when
- any of the repo management commands are used.
+ The config is read from config_file, which is in the current
+ directory when any of the repo management commands are used. If
+ there is a local metadata file in the git repo, then config.py is
+ not required, just use defaults.
+
"""
- global config, options, env, orig_path
+ global config, options
if config is not None:
return config
- if not os.path.isfile(config_file):
- logging.critical("Missing config file - is this a repo directory?")
- sys.exit(2)
options = opts
config = {}
- logging.debug("Reading %s" % config_file)
- with io.open(config_file, "rb") as f:
- code = compile(f.read(), config_file, 'exec')
- exec(code, None, config)
+ if os.path.isfile(config_file):
+ logging.debug("Reading %s" % config_file)
+ with io.open(config_file, "rb") as f:
+ code = compile(f.read(), config_file, 'exec')
+ exec(code, None, config)
+ elif len(get_local_metadata_files()) == 0:
+ logging.critical("Missing config file - is this a repo directory?")
+ sys.exit(2)
# smartcardoptions must be a list since its command line args for Popen
if 'smartcardoptions' in config:
fill_config_defaults(config)
- # There is no standard, so just set up the most common environment
- # variables
- env = os.environ
- orig_path = env['PATH']
- for n in ['ANDROID_HOME', 'ANDROID_SDK']:
- env[n] = config['sdk_path']
-
- for k, v in config['java_paths'].items():
- env['JAVA%s_HOME' % k] = v
-
for k in ["keystorepass", "keypass"]:
if k in config:
write_password_file(k)
filename = '.fdroid.' + pwtype + '.txt'
fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
if password is None:
- os.write(fd, config[pwtype])
+ os.write(fd, config[pwtype].encode('utf-8'))
else:
- os.write(fd, password)
+ os.write(fd, password.encode('utf-8'))
os.close(fd)
config[pwtype + 'file'] = filename
+def get_local_metadata_files():
+ '''get any metadata files local to an app's source repo
+
+ This tries to ignore anything that does not count as app metdata,
+ including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
+
+ '''
+ return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
+
+
# Given the arguments in the form of multiple appid:[vc] strings, this returns
# a dictionary with the set of vercodes specified for each package.
def read_pkg_args(args, allow_vercodes=False):
# automatically if either of those things changes.
fdpath = os.path.join(self.local, '..',
'.fdroidvcs-' + os.path.basename(self.local))
+ fdpath = os.path.normpath(fdpath)
cdata = self.repotype() + ' ' + self.remote
writeback = True
deleterepo = False
# If necessary, write the .fdroidvcs file.
if writeback and not self.clone_failed:
- with open(fdpath, 'w') as f:
+ os.makedirs(os.path.dirname(fdpath), exist_ok=True)
+ with open(fdpath, 'w+') as f:
f.write(cdata)
if exc is not None:
rtags.append(tag)
return rtags
- def latesttags(self, tags, number):
- """Get the most recent tags in a given list.
-
- :param tags: a list of tags
- :param number: the number to return
- :returns: A list containing the most recent tags in the provided
- list, up to the maximum number given.
- """
+ # Get a list of all the known tags, sorted from newest to oldest
+ def latesttags(self):
raise VCSException('latesttags not supported for this vcs type')
# Get current commit reference (hash, revision, etc)
p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
return p.output.splitlines()
- def latesttags(self, tags, number):
+ tag_format = re.compile(r'tag: ([^),]*)')
+
+ def latesttags(self):
self.checkrepo()
- tl = []
- for tag in tags:
- p = FDroidPopen(
- ['git', 'show', '--format=format:%ct', '-s', tag],
- cwd=self.local, output=False)
- # Timestamp is on the last line. For a normal tag, it's the only
- # line, but for annotated tags, the rest of the info precedes it.
- ts = int(p.output.splitlines()[-1])
- tl.append((ts, tag))
- latest = []
- for _, t in sorted(tl)[-number:]:
- latest.append(t)
- return latest
+ p = FDroidPopen(['git', 'log', '--tags',
+ '--simplify-by-decoration', '--pretty=format:%d'],
+ cwd=self.local, output=False)
+ tags = []
+ for line in p.output.splitlines():
+ for tag in self.tag_format.findall(line):
+ tags.append(tag)
+ return tags
class vcs_gitsvn(vcs):
proppath = os.path.join(root_dir, 'project.properties')
if not os.path.isfile(proppath):
return libraries
- with open(proppath, 'r') as f:
+ with open(proppath, 'r', encoding='iso-8859-1') as f:
for line in f:
if not line.startswith('android.library.reference.'):
continue
# it, which may be a subdirectory of the actual project. If you want the base
# directory of the project, pass 'basepath=True'.
def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
- raw=False, prepare=True, preponly=False, refresh=True):
+ raw=False, prepare=True, preponly=False, refresh=True,
+ build=None):
number = None
subdir = None
if prepare:
if srclib["Prepare"]:
- cmd = replace_config_vars(srclib["Prepare"], None)
+ cmd = replace_config_vars(srclib["Prepare"], build)
p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
if p.returncode != 0:
if build.srclibs:
logging.info("Collecting source libraries")
for lib in build.srclibs:
- srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
+ srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
+ refresh=refresh, build=build))
for name, number, libpath in srclibpaths:
place_srclib(root_dir, int(number) if number else None, libpath)
props = ""
if os.path.isfile(path):
logging.info("Updating local.properties file at %s" % path)
- with open(path, 'r') as f:
+ with open(path, 'r', encoding='iso-8859-1') as f:
props += f.read()
props += '\n'
else:
props += "sdk.dir=%s\n" % config['sdk_path']
props += "sdk-location=%s\n" % config['sdk_path']
ndk_path = build.ndk_path()
- if ndk_path:
+ # if for any reason the path isn't valid or the directory
+ # doesn't exist, some versions of Gradle will error with a
+ # cryptic message (even if the NDK is not even necessary).
+ # https://gitlab.com/fdroid/fdroidserver/issues/171
+ if ndk_path and os.path.exists(ndk_path):
# Add ndk location
props += "ndk.dir=%s\n" % ndk_path
props += "ndk-location=%s\n" % ndk_path
# Add java.encoding if necessary
if build.encoding:
props += "java.encoding=%s\n" % build.encoding
- with open(path, 'w') as f:
+ with open(path, 'w', encoding='iso-8859-1') as f:
f.write(props)
flavours = []
self.path = os.path.join('stats', 'known_apks.txt')
self.apks = {}
if os.path.isfile(self.path):
- with open(self.path, 'r') as f:
+ with open(self.path, 'r', encoding='utf8') as f:
for line in f:
t = line.rstrip().split(' ')
if len(t) == 2:
line += ' ' + time.strftime('%Y-%m-%d', added)
lst.append(line)
- with open(self.path, 'w') as f:
+ with open(self.path, 'w', encoding='utf8') as f:
for line in sorted(lst, key=natural_key):
f.write(line + '\n')
# Record an apk (if it's new, otherwise does nothing)
# Returns the date it was added.
- def recordapk(self, apk, app):
+ def recordapk(self, apk, app, default_date=None):
if apk not in self.apks:
- self.apks[apk] = (app, time.gmtime(time.time()))
+ if default_date is None:
+ default_date = time.gmtime(time.time())
+ self.apks[apk] = (app, default_date)
self.changed = True
_, added = self.apks[apk]
return added
class PopenResult:
- returncode = None
- output = ''
+ def __init__(self):
+ self.returncode = None
+ self.output = None
def SdkToolsPopen(commands, cwd=None, output=True):
cwd=cwd, output=output)
-def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
+def FDroidPopenBytes(commands, cwd=None, output=True, stderr_to_stdout=True):
"""
- Run a command and capture the possibly huge output.
+ Run a command and capture the possibly huge output as bytes.
:param commands: command and argument list like in subprocess.Popen
:param cwd: optionally specifies a working directory
"""
global env
+ if env is None:
+ set_FDroidPopen_env()
if cwd:
cwd = os.path.normpath(cwd)
while not stderr_reader.eof():
while not stderr_queue.empty():
line = stderr_queue.get()
- sys.stderr.write(line)
+ sys.stderr.buffer.write(line)
sys.stderr.flush()
time.sleep(0.1)
stdout_queue = Queue()
stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
+ buf = io.BytesIO()
# Check the queue for output (until there is no more to get)
while not stdout_reader.eof():
# Output directly to console
sys.stderr.buffer.write(line)
sys.stderr.flush()
- result.output += line.decode('utf-8')
+ buf.write(line)
time.sleep(0.1)
result.returncode = p.wait()
+ result.output = buf.getvalue()
+ buf.close()
+ return result
+
+
+def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
+ """
+ Run a command and capture the possibly huge output as a str.
+
+ :param commands: command and argument list like in subprocess.Popen
+ :param cwd: optionally specifies a working directory
+ :returns: A PopenResult.
+ """
+ result = FDroidPopenBytes(commands, cwd, output, stderr_to_stdout)
+ result.output = result.output.decode('utf-8')
return result
if 'build.gradle' in files:
path = os.path.join(root, 'build.gradle')
- with open(path, "r") as o:
+ with open(path, "r", encoding='utf8') as o:
lines = o.readlines()
changed = False
opened = 0
i = 0
- with open(path, "w") as o:
+ with open(path, "w", encoding='utf8') as o:
while i < len(lines):
line = lines[i]
i += 1
if propfile in files:
path = os.path.join(root, propfile)
- with open(path, "r") as o:
+ with open(path, "r", encoding='iso-8859-1') as o:
lines = o.readlines()
changed = False
- with open(path, "w") as o:
+ with open(path, "w", encoding='iso-8859-1') as o:
for line in lines:
if any(line.startswith(s) for s in ('key.store', 'key.alias')):
changed = True
logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
-def reset_env_path():
+def set_FDroidPopen_env(build=None):
+ '''
+ set up the environment variables for the build environment
+
+ There is only a weak standard, the variables used by gradle, so also set
+ up the most commonly used environment variables for SDK and NDK. Also, if
+ there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
+ '''
global env, orig_path
- env['PATH'] = orig_path
+ if env is None:
+ env = os.environ
+ orig_path = env['PATH']
+ for n in ['ANDROID_HOME', 'ANDROID_SDK']:
+ env[n] = config['sdk_path']
+ for k, v in config['java_paths'].items():
+ env['JAVA%s_HOME' % k] = v
+
+ missinglocale = True
+ for k, v in env.items():
+ if k == 'LANG' and v != 'C':
+ missinglocale = False
+ elif k == 'LC_ALL':
+ missinglocale = False
+ if missinglocale:
+ env['LANG'] = 'en_US.UTF-8'
-def add_to_env_path(path):
- global env
- paths = env['PATH'].split(os.pathsep)
- if path in paths:
- return
- paths.append(path)
- env['PATH'] = os.pathsep.join(paths)
+ if build is not None:
+ path = build.ndk_path()
+ paths = orig_path.split(os.pathsep)
+ if path not in paths:
+ paths = [path] + paths
+ env['PATH'] = os.pathsep.join(paths)
+ for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
+ env[n] = build.ndk_path()
def replace_config_vars(cmd, build):
- global env
cmd = cmd.replace('$$SDK$$', config['sdk_path'])
- # env['ANDROID_NDK'] is set in build_local right before prepare_source
- cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
+ cmd = cmd.replace('$$NDK$$', build.ndk_path())
cmd = cmd.replace('$$MVN3$$', config['mvn3'])
if build is not None:
cmd = cmd.replace('$$COMMIT$$', build.commit)
lines = []
if os.path.isfile(proppath):
- with open(proppath, "r") as o:
+ with open(proppath, "r", encoding='iso-8859-1') as o:
lines = o.readlines()
- with open(proppath, "w") as o:
+ with open(proppath, "w", encoding='iso-8859-1') as o:
placed = False
for line in lines:
if line.startswith('android.library.reference.%d=' % number):
if not placed:
o.write('android.library.reference.%d=%s\n' % (number, relpath))
-apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
+apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
def verify_apks(signed_apk, unsigned_apk, tmp_dir):
if value is None:
origkey = key + '_orig'
value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
- with open('config.py', 'r') as f:
+ with open('config.py', 'r', encoding='utf8') as f:
data = f.read()
pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
repl = '\n' + key + ' = "' + value + '"'
# make sure the file ends with a carraige return
if not re.match('\n$', data):
data += '\n'
- with open('config.py', 'w') as f:
+ with open('config.py', 'w', encoding='utf8') as f:
f.writelines(data)