chiark / gitweb /
FDroidPopen must have a locale to support UTF-8 filenames
[fdroidserver.git] / fdroidserver / common.py
index ed8fa1f99cec5b15f38d5655392c8e1f4f1f8d30..2c2bb4e062b77472f7a2deb67d107897de7bc567 100644 (file)
@@ -1,4 +1,4 @@
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
 #
 # common.py - part of the FDroid server tools
 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
@@ -58,12 +58,13 @@ default_config = {
         'r9b': None,
         'r10e': "$ANDROID_NDK",
     },
-    'build_tools': "23.0.2",
+    'build_tools': "24.0.0",
+    'force_build_tools': False,
     'java_paths': None,
     'ant': "ant",
     'mvn3': "mvn",
     'gradle': 'gradle',
-    'accepted_formats': ['txt', 'yaml'],
+    'accepted_formats': ['txt', 'yml'],
     'sync_from_local_copy_dir': False,
     'per_app_repos': False,
     'make_current_version_link': True,
@@ -128,10 +129,16 @@ def fill_config_defaults(thisconfig):
     # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
     if thisconfig['java_paths'] is None:
         thisconfig['java_paths'] = dict()
-        for d in sorted(glob.glob('/usr/lib/jvm/j*[6-9]*')
-                        + glob.glob('/usr/java/jdk1.[6-9]*')
-                        + glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
-                        + glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')):
+        pathlist = []
+        pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
+        pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
+        pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
+        pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
+        if os.getenv('JAVA_HOME') is not None:
+            pathlist += os.getenv('JAVA_HOME')
+        if os.getenv('PROGRAMFILES') is not None:
+            pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
+        for d in sorted(pathlist):
             if os.path.islink(d):
                 continue
             j = os.path.basename(d)
@@ -139,6 +146,7 @@ def fill_config_defaults(thisconfig):
             for regex in [
                     r'^1\.([6-9])\.0\.jdk$',  # OSX
                     r'^jdk1\.([6-9])\.0_[0-9]+.jdk$',  # OSX and Oracle tarball
+                    r'^jdk1\.([6-9])\.0_[0-9]+$',  # Oracle Windows
                     r'^jdk([6-9])-openjdk$',  # Arch
                     r'^java-([6-9])-openjdk$',  # Arch
                     r'^java-([6-9])-jdk$',  # Arch (oracle)
@@ -177,35 +185,39 @@ def fill_config_defaults(thisconfig):
 
 
 def regsub_file(pattern, repl, path):
-    with open(path, 'r') as f:
+    with open(path, 'rb') as f:
         text = f.read()
-    text = re.sub(pattern, repl, text)
-    with open(path, 'w') as f:
+    text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
+    with open(path, 'wb') as f:
         f.write(text)
 
 
 def read_config(opts, config_file='config.py'):
     """Read the repository config
 
-    The config is read from config_file, which is in the current directory when
-    any of the repo management commands are used.
+    The config is read from config_file, which is in the current
+    directory when any of the repo management commands are used. If
+    there is a local metadata file in the git repo, then config.py is
+    not required, just use defaults.
+
     """
-    global config, options, env, orig_path
+    global config, options
 
     if config is not None:
         return config
-    if not os.path.isfile(config_file):
-        logging.critical("Missing config file - is this a repo directory?")
-        sys.exit(2)
 
     options = opts
 
     config = {}
 
-    logging.debug("Reading %s" % config_file)
-    with io.open(config_file, "rb") as f:
-        code = compile(f.read(), config_file, 'exec')
-        exec(code, None, config)
+    if os.path.isfile(config_file):
+        logging.debug("Reading %s" % config_file)
+        with io.open(config_file, "rb") as f:
+            code = compile(f.read(), config_file, 'exec')
+            exec(code, None, config)
+    elif len(get_local_metadata_files()) == 0:
+        logging.critical("Missing config file - is this a repo directory?")
+        sys.exit(2)
 
     # smartcardoptions must be a list since its command line args for Popen
     if 'smartcardoptions' in config:
@@ -224,16 +236,6 @@ def read_config(opts, config_file='config.py'):
 
     fill_config_defaults(config)
 
-    # There is no standard, so just set up the most common environment
-    # variables
-    env = os.environ
-    orig_path = env['PATH']
-    for n in ['ANDROID_HOME', 'ANDROID_SDK']:
-        env[n] = config['sdk_path']
-
-    for k, v in config['java_paths'].items():
-        env['JAVA%s_HOME' % k] = v
-
     for k in ["keystorepass", "keypass"]:
         if k in config:
             write_password_file(k)
@@ -338,13 +340,23 @@ def write_password_file(pwtype, password=None):
     filename = '.fdroid.' + pwtype + '.txt'
     fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
     if password is None:
-        os.write(fd, config[pwtype])
+        os.write(fd, config[pwtype].encode('utf-8'))
     else:
-        os.write(fd, password)
+        os.write(fd, password.encode('utf-8'))
     os.close(fd)
     config[pwtype + 'file'] = filename
 
 
+def get_local_metadata_files():
+    '''get any metadata files local to an app's source repo
+
+    This tries to ignore anything that does not count as app metdata,
+    including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
+
+    '''
+    return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
+
+
 # Given the arguments in the form of multiple appid:[vc] strings, this returns
 # a dictionary with the set of vercodes specified for each package.
 def read_pkg_args(args, allow_vercodes=False):
@@ -531,6 +543,7 @@ class vcs:
         # automatically if either of those things changes.
         fdpath = os.path.join(self.local, '..',
                               '.fdroidvcs-' + os.path.basename(self.local))
+        fdpath = os.path.normpath(fdpath)
         cdata = self.repotype() + ' ' + self.remote
         writeback = True
         deleterepo = False
@@ -562,7 +575,8 @@ class vcs:
 
         # If necessary, write the .fdroidvcs file.
         if writeback and not self.clone_failed:
-            with open(fdpath, 'w') as f:
+            os.makedirs(os.path.dirname(fdpath), exist_ok=True)
+            with open(fdpath, 'w+') as f:
                 f.write(cdata)
 
         if exc is not None:
@@ -587,14 +601,8 @@ class vcs:
                 rtags.append(tag)
         return rtags
 
-    def latesttags(self, tags, number):
-        """Get the most recent tags in a given list.
-
-        :param tags: a list of tags
-        :param number: the number to return
-        :returns: A list containing the most recent tags in the provided
-                  list, up to the maximum number given.
-        """
+    # Get a list of all the known tags, sorted from newest to oldest
+    def latesttags(self):
         raise VCSException('latesttags not supported for this vcs type')
 
     # Get current commit reference (hash, revision, etc)
@@ -702,21 +710,18 @@ class vcs_git(vcs):
         p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
         return p.output.splitlines()
 
-    def latesttags(self, tags, number):
+    tag_format = re.compile(r'tag: ([^),]*)')
+
+    def latesttags(self):
         self.checkrepo()
-        tl = []
-        for tag in tags:
-            p = FDroidPopen(
-                ['git', 'show', '--format=format:%ct', '-s', tag],
-                cwd=self.local, output=False)
-            # Timestamp is on the last line. For a normal tag, it's the only
-            # line, but for annotated tags, the rest of the info precedes it.
-            ts = int(p.output.splitlines()[-1])
-            tl.append((ts, tag))
-        latest = []
-        for _, t in sorted(tl)[-number:]:
-            latest.append(t)
-        return latest
+        p = FDroidPopen(['git', 'log', '--tags',
+                         '--simplify-by-decoration', '--pretty=format:%d'],
+                        cwd=self.local, output=False)
+        tags = []
+        for line in p.output.splitlines():
+            for tag in self.tag_format.findall(line):
+                tags.append(tag)
+        return tags
 
 
 class vcs_gitsvn(vcs):
@@ -952,7 +957,7 @@ def retrieve_string(app_dir, string, xmlfiles=None):
         if element.text is None:
             return ""
         s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
-        return s.strip()
+        return s.decode('utf-8').strip()
 
     for path in xmlfiles:
         if not os.path.isfile(path):
@@ -1000,7 +1005,7 @@ def fetch_real_name(app_dir, flavours):
             continue
         if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
             continue
-        label = app.attrib["{http://schemas.android.com/apk/res/android}label"].encode('utf-8')
+        label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
         result = retrieve_string_singleline(app_dir, label)
         if result:
             result = result.strip()
@@ -1013,15 +1018,16 @@ def get_library_references(root_dir):
     proppath = os.path.join(root_dir, 'project.properties')
     if not os.path.isfile(proppath):
         return libraries
-    for line in file(proppath):
-        if not line.startswith('android.library.reference.'):
-            continue
-        path = line.split('=')[1].strip()
-        relpath = os.path.join(root_dir, path)
-        if not os.path.isdir(relpath):
-            continue
-        logging.debug("Found subproject at %s" % path)
-        libraries.append(path)
+    with open(proppath, 'r', encoding='iso-8859-1') as f:
+        for line in f:
+            if not line.startswith('android.library.reference.'):
+                continue
+            path = line.split('=')[1].strip()
+            relpath = os.path.join(root_dir, path)
+            if not os.path.isdir(relpath):
+                continue
+            logging.debug("Found subproject at %s" % path)
+            libraries.append(path)
     return libraries
 
 
@@ -1087,38 +1093,39 @@ def parse_androidmanifests(paths, app):
         package = None
 
         if gradle:
-            for line in file(path):
-                if gradle_comment.match(line):
-                    continue
-                # Grab first occurence of each to avoid running into
-                # alternative flavours and builds.
-                if not package:
-                    matches = psearch_g(line)
-                    if matches:
-                        s = matches.group(2)
-                        if app_matches_packagename(app, s):
-                            package = s
-                if not version:
-                    matches = vnsearch_g(line)
-                    if matches:
-                        version = matches.group(2)
-                if not vercode:
-                    matches = vcsearch_g(line)
-                    if matches:
-                        vercode = matches.group(1)
+            with open(path, 'r') as f:
+                for line in f:
+                    if gradle_comment.match(line):
+                        continue
+                    # Grab first occurence of each to avoid running into
+                    # alternative flavours and builds.
+                    if not package:
+                        matches = psearch_g(line)
+                        if matches:
+                            s = matches.group(2)
+                            if app_matches_packagename(app, s):
+                                package = s
+                    if not version:
+                        matches = vnsearch_g(line)
+                        if matches:
+                            version = matches.group(2)
+                    if not vercode:
+                        matches = vcsearch_g(line)
+                        if matches:
+                            vercode = matches.group(1)
         else:
             try:
                 xml = parse_xml(path)
                 if "package" in xml.attrib:
-                    s = xml.attrib["package"].encode('utf-8')
+                    s = xml.attrib["package"]
                     if app_matches_packagename(app, s):
                         package = s
                 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
-                    version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"].encode('utf-8')
+                    version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
                     base_dir = os.path.dirname(path)
                     version = retrieve_string_singleline(base_dir, version)
                 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
-                    a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"].encode('utf-8')
+                    a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
                     if string_is_integer(a):
                         vercode = a
             except Exception:
@@ -1200,7 +1207,8 @@ class BuildException(FDroidException):
 # it, which may be a subdirectory of the actual project. If you want the base
 # directory of the project, pass 'basepath=True'.
 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
-              raw=False, prepare=True, preponly=False, refresh=True):
+              raw=False, prepare=True, preponly=False, refresh=True,
+              build=None):
 
     number = None
     subdir = None
@@ -1249,7 +1257,7 @@ def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
     if prepare:
 
         if srclib["Prepare"]:
-            cmd = replace_config_vars(srclib["Prepare"], None)
+            cmd = replace_config_vars(srclib["Prepare"], build)
 
             p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
             if p.returncode != 0:
@@ -1326,7 +1334,8 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
     if build.srclibs:
         logging.info("Collecting source libraries")
         for lib in build.srclibs:
-            srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
+            srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
+                                         refresh=refresh, build=build))
 
     for name, number, libpath in srclibpaths:
         place_srclib(root_dir, int(number) if number else None, libpath)
@@ -1348,7 +1357,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
         props = ""
         if os.path.isfile(path):
             logging.info("Updating local.properties file at %s" % path)
-            with open(path, 'r') as f:
+            with open(path, 'r', encoding='iso-8859-1') as f:
                 props += f.read()
             props += '\n'
         else:
@@ -1370,7 +1379,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
         # Add java.encoding if necessary
         if build.encoding:
             props += "java.encoding=%s\n" % build.encoding
-        with open(path, 'w') as f:
+        with open(path, 'w', encoding='iso-8859-1') as f:
             f.write(props)
 
     flavours = []
@@ -1531,12 +1540,13 @@ class KnownApks:
         self.path = os.path.join('stats', 'known_apks.txt')
         self.apks = {}
         if os.path.isfile(self.path):
-            for line in file(self.path):
-                t = line.rstrip().split(' ')
-                if len(t) == 2:
-                    self.apks[t[0]] = (t[1], None)
-                else:
-                    self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
+            with open(self.path, 'r', encoding='utf8') as f:
+                for line in f:
+                    t = line.rstrip().split(' ')
+                    if len(t) == 2:
+                        self.apks[t[0]] = (t[1], None)
+                    else:
+                        self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
         self.changed = False
 
     def writeifchanged(self):
@@ -1554,7 +1564,7 @@ class KnownApks:
                 line += ' ' + time.strftime('%Y-%m-%d', added)
             lst.append(line)
 
-        with open(self.path, 'w') as f:
+        with open(self.path, 'w', encoding='utf8') as f:
             for line in sorted(lst, key=natural_key):
                 f.write(line + '\n')
 
@@ -1609,8 +1619,9 @@ def isApkDebuggable(apkfile, config):
 
 
 class PopenResult:
-    returncode = None
-    output = ''
+    def __init__(self):
+        self.returncode = None
+        self.output = None
 
 
 def SdkToolsPopen(commands, cwd=None, output=True):
@@ -1625,9 +1636,9 @@ def SdkToolsPopen(commands, cwd=None, output=True):
                        cwd=cwd, output=output)
 
 
-def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
+def FDroidPopenBytes(commands, cwd=None, output=True, stderr_to_stdout=True):
     """
-    Run a command and capture the possibly huge output.
+    Run a command and capture the possibly huge output as bytes.
 
     :param commands: command and argument list like in subprocess.Popen
     :param cwd: optionally specifies a working directory
@@ -1635,6 +1646,8 @@ def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
     """
 
     global env
+    if env is None:
+        set_FDroidPopen_env()
 
     if cwd:
         cwd = os.path.normpath(cwd)
@@ -1658,13 +1671,14 @@ def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
         while not stderr_reader.eof():
             while not stderr_queue.empty():
                 line = stderr_queue.get()
-                sys.stderr.write(line)
+                sys.stderr.buffer.write(line)
                 sys.stderr.flush()
 
             time.sleep(0.1)
 
     stdout_queue = Queue()
     stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
+    buf = io.BytesIO()
 
     # Check the queue for output (until there is no more to get)
     while not stdout_reader.eof():
@@ -1674,11 +1688,26 @@ def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
                 # Output directly to console
                 sys.stderr.buffer.write(line)
                 sys.stderr.flush()
-            result.output += line.decode('utf-8')
+            buf.write(line)
 
         time.sleep(0.1)
 
     result.returncode = p.wait()
+    result.output = buf.getvalue()
+    buf.close()
+    return result
+
+
+def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
+    """
+    Run a command and capture the possibly huge output as a str.
+
+    :param commands: command and argument list like in subprocess.Popen
+    :param cwd: optionally specifies a working directory
+    :returns: A PopenResult.
+    """
+    result = FDroidPopenBytes(commands, cwd, output, stderr_to_stdout)
+    result.output = result.output.decode('utf-8')
     return result
 
 
@@ -1696,14 +1725,14 @@ def remove_signing_keys(build_dir):
         if 'build.gradle' in files:
             path = os.path.join(root, 'build.gradle')
 
-            with open(path, "r") as o:
+            with open(path, "r", encoding='utf8') as o:
                 lines = o.readlines()
 
             changed = False
 
             opened = 0
             i = 0
-            with open(path, "w") as o:
+            with open(path, "w", encoding='utf8') as o:
                 while i < len(lines):
                     line = lines[i]
                     i += 1
@@ -1743,12 +1772,12 @@ def remove_signing_keys(build_dir):
             if propfile in files:
                 path = os.path.join(root, propfile)
 
-                with open(path, "r") as o:
+                with open(path, "r", encoding='iso-8859-1') as o:
                     lines = o.readlines()
 
                 changed = False
 
-                with open(path, "w") as o:
+                with open(path, "w", encoding='iso-8859-1') as o:
                     for line in lines:
                         if any(line.startswith(s) for s in ('key.store', 'key.alias')):
                             changed = True
@@ -1760,25 +1789,46 @@ def remove_signing_keys(build_dir):
                     logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
 
 
-def reset_env_path():
+def set_FDroidPopen_env(build=None):
+    '''
+    set up the environment variables for the build environment
+
+    There is only a weak standard, the variables used by gradle, so also set
+    up the most commonly used environment variables for SDK and NDK.  Also, if
+    there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
+    '''
     global env, orig_path
-    env['PATH'] = orig_path
 
+    if env is None:
+        env = os.environ
+        orig_path = env['PATH']
+        for n in ['ANDROID_HOME', 'ANDROID_SDK']:
+            env[n] = config['sdk_path']
+        for k, v in config['java_paths'].items():
+            env['JAVA%s_HOME' % k] = v
+
+    missinglocale = True
+    for k, v in env.items():
+        if k == 'LANG' and v != 'C':
+            missinglocale = False
+        elif k == 'LC_ALL':
+            missinglocale = False
+    if missinglocale:
+        env['LANG'] = 'en_US.UTF-8'
 
-def add_to_env_path(path):
-    global env
-    paths = env['PATH'].split(os.pathsep)
-    if path in paths:
-        return
-    paths.append(path)
-    env['PATH'] = os.pathsep.join(paths)
+    if build is not None:
+        path = build.ndk_path()
+        paths = orig_path.split(os.pathsep)
+        if path not in paths:
+            paths = [path] + paths
+            env['PATH'] = os.pathsep.join(paths)
+        for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
+            env[n] = build.ndk_path()
 
 
 def replace_config_vars(cmd, build):
-    global env
     cmd = cmd.replace('$$SDK$$', config['sdk_path'])
-    # env['ANDROID_NDK'] is set in build_local right before prepare_source
-    cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
+    cmd = cmd.replace('$$NDK$$', build.ndk_path())
     cmd = cmd.replace('$$MVN3$$', config['mvn3'])
     if build is not None:
         cmd = cmd.replace('$$COMMIT$$', build.commit)
@@ -1795,10 +1845,10 @@ def place_srclib(root_dir, number, libpath):
 
     lines = []
     if os.path.isfile(proppath):
-        with open(proppath, "r") as o:
+        with open(proppath, "r", encoding='iso-8859-1') as o:
             lines = o.readlines()
 
-    with open(proppath, "w") as o:
+    with open(proppath, "w", encoding='iso-8859-1') as o:
         placed = False
         for line in lines:
             if line.startswith('android.library.reference.%d=' % number):
@@ -1809,7 +1859,7 @@ def place_srclib(root_dir, number, libpath):
         if not placed:
             o.write('android.library.reference.%d=%s\n' % (number, relpath))
 
-apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
+apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
 
 
 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
@@ -1966,7 +2016,7 @@ def write_to_config(thisconfig, key, value=None):
     if value is None:
         origkey = key + '_orig'
         value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
-    with open('config.py', 'r') as f:
+    with open('config.py', 'r', encoding='utf8') as f:
         data = f.read()
     pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
     repl = '\n' + key + ' = "' + value + '"'
@@ -1977,7 +2027,7 @@ def write_to_config(thisconfig, key, value=None):
     # make sure the file ends with a carraige return
     if not re.match('\n$', data):
         data += '\n'
-    with open('config.py', 'w') as f:
+    with open('config.py', 'w', encoding='utf8') as f:
         f.writelines(data)