Add asynchronous filereader, fix python3 lockups

[fdroidserver.git] / fdroidserver / common.py

diff --git a/fdroidserver/common.py b/fdroidserver/common.py
index d3c7f176fde191567f8512bc26bcdaea1de244b7..ae848df5a96ee94fb75cae4d554bafecaf351f40 100644 (file)
--- a/fdroidserver/common.py
+++ b/fdroidserver/common.py
@@ -30,7 +30,6 @@ import subprocess
 import time
 import operator
 import Queue
-import threading
 import logging
 import hashlib
 import socket
@@ -40,6 +39,8 @@ from distutils.version import LooseVersion
 from zipfile import ZipFile
 
 import metadata
+from fdroidserver.asynchronousfilereader import AsynchronousFileReader
+
 
 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
 
@@ -53,12 +54,13 @@ default_config = {
     'sdk_path': "$ANDROID_HOME",
     'ndk_paths': {
         'r9b': None,
-        'r10e': "$ANDROID_NDK"
+        'r10e': "$ANDROID_NDK",
     },
-    'build_tools': "23.0.0",
+    'build_tools': "23.0.1",
     'ant': "ant",
     'mvn3': "mvn",
     'gradle': 'gradle',
+    'accepted_formats': ['txt', 'yaml'],
     'sync_from_local_copy_dir': False,
     'per_app_repos': False,
     'make_current_version_link': True,
@@ -74,7 +76,7 @@ default_config = {
     'smartcardoptions': [],
     'char_limits': {
         'Summary': 80,
-        'Description': 4000
+        'Description': 4000,
     },
     'keyaliases': {},
     'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
@@ -90,6 +92,13 @@ default_config = {
 }
 
 
+def setup_global_opts(parser):
+    parser.add_argument("-v", "--verbose", action="store_true", default=False,
+                        help="Spew out even more information than normal")
+    parser.add_argument("-q", "--quiet", action="store_true", default=False,
+                        help="Restrict output to warnings and errors")
+
+
 def fill_config_defaults(thisconfig):
     for k, v in default_config.items():
         if k not in thisconfig:
@@ -359,10 +368,16 @@ def read_app_args(args, allapps, allow_vercodes=False):
     return apps
 
 
-def has_extension(filename, extension):
-    name, ext = os.path.splitext(filename)
-    ext = ext.lower()[1:]
-    return ext == extension
+def get_extension(filename):
+    _, ext = os.path.splitext(filename)
+    if not ext:
+        return ''
+    return ext.lower()[1:]
+
+
+def has_extension(filename, ext):
+    return ext == get_extension(filename)
+
 
 apk_regex = None
 
@@ -530,7 +545,7 @@ class vcs:
             raise VCSException('gettags not supported for this vcs type')
         rtags = []
         for tag in self._gettags():
-            if re.match('[-A-Za-z0-9_. ]+$', tag):
+            if re.match('[-A-Za-z0-9_. /]+$', tag):
                 rtags.append(tag)
         return rtags
 
@@ -990,7 +1005,7 @@ def parse_androidmanifests(paths, ignoreversions=None):
 
     vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
     vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
-    psearch_g = re.compile(r'.*packageName *=* *["\']([^"]+)["\'].*').search
+    psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
 
     ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
 
@@ -1007,15 +1022,16 @@ def parse_androidmanifests(paths, ignoreversions=None):
         gradle = has_extension(path, 'gradle')
         version = None
         vercode = None
-        # Remember package name, may be defined separately from version+vercode
-        package = max_package
+        package = None
 
         if gradle:
             for line in file(path):
+                # Grab first occurence of each to avoid running into
+                # alternative flavours and builds.
                 if not package:
                     matches = psearch_g(line)
                     if matches:
-                        package = matches.group(1)
+                        package = matches.group(2)
                 if not version:
                     matches = vnsearch_g(line)
                     if matches:
@@ -1037,6 +1053,10 @@ def parse_androidmanifests(paths, ignoreversions=None):
                 if string_is_integer(a):
                     vercode = a
 
+        # Remember package name, may be defined separately from version+vercode
+        if package is None:
+            package = max_package
+
         logging.debug("..got package={0}, version={1}, vercode={2}"
                       .format(package, version, vercode))
 
@@ -1528,31 +1548,6 @@ def isApkDebuggable(apkfile, config):
     return False
 
 
-class AsynchronousFileReader(threading.Thread):
-
-    '''
-    Helper class to implement asynchronous reading of a file
-    in a separate thread. Pushes read lines on a queue to
-    be consumed in another thread.
-    '''
-
-    def __init__(self, fd, queue):
-        assert isinstance(queue, Queue.Queue)
-        assert callable(fd.readline)
-        threading.Thread.__init__(self)
-        self._fd = fd
-        self._queue = queue
-
-    def run(self):
-        '''The body of the tread: read lines and put them on the queue.'''
-        for line in iter(self._fd.readline, ''):
-            self._queue.put(line)
-
-    def eof(self):
-        '''Check whether there is no more content to expect.'''
-        return not self.is_alive() and self._queue.empty()
-
-
 class PopenResult:
     returncode = None
     output = ''
@@ -1593,7 +1588,6 @@ def FDroidPopen(commands, cwd=None, output=True):
 
     stdout_queue = Queue.Queue()
     stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
-    stdout_reader.start()
 
     # Check the queue for output (until there is no more to get)
     while not stdout_reader.eof():