# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# common.py is imported by all modules, so do not import third-party
+# libraries here as they will become a requirement for all commands.
+
import os
import sys
import re
import shutil
import glob
-import requests
import stat
import subprocess
import time
import operator
import Queue
-import threading
import logging
import hashlib
import socket
from zipfile import ZipFile
import metadata
+from fdroidserver.asynchronousfilereader import AsynchronousFileReader
+
XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
'sdk_path': "$ANDROID_HOME",
'ndk_paths': {
'r9b': None,
- 'r10e': "$ANDROID_NDK"
+ 'r10e': "$ANDROID_NDK",
},
- 'build_tools': "22.0.1",
+ 'build_tools': "23.0.1",
'ant': "ant",
'mvn3': "mvn",
'gradle': 'gradle',
+ 'accepted_formats': ['txt', 'yaml'],
'sync_from_local_copy_dir': False,
+ 'per_app_repos': False,
'make_current_version_link': True,
'current_version_name_source': 'Name',
'update_stats': False,
'smartcardoptions': [],
'char_limits': {
'Summary': 80,
- 'Description': 4000
+ 'Description': 4000,
},
'keyaliases': {},
'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
}
+def setup_global_opts(parser):
+ parser.add_argument("-v", "--verbose", action="store_true", default=False,
+ help="Spew out even more information than normal")
+ parser.add_argument("-q", "--quiet", action="store_true", default=False,
+ help="Restrict output to warnings and errors")
+
+
def fill_config_defaults(thisconfig):
for k, v in default_config.items():
if k not in thisconfig:
return apps
-def has_extension(filename, extension):
- name, ext = os.path.splitext(filename)
- ext = ext.lower()[1:]
- return ext == extension
+def get_extension(filename):
+ _, ext = os.path.splitext(filename)
+ if not ext:
+ return ''
+ return ext.lower()[1:]
+
+
+def has_extension(filename, ext):
+ return ext == get_extension(filename)
+
apk_regex = None
raise VCSException('gettags not supported for this vcs type')
rtags = []
for tag in self._gettags():
- if re.match('[-A-Za-z0-9_. ]+$', tag):
+ if re.match('[-A-Za-z0-9_. /]+$', tag):
rtags.append(tag)
return rtags
vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
- psearch_g = re.compile(r'.*packageName *=* *["\']([^"]+)["\'].*').search
+ psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
gradle = has_extension(path, 'gradle')
version = None
vercode = None
- # Remember package name, may be defined separately from version+vercode
- package = max_package
+ package = None
if gradle:
for line in file(path):
+ # Grab first occurence of each to avoid running into
+ # alternative flavours and builds.
if not package:
matches = psearch_g(line)
if matches:
- package = matches.group(1)
+ package = matches.group(2)
if not version:
matches = vnsearch_g(line)
if matches:
if string_is_integer(a):
vercode = a
+ # Remember package name, may be defined separately from version+vercode
+ if package is None:
+ package = max_package
+
logging.debug("..got package={0}, version={1}, vercode={2}"
.format(package, version, vercode))
props = ""
if os.path.isfile(path):
logging.info("Updating local.properties file at %s" % path)
- f = open(path, 'r')
- props += f.read()
- f.close()
+ with open(path, 'r') as f:
+ props += f.read()
props += '\n'
else:
logging.info("Creating local.properties file at %s" % path)
# Add java.encoding if necessary
if build['encoding']:
props += "java.encoding=%s\n" % build['encoding']
- f = open(path, 'w')
- f.write(props)
- f.close()
+ with open(path, 'w') as f:
+ f.write(props)
flavours = []
if build['type'] == 'gradle':
return paths
-def init_mime_type():
- '''
- There are two incompatible versions of the 'magic' module, one
- that comes as part of libmagic, which is what Debian includes as
- python-magic, then another called python-magic that is a separate
- project that wraps libmagic. The second is 'magic' on pypi, so
- both need to be supported. Then on platforms where libmagic is
- not easily included, e.g. OSX and Windows, fallback to the
- built-in 'mimetypes' module so this will work without
- libmagic. Hence this function with the following hacks:
- '''
-
- init_path = ''
- method = ''
- ms = None
-
- def mime_from_file(path):
- try:
- return magic.from_file(path, mime=True)
- except UnicodeError:
- return None
-
- def mime_file(path):
- try:
- return ms.file(path)
- except UnicodeError:
- return None
-
- def mime_guess_type(path):
- return mimetypes.guess_type(path, strict=False)
-
- try:
- import magic
- try:
- ms = magic.open(magic.MIME_TYPE)
- ms.load()
- magic.from_file(init_path, mime=True)
- method = 'from_file'
- except AttributeError:
- ms.file(init_path)
- method = 'file'
- except ImportError:
- import mimetypes
- mimetypes.init()
- method = 'guess_type'
-
- logging.info("Using magic method " + method)
- if method == 'from_file':
- return mime_from_file
- if method == 'file':
- return mime_file
- if method == 'guess_type':
- return mime_guess_type
-
- logging.critical("unknown magic method!")
-
-
-# Scan the source code in the given directory (and all subdirectories)
-# and return the number of fatal problems encountered
-def scan_source(build_dir, root_dir, thisbuild):
-
- count = 0
-
- # Common known non-free blobs (always lower case):
- usual_suspects = [
- re.compile(r'.*flurryagent', re.IGNORECASE),
- re.compile(r'.*paypal.*mpl', re.IGNORECASE),
- re.compile(r'.*google.*analytics', re.IGNORECASE),
- re.compile(r'.*admob.*sdk.*android', re.IGNORECASE),
- re.compile(r'.*google.*ad.*view', re.IGNORECASE),
- re.compile(r'.*google.*admob', re.IGNORECASE),
- re.compile(r'.*google.*play.*services', re.IGNORECASE),
- re.compile(r'.*crittercism', re.IGNORECASE),
- re.compile(r'.*heyzap', re.IGNORECASE),
- re.compile(r'.*jpct.*ae', re.IGNORECASE),
- re.compile(r'.*youtube.*android.*player.*api', re.IGNORECASE),
- re.compile(r'.*bugsense', re.IGNORECASE),
- re.compile(r'.*crashlytics', re.IGNORECASE),
- re.compile(r'.*ouya.*sdk', re.IGNORECASE),
- re.compile(r'.*libspen23', re.IGNORECASE),
- ]
-
- scanignore = getpaths(build_dir, thisbuild, 'scanignore')
- scandelete = getpaths(build_dir, thisbuild, 'scandelete')
-
- scanignore_worked = set()
- scandelete_worked = set()
-
- def toignore(fd):
- for p in scanignore:
- if fd.startswith(p):
- scanignore_worked.add(p)
- return True
- return False
-
- def todelete(fd):
- for p in scandelete:
- if fd.startswith(p):
- scandelete_worked.add(p)
- return True
- return False
-
- def ignoreproblem(what, fd, fp):
- logging.info('Ignoring %s at %s' % (what, fd))
- return 0
-
- def removeproblem(what, fd, fp):
- logging.info('Removing %s at %s' % (what, fd))
- os.remove(fp)
- return 0
-
- def warnproblem(what, fd):
- logging.warn('Found %s at %s' % (what, fd))
-
- def handleproblem(what, fd, fp):
- if toignore(fd):
- return ignoreproblem(what, fd, fp)
- if todelete(fd):
- return removeproblem(what, fd, fp)
- logging.error('Found %s at %s' % (what, fd))
- return 1
-
- get_mime_type = init_mime_type()
-
- # Iterate through all files in the source code
- for r, d, f in os.walk(build_dir, topdown=True):
-
- # It's topdown, so checking the basename is enough
- for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
- if ignoredir in d:
- d.remove(ignoredir)
-
- for curfile in f:
-
- # Path (relative) to the file
- fp = os.path.join(r, curfile)
- fd = fp[len(build_dir) + 1:]
-
- mime = get_mime_type(fp)
-
- if mime == 'application/x-sharedlib':
- count += handleproblem('shared library', fd, fp)
-
- elif mime == 'application/x-archive':
- count += handleproblem('static library', fd, fp)
-
- elif mime == 'application/x-executable' or mime == 'application/x-mach-binary':
- count += handleproblem('binary executable', fd, fp)
-
- elif mime == 'application/x-java-applet':
- count += handleproblem('Java compiled class', fd, fp)
-
- elif mime in (
- 'application/jar',
- 'application/zip',
- 'application/java-archive',
- 'application/octet-stream',
- 'binary', ):
-
- if has_extension(fp, 'apk'):
- removeproblem('APK file', fd, fp)
-
- elif has_extension(fp, 'jar'):
-
- if any(suspect.match(curfile) for suspect in usual_suspects):
- count += handleproblem('usual supect', fd, fp)
- else:
- warnproblem('JAR file', fd)
-
- elif has_extension(fp, 'zip'):
- warnproblem('ZIP file', fd)
-
- else:
- warnproblem('unknown compressed or binary file', fd)
-
- elif has_extension(fp, 'java'):
- if not os.path.isfile(fp):
- continue
- for line in file(fp):
- if 'DexClassLoader' in line:
- count += handleproblem('DexClassLoader', fd, fp)
- break
-
- elif has_extension(fp, 'gradle'):
- if not os.path.isfile(fp):
- continue
- for i, line in enumerate(file(fp)):
- i = i + 1
- if any(suspect.match(line) for suspect in usual_suspects):
- count += handleproblem('usual suspect at line %d' % i, fd, fp)
- break
-
- for p in scanignore:
- if p not in scanignore_worked:
- logging.error('Unused scanignore path: %s' % p)
- count += 1
-
- for p in scandelete:
- if p not in scandelete_worked:
- logging.error('Unused scandelete path: %s' % p)
- count += 1
-
- # Presence of a jni directory without buildjni=yes might
- # indicate a problem (if it's not a problem, explicitly use
- # buildjni=no to bypass this check)
- if (os.path.exists(os.path.join(root_dir, 'jni')) and
- not thisbuild['buildjni']):
- logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
- count += 1
-
- return count
+def natural_key(s):
+ return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
class KnownApks:
self.changed = False
def writeifchanged(self):
- if self.changed:
- if not os.path.exists('stats'):
- os.mkdir('stats')
- f = open(self.path, 'w')
- lst = []
- for apk, app in self.apks.iteritems():
- appid, added = app
- line = apk + ' ' + appid
- if added:
- line += ' ' + time.strftime('%Y-%m-%d', added)
- lst.append(line)
- for line in sorted(lst):
+ if not self.changed:
+ return
+
+ if not os.path.exists('stats'):
+ os.mkdir('stats')
+
+ lst = []
+ for apk, app in self.apks.iteritems():
+ appid, added = app
+ line = apk + ' ' + appid
+ if added:
+ line += ' ' + time.strftime('%Y-%m-%d', added)
+ lst.append(line)
+
+ with open(self.path, 'w') as f:
+ for line in sorted(lst, key=natural_key):
f.write(line + '\n')
- f.close()
# Record an apk (if it's new, otherwise does nothing)
# Returns the date it was added.
return False
-class AsynchronousFileReader(threading.Thread):
-
- '''
- Helper class to implement asynchronous reading of a file
- in a separate thread. Pushes read lines on a queue to
- be consumed in another thread.
- '''
-
- def __init__(self, fd, queue):
- assert isinstance(queue, Queue.Queue)
- assert callable(fd.readline)
- threading.Thread.__init__(self)
- self._fd = fd
- self._queue = queue
-
- def run(self):
- '''The body of the tread: read lines and put them on the queue.'''
- for line in iter(self._fd.readline, ''):
- self._queue.put(line)
-
- def eof(self):
- '''Check whether there is no more content to expect.'''
- return not self.is_alive() and self._queue.empty()
-
-
class PopenResult:
returncode = None
output = ''
stdout_queue = Queue.Queue()
stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
- stdout_reader.start()
# Check the queue for output (until there is no more to get)
while not stdout_reader.eof():
return False
-def download_file(url, local_filename=None, dldir='tmp'):
- filename = url.split('/')[-1]
- if local_filename is None:
- local_filename = os.path.join(dldir, filename)
- # the stream=True parameter keeps memory usage low
- r = requests.get(url, stream=True)
- with open(local_filename, 'wb') as f:
- for chunk in r.iter_content(chunk_size=1024):
- if chunk: # filter out keep-alive new chunks
- f.write(chunk)
- f.flush()
- return local_filename
+def get_per_app_repos():
+ '''per-app repos are dirs named with the packageName of a single app'''
+
+ # Android packageNames are Java packages, they may contain uppercase or
+ # lowercase letters ('A' through 'Z'), numbers, and underscores
+ # ('_'). However, individual package name parts may only start with
+ # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
+ p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
+
+ repos = []
+ for root, dirs, files in os.walk(os.getcwd()):
+ for d in dirs:
+ print 'checking', root, 'for', d
+ if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
+ # standard parts of an fdroid repo, so never packageNames
+ continue
+ elif p.match(d) \
+ and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
+ repos.append(d)
+ break
+ return repos
~ianmdlvl / fdroidserver.git / blobdiff