import xml.etree.ElementTree as XMLElementTree
from binascii import hexlify
-from datetime import datetime
+from datetime import datetime, timedelta
from distutils.version import LooseVersion
from queue import Queue
from zipfile import ZipFile
'r13b': None,
'r14b': None,
'r15c': None,
+ 'r16': None,
},
'qt_sdk_path': None,
'build_tools': "25.0.2",
help=_("Restrict output to warnings and errors"))
+def _add_java_paths_to_config(pathlist, thisconfig):
+ def path_version_key(s):
+ versionlist = []
+ for u in re.split('[^0-9]+', s):
+ try:
+ versionlist.append(int(u))
+ except ValueError:
+ pass
+ return versionlist
+
+ for d in sorted(pathlist, key=path_version_key):
+ if os.path.islink(d):
+ continue
+ j = os.path.basename(d)
+ # the last one found will be the canonical one, so order appropriately
+ for regex in [
+ r'^1\.([6-9])\.0\.jdk$', # OSX
+ r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
+ r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
+ r'^jdk([6-9])-openjdk$', # Arch
+ r'^java-([6-9])-openjdk$', # Arch
+ r'^java-([6-9])-jdk$', # Arch (oracle)
+ r'^java-1\.([6-9])\.0-.*$', # RedHat
+ r'^java-([6-9])-oracle$', # Debian WebUpd8
+ r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
+ r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
+ ]:
+ m = re.match(regex, j)
+ if not m:
+ continue
+ for p in [d, os.path.join(d, 'Contents', 'Home')]:
+ if os.path.exists(os.path.join(p, 'bin', 'javac')):
+ thisconfig['java_paths'][m.group(1)] = p
+
+
def fill_config_defaults(thisconfig):
for k, v in default_config.items():
if k not in thisconfig:
pathlist.append(os.getenv('JAVA_HOME'))
if os.getenv('PROGRAMFILES') is not None:
pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
- for d in sorted(pathlist):
- if os.path.islink(d):
- continue
- j = os.path.basename(d)
- # the last one found will be the canonical one, so order appropriately
- for regex in [
- r'^1\.([6-9])\.0\.jdk$', # OSX
- r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
- r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
- r'^jdk([6-9])-openjdk$', # Arch
- r'^java-([6-9])-openjdk$', # Arch
- r'^java-([6-9])-jdk$', # Arch (oracle)
- r'^java-1\.([6-9])\.0-.*$', # RedHat
- r'^java-([6-9])-oracle$', # Debian WebUpd8
- r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
- r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
- ]:
- m = re.match(regex, j)
- if not m:
- continue
- for p in [d, os.path.join(d, 'Contents', 'Home')]:
- if os.path.exists(os.path.join(p, 'bin', 'javac')):
- thisconfig['java_paths'][m.group(1)] = p
+ _add_java_paths_to_config(pathlist, thisconfig)
for java_version in ('7', '8', '9'):
if java_version not in thisconfig['java_paths']:
return config
+def assert_config_keystore(config):
+ """Check weather keystore is configured correctly and raise exception if not."""
+
+ nosigningkey = False
+ if 'repo_keyalias' not in config:
+ nosigningkey = True
+ logging.critical(_("'repo_keyalias' not found in config.py!"))
+ if 'keystore' not in config:
+ nosigningkey = True
+ logging.critical(_("'keystore' not found in config.py!"))
+ elif not os.path.exists(config['keystore']):
+ nosigningkey = True
+ logging.critical("'" + config['keystore'] + "' does not exist!")
+ if 'keystorepass' not in config:
+ nosigningkey = True
+ logging.critical(_("'keystorepass' not found in config.py!"))
+ if 'keypass' not in config:
+ nosigningkey = True
+ logging.critical(_("'keypass' not found in config.py!"))
+ if nosigningkey:
+ raise FDroidException("This command requires a signing key, " +
+ "you can create one using: fdroid update --create-key")
+
+
def find_sdk_tools_cmd(cmd):
'''find a working path to a tool from the Android SDK'''
versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
if not os.path.isdir(versioned_build_tools):
raise FDroidException(
- _("Android Build Tools path '{path}' does not exist!")
+ _("Android build-tools path '{path}' does not exist!")
.format(path=versioned_build_tools))
return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
-def read_pkg_args(args, allow_vercodes=False):
+def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False):
"""
- :param args: arguments in the form of multiple appid:[vc] strings
+ :param appids: arguments in the form of multiple appid:[vc] strings
:returns: a dictionary with the set of vercodes specified for each package
"""
-
vercodes = {}
- if not args:
+ if not appid_versionCode_pairs:
return vercodes
- for p in args:
+ for p in appid_versionCode_pairs:
if allow_vercodes and ':' in p:
package, vercode = p.split(':')
else:
return vercodes
-def read_app_args(args, allapps, allow_vercodes=False):
- """
- On top of what read_pkg_args does, this returns the whole app metadata, but
- limiting the builds list to the builds matching the vercodes specified.
+def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
+ """Build a list of App instances for processing
+
+ On top of what read_pkg_args does, this returns the whole app
+ metadata, but limiting the builds list to the builds matching the
+ appid_versionCode_pairs and vercodes specified. If no appid_versionCode_pairs are specified, then
+ all App and Build instances are returned.
+
"""
- vercodes = read_pkg_args(args, allow_vercodes)
+ vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes)
if not vercodes:
return allapps
def has_extension(filename, ext):
- _, f_ext = get_extension(filename)
+ _ignored, f_ext = get_extension(filename)
return ext == f_ext
def repotype(self):
return None
+ def clientversion(self):
+ versionstr = FDroidPopen(self.clientversioncmd()).output
+ return versionstr[0:versionstr.find('\n')]
+
+ def clientversioncmd(self):
+ return None
+
def gotorevision(self, rev, refresh=True):
"""Take the local repository to a clean version of the given
revision, which is specificed in the VCS's native
def repotype(self):
return 'git'
+ def clientversioncmd(self):
+ return ['git', '--version']
+
+ def GitFetchFDroidPopen(self, gitargs, envs=dict(), cwd=None, output=True):
+ '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
+
+ While fetch/pull/clone respect the command line option flags,
+ it seems that submodule commands do not. They do seem to
+ follow whatever is in env vars, if the version of git is new
+ enough. So we just throw the kitchen sink at it to see what
+ sticks.
+
+ '''
+ if cwd is None:
+ cwd = self.local
+ git_config = []
+ for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
+ git_config.append('-c')
+ git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':')
+ git_config.append('-c')
+ git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain)
+ git_config.append('-c')
+ git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain)
+ # add helpful tricks supported in git >= 2.3
+ ssh_command = 'ssh -oBatchMode=yes -oStrictHostKeyChecking=yes'
+ git_config.append('-c')
+ git_config.append('core.sshCommand="' + ssh_command + '"') # git >= 2.10
+ envs.update({
+ 'GIT_TERMINAL_PROMPT': '0',
+ 'GIT_SSH_COMMAND': ssh_command, # git >= 2.3
+ })
+ return FDroidPopen(['git', ] + git_config + gitargs,
+ envs=envs, cwd=cwd, output=output)
+
def checkrepo(self):
"""If the local directory exists, but is somehow not a git repository,
git will traverse up the directory tree until it finds one
def gotorevisionx(self, rev):
if not os.path.exists(self.local):
# Brand new checkout
- p = FDroidPopen(['git', 'clone', self.remote, self.local])
+ p = FDroidPopen(['git', 'clone', self.remote, self.local], cwd=None)
if p.returncode != 0:
self.clone_failed = True
raise VCSException("Git clone failed", p.output)
raise VCSException(_("Git clean failed"), p.output)
if not self.refreshed:
# Get latest commits and tags from remote
- p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
+ p = self.GitFetchFDroidPopen(['fetch', 'origin'])
if p.returncode != 0:
raise VCSException(_("Git fetch failed"), p.output)
- p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
+ p = self.GitFetchFDroidPopen(['fetch', '--prune', '--tags', 'origin'], output=False)
if p.returncode != 0:
raise VCSException(_("Git fetch failed"), p.output)
# Recreate origin/HEAD as git clone would do it, in case it disappeared
lines = f.readlines()
with open(submfile, 'w') as f:
for line in lines:
- if 'git@github.com' in line:
- line = line.replace('git@github.com:', 'https://github.com/')
- if 'git@gitlab.com' in line:
- line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
+ for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
+ line = re.sub('git@' + domain + ':', 'https://u:p@' + domain + '/', line)
f.write(line)
p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
if p.returncode != 0:
raise VCSException(_("Git submodule sync failed"), p.output)
- p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
+ p = self.GitFetchFDroidPopen(['submodule', 'update', '--init', '--force', '--recursive'])
if p.returncode != 0:
raise VCSException(_("Git submodule update failed"), p.output)
def repotype(self):
return 'git-svn'
+ def clientversioncmd(self):
+ return ['git', 'svn', '--version']
+
def checkrepo(self):
"""If the local directory exists, but is somehow not a git repository,
git will traverse up the directory tree until it finds one that
def repotype(self):
return 'hg'
+ def clientversioncmd(self):
+ return ['hg', '--version']
+
def gotorevisionx(self, rev):
if not os.path.exists(self.local):
p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
def repotype(self):
return 'bzr'
+ def clientversioncmd(self):
+ return ['bzr', '--version']
+
def gotorevisionx(self, rev):
if not os.path.exists(self.local):
p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
# Remove forced debuggable flags
logging.debug("Removing debuggable flags from %s" % root_dir)
for root, dirs, files in os.walk(root_dir):
- if 'AndroidManifest.xml' in files:
+ if 'AndroidManifest.xml' in files and os.path.isfile(os.path.join(root, 'AndroidManifest.xml')):
regsub_file(r'android:debuggable="[^"]*"',
'',
os.path.join(root, 'AndroidManifest.xml'))
vercode = None
package = None
+ flavour = ""
+ if app.builds and 'gradle' in app.builds[-1] and app.builds[-1].gradle:
+ flavour = app.builds[-1].gradle[-1]
+
if has_extension(path, 'gradle'):
+ # first try to get version name and code from correct flavour
with open(path, 'r') as f:
- for line in f:
- if gradle_comment.match(line):
- continue
- # Grab first occurence of each to avoid running into
- # alternative flavours and builds.
- if not package:
- matches = psearch_g(line)
- if matches:
- s = matches.group(2)
- if app_matches_packagename(app, s):
- package = s
- if not version:
- matches = vnsearch_g(line)
- if matches:
- version = matches.group(2)
- if not vercode:
- matches = vcsearch_g(line)
- if matches:
- vercode = matches.group(1)
+ buildfile = f.read()
+
+ regex_string = r"" + flavour + ".*?}"
+ search = re.compile(regex_string, re.DOTALL)
+ result = search.search(buildfile)
+
+ if result is not None:
+ resultgroup = result.group()
+
+ if not package:
+ matches = psearch_g(resultgroup)
+ if matches:
+ s = matches.group(2)
+ if app_matches_packagename(app, s):
+ package = s
+ if not version:
+ matches = vnsearch_g(resultgroup)
+ if matches:
+ version = matches.group(2)
+ if not vercode:
+ matches = vcsearch_g(resultgroup)
+ if matches:
+ vercode = matches.group(1)
+ else:
+ # fall back to parse file line by line
+ with open(path, 'r') as f:
+ for line in f:
+ if gradle_comment.match(line):
+ continue
+ # Grab first occurence of each to avoid running into
+ # alternative flavours and builds.
+ if not package:
+ matches = psearch_g(line)
+ if matches:
+ s = matches.group(2)
+ if app_matches_packagename(app, s):
+ package = s
+ if not version:
+ matches = vnsearch_g(line)
+ if matches:
+ version = matches.group(2)
+ if not vercode:
+ matches = vcsearch_g(line)
+ if matches:
+ vercode = matches.group(1)
else:
try:
xml = parse_xml(path)
dest = os.path.join(build_dir, part)
logging.info("Removing {0}".format(part))
if os.path.lexists(dest):
- if os.path.islink(dest):
- FDroidPopen(['unlink', dest], output=False)
+ # rmtree can only handle directories that are not symlinks, so catch anything else
+ if not os.path.isdir(dest) or os.path.islink(dest):
+ os.remove(dest)
else:
- FDroidPopen(['rm', '-rf', dest], output=False)
+ shutil.rmtree(dest)
else:
logging.info("...but it didn't exist")
return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
+def check_system_clock(dt_obj, path):
+ """Check if system clock is updated based on provided date
+
+ If an APK has files newer than the system time, suggest updating
+ the system clock. This is useful for offline systems, used for
+ signing, which do not have another source of clock sync info. It
+ has to be more than 24 hours newer because ZIP/APK files do not
+ store timezone info
+
+ """
+ checkdt = dt_obj - timedelta(1)
+ if datetime.today() < checkdt:
+ logging.warning(_('System clock is older than date in {path}!').format(path=path)
+ + '\n' + _('Set clock to that time using:') + '\n'
+ + 'sudo date -s "' + str(dt_obj) + '"')
+
+
class KnownApks:
"""permanent store of existing APKs with the date they were added
date = datetime.strptime(t[-1], '%Y-%m-%d')
filename = line[0:line.rfind(appid) - 1]
self.apks[filename] = (appid, date)
+ check_system_clock(date, self.path)
self.changed = False
def writeifchanged(self):
default_date = datetime.utcnow()
self.apks[apkName] = (app, default_date)
self.changed = True
- _, added = self.apks[apkName]
+ _ignored, added = self.apks[apkName]
return added
def getapp(self, apkname):
os.rename(signed_apk, tmp_apk)
with ZipFile(tmp_apk, 'r') as in_apk:
with ZipFile(signed_apk, 'w') as out_apk:
- for f in in_apk.infolist():
- if not apk_sigfile.match(f.filename):
+ for info in in_apk.infolist():
+ if not apk_sigfile.match(info.filename):
if strip_manifest:
- if f.filename != 'META-INF/MANIFEST.MF':
- buf = in_apk.read(f.filename)
- out_apk.writestr(f.filename, buf)
+ if info.filename != 'META-INF/MANIFEST.MF':
+ buf = in_apk.read(info.filename)
+ out_apk.writestr(info, buf)
else:
- buf = in_apk.read(f.filename)
- out_apk.writestr(f.filename, buf)
+ buf = in_apk.read(info.filename)
+ out_apk.writestr(info, buf)
def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest):
"""
# get list of available signature files in metadata
with tempfile.TemporaryDirectory() as tmpdir:
- # orig_apk = os.path.join(tmpdir, 'orig.apk')
- # os.rename(apkpath, orig_apk)
apkwithnewsig = os.path.join(tmpdir, 'newsig.apk')
with ZipFile(apkpath, 'r') as in_apk:
with ZipFile(apkwithnewsig, 'w') as out_apk:
for sig_file in [signaturefile, signedfile, manifest]:
- out_apk.write(sig_file, arcname='META-INF/' +
- os.path.basename(sig_file))
- for f in in_apk.infolist():
- if not apk_sigfile.match(f.filename):
- if f.filename != 'META-INF/MANIFEST.MF':
- buf = in_apk.read(f.filename)
- out_apk.writestr(f.filename, buf)
+ with open(sig_file, 'rb') as fp:
+ buf = fp.read()
+ info = zipfile.ZipInfo('META-INF/' + os.path.basename(sig_file))
+ info.compress_type = zipfile.ZIP_DEFLATED
+ info.create_system = 0 # "Windows" aka "FAT", what Android SDK uses
+ out_apk.writestr(info, buf)
+ for info in in_apk.infolist():
+ if not apk_sigfile.match(info.filename):
+ if info.filename != 'META-INF/MANIFEST.MF':
+ buf = in_apk.read(info.filename)
+ out_apk.writestr(info, buf)
os.remove(apkpath)
p = SdkToolsPopen(['zipalign', '-v', '4', apkwithnewsig, apkpath])
if p.returncode != 0:
try:
verify_jar_signature(apk)
return True
- except:
+ except Exception:
pass
return False
return False
+def local_rsync(options, fromdir, todir):
+ '''Rsync method for local to local copying of things
+
+ This is an rsync wrapper with all the settings for safe use within
+ the various fdroidserver use cases. This uses stricter rsync
+ checking on all files since people using offline mode are already
+ prioritizing security above ease and speed.
+
+ '''
+ rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
+ '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
+ if not options.no_checksum:
+ rsyncargs.append('--checksum')
+ if options.verbose:
+ rsyncargs += ['--verbose']
+ if options.quiet:
+ rsyncargs += ['--quiet']
+ logging.debug(' '.join(rsyncargs + [fromdir, todir]))
+ if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
+ raise FDroidException()
+
+
def get_per_app_repos():
'''per-app repos are dirs named with the packageName of a single app'''
b'index-v1.json',
b'categories.txt',
]
+
+
+def get_examples_dir():
+ '''Return the dir where the fdroidserver example files are available'''
+ examplesdir = None
+ tmp = os.path.dirname(sys.argv[0])
+ if os.path.basename(tmp) == 'bin':
+ egg_links = glob.glob(os.path.join(tmp, '..',
+ 'local/lib/python3.*/site-packages/fdroidserver.egg-link'))
+ if egg_links:
+ # installed from local git repo
+ examplesdir = os.path.join(open(egg_links[0]).readline().rstrip(), 'examples')
+ else:
+ # try .egg layout
+ examplesdir = os.path.dirname(os.path.dirname(__file__)) + '/share/doc/fdroidserver/examples'
+ if not os.path.exists(examplesdir): # use UNIX layout
+ examplesdir = os.path.dirname(tmp) + '/share/doc/fdroidserver/examples'
+ else:
+ # we're running straight out of the git repo
+ prefix = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
+ examplesdir = prefix + '/examples'
+
+ return examplesdir