3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
40 import xml.etree.ElementTree as XMLElementTree
42 from binascii import hexlify
43 from datetime import datetime, timedelta
44 from distutils.version import LooseVersion
45 from queue import Queue
46 from zipfile import ZipFile
48 from pyasn1.codec.der import decoder, encoder
49 from pyasn1_modules import rfc2315
50 from pyasn1.error import PyAsn1Error
52 from distutils.util import strtobool
54 import fdroidserver.metadata
55 from fdroidserver import _
56 from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException,\
57 BuildException, VerificationException
58 from .asynchronousfilereader import AsynchronousFileReader
60 # this is the build-tools version, aapt has a separate version that
61 # has to be manually set in test_aapt_version()
62 MINIMUM_AAPT_VERSION = '26.0.0'
64 # A signature block file with a .DSA, .RSA, or .EC extension
65 CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
66 APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
67 STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
69 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
78 'sdk_path': "$ANDROID_HOME",
82 'r12b': "$ANDROID_NDK",
88 'build_tools': MINIMUM_AAPT_VERSION,
89 'force_build_tools': False,
94 'accepted_formats': ['txt', 'yml'],
95 'sync_from_local_copy_dir': False,
96 'allow_disabled_algorithms': False,
97 'per_app_repos': False,
98 'make_current_version_link': True,
99 'current_version_name_source': 'Name',
100 'update_stats': False,
102 'stats_server': None,
104 'stats_to_carbon': False,
106 'build_server_always': False,
107 'keystore': 'keystore.jks',
108 'smartcardoptions': [],
118 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
119 'repo_name': "My First FDroid Repo Demo",
120 'repo_icon': "fdroid-icon.png",
121 'repo_description': '''
122 This is a repository of apps to be used with FDroid. Applications in this
123 repository are either official binaries built by the original application
124 developers, or are binaries built from source by the admin of f-droid.org
125 using the tools on https://gitlab.com/u/fdroid.
131 def setup_global_opts(parser):
132 try: # the buildserver VM might not have PIL installed
133 from PIL import PngImagePlugin
134 logger = logging.getLogger(PngImagePlugin.__name__)
135 logger.setLevel(logging.INFO) # tame the "STREAM" debug messages
139 parser.add_argument("-v", "--verbose", action="store_true", default=False,
140 help=_("Spew out even more information than normal"))
141 parser.add_argument("-q", "--quiet", action="store_true", default=False,
142 help=_("Restrict output to warnings and errors"))
145 def _add_java_paths_to_config(pathlist, thisconfig):
146 def path_version_key(s):
148 for u in re.split('[^0-9]+', s):
150 versionlist.append(int(u))
155 for d in sorted(pathlist, key=path_version_key):
156 if os.path.islink(d):
158 j = os.path.basename(d)
159 # the last one found will be the canonical one, so order appropriately
161 r'^1\.([6-9])\.0\.jdk$', # OSX
162 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
163 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
164 r'^jdk([6-9])-openjdk$', # Arch
165 r'^java-([6-9])-openjdk$', # Arch
166 r'^java-([6-9])-jdk$', # Arch (oracle)
167 r'^java-1\.([6-9])\.0-.*$', # RedHat
168 r'^java-([6-9])-oracle$', # Debian WebUpd8
169 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
170 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
172 m = re.match(regex, j)
175 for p in [d, os.path.join(d, 'Contents', 'Home')]:
176 if os.path.exists(os.path.join(p, 'bin', 'javac')):
177 thisconfig['java_paths'][m.group(1)] = p
180 def fill_config_defaults(thisconfig):
181 for k, v in default_config.items():
182 if k not in thisconfig:
185 # Expand paths (~users and $vars)
186 def expand_path(path):
190 path = os.path.expanduser(path)
191 path = os.path.expandvars(path)
196 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
201 thisconfig[k + '_orig'] = v
203 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
204 if thisconfig['java_paths'] is None:
205 thisconfig['java_paths'] = dict()
207 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
208 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
209 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
210 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
211 if os.getenv('JAVA_HOME') is not None:
212 pathlist.append(os.getenv('JAVA_HOME'))
213 if os.getenv('PROGRAMFILES') is not None:
214 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
215 _add_java_paths_to_config(pathlist, thisconfig)
217 for java_version in ('7', '8', '9'):
218 if java_version not in thisconfig['java_paths']:
220 java_home = thisconfig['java_paths'][java_version]
221 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
222 if os.path.exists(jarsigner):
223 thisconfig['jarsigner'] = jarsigner
224 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
225 break # Java7 is preferred, so quit if found
227 for k in ['ndk_paths', 'java_paths']:
233 thisconfig[k][k2] = exp
234 thisconfig[k][k2 + '_orig'] = v
237 def regsub_file(pattern, repl, path):
238 with open(path, 'rb') as f:
240 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
241 with open(path, 'wb') as f:
245 def read_config(opts, config_file='config.py'):
246 """Read the repository config
248 The config is read from config_file, which is in the current
249 directory when any of the repo management commands are used. If
250 there is a local metadata file in the git repo, then config.py is
251 not required, just use defaults.
254 global config, options
256 if config is not None:
263 if os.path.isfile(config_file):
264 logging.debug(_("Reading '{config_file}'").format(config_file=config_file))
265 with io.open(config_file, "rb") as f:
266 code = compile(f.read(), config_file, 'exec')
267 exec(code, None, config)
269 logging.warning(_("No 'config.py' found, using defaults."))
271 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
273 if not type(config[k]) in (str, list, tuple):
275 _("'{field}' will be in random order! Use () or [] brackets if order is important!")
278 # smartcardoptions must be a list since its command line args for Popen
279 if 'smartcardoptions' in config:
280 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
281 elif 'keystore' in config and config['keystore'] == 'NONE':
282 # keystore='NONE' means use smartcard, these are required defaults
283 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
284 'SunPKCS11-OpenSC', '-providerClass',
285 'sun.security.pkcs11.SunPKCS11',
286 '-providerArg', 'opensc-fdroid.cfg']
288 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
289 st = os.stat(config_file)
290 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
291 logging.warning(_("unsafe permissions on '{config_file}' (should be 0600)!")
292 .format(config_file=config_file))
294 fill_config_defaults(config)
296 for k in ["repo_description", "archive_description"]:
298 config[k] = clean_description(config[k])
300 if 'serverwebroot' in config:
301 if isinstance(config['serverwebroot'], str):
302 roots = [config['serverwebroot']]
303 elif all(isinstance(item, str) for item in config['serverwebroot']):
304 roots = config['serverwebroot']
306 raise TypeError(_('only accepts strings, lists, and tuples'))
308 for rootstr in roots:
309 # since this is used with rsync, where trailing slashes have
310 # meaning, ensure there is always a trailing slash
311 if rootstr[-1] != '/':
313 rootlist.append(rootstr.replace('//', '/'))
314 config['serverwebroot'] = rootlist
316 if 'servergitmirrors' in config:
317 if isinstance(config['servergitmirrors'], str):
318 roots = [config['servergitmirrors']]
319 elif all(isinstance(item, str) for item in config['servergitmirrors']):
320 roots = config['servergitmirrors']
322 raise TypeError(_('only accepts strings, lists, and tuples'))
323 config['servergitmirrors'] = roots
328 def assert_config_keystore(config):
329 """Check weather keystore is configured correctly and raise exception if not."""
332 if 'repo_keyalias' not in config:
334 logging.critical(_("'repo_keyalias' not found in config.py!"))
335 if 'keystore' not in config:
337 logging.critical(_("'keystore' not found in config.py!"))
338 elif not os.path.exists(config['keystore']):
340 logging.critical("'" + config['keystore'] + "' does not exist!")
341 if 'keystorepass' not in config:
343 logging.critical(_("'keystorepass' not found in config.py!"))
344 if 'keypass' not in config:
346 logging.critical(_("'keypass' not found in config.py!"))
348 raise FDroidException("This command requires a signing key, " +
349 "you can create one using: fdroid update --create-key")
352 def find_sdk_tools_cmd(cmd):
353 '''find a working path to a tool from the Android SDK'''
356 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
357 # try to find a working path to this command, in all the recent possible paths
358 if 'build_tools' in config:
359 build_tools = os.path.join(config['sdk_path'], 'build-tools')
360 # if 'build_tools' was manually set and exists, check only that one
361 configed_build_tools = os.path.join(build_tools, config['build_tools'])
362 if os.path.exists(configed_build_tools):
363 tooldirs.append(configed_build_tools)
365 # no configed version, so hunt known paths for it
366 for f in sorted(os.listdir(build_tools), reverse=True):
367 if os.path.isdir(os.path.join(build_tools, f)):
368 tooldirs.append(os.path.join(build_tools, f))
369 tooldirs.append(build_tools)
370 sdk_tools = os.path.join(config['sdk_path'], 'tools')
371 if os.path.exists(sdk_tools):
372 tooldirs.append(sdk_tools)
373 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
374 if os.path.exists(sdk_platform_tools):
375 tooldirs.append(sdk_platform_tools)
376 tooldirs.append('/usr/bin')
378 path = os.path.join(d, cmd)
379 if os.path.isfile(path):
381 test_aapt_version(path)
383 # did not find the command, exit with error message
384 ensure_build_tools_exists(config)
387 def test_aapt_version(aapt):
388 '''Check whether the version of aapt is new enough'''
389 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
390 if output is None or output == '':
391 logging.error(_("'{path}' failed to execute!").format(path=aapt))
393 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
398 # the Debian package has the version string like "v0.2-23.0.2"
401 if LooseVersion(bugfix) < LooseVersion(MINIMUM_AAPT_VERSION):
403 elif LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.4062713'):
406 logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-{version} or newer!")
407 .format(aapt=aapt, version=MINIMUM_AAPT_VERSION))
409 logging.warning(_('Unknown version of aapt, might cause problems: ') + output)
412 def test_sdk_exists(thisconfig):
413 if 'sdk_path' not in thisconfig:
414 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
415 test_aapt_version(thisconfig['aapt'])
418 logging.error(_("'sdk_path' not set in 'config.py'!"))
420 if thisconfig['sdk_path'] == default_config['sdk_path']:
421 logging.error(_('No Android SDK found!'))
422 logging.error(_('You can use ANDROID_HOME to set the path to your SDK, i.e.:'))
423 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
425 if not os.path.exists(thisconfig['sdk_path']):
426 logging.critical(_("Android SDK path '{path}' does not exist!")
427 .format(path=thisconfig['sdk_path']))
429 if not os.path.isdir(thisconfig['sdk_path']):
430 logging.critical(_("Android SDK path '{path}' is not a directory!")
431 .format(path=thisconfig['sdk_path']))
433 for d in ['build-tools', 'platform-tools', 'tools']:
434 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
435 logging.critical(_("Android SDK '{path}' does not have '{dirname}' installed!")
436 .format(path=thisconfig['sdk_path'], dirname=d))
441 def ensure_build_tools_exists(thisconfig):
442 if not test_sdk_exists(thisconfig):
443 raise FDroidException(_("Android SDK not found!"))
444 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
445 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
446 if not os.path.isdir(versioned_build_tools):
447 raise FDroidException(
448 _("Android build-tools path '{path}' does not exist!")
449 .format(path=versioned_build_tools))
452 def get_local_metadata_files():
453 '''get any metadata files local to an app's source repo
455 This tries to ignore anything that does not count as app metdata,
456 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
459 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
462 def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False):
464 :param appids: arguments in the form of multiple appid:[vc] strings
465 :returns: a dictionary with the set of vercodes specified for each package
468 if not appid_versionCode_pairs:
471 for p in appid_versionCode_pairs:
472 if allow_vercodes and ':' in p:
473 package, vercode = p.split(':')
475 package, vercode = p, None
476 if package not in vercodes:
477 vercodes[package] = [vercode] if vercode else []
479 elif vercode and vercode not in vercodes[package]:
480 vercodes[package] += [vercode] if vercode else []
485 def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
486 """Build a list of App instances for processing
488 On top of what read_pkg_args does, this returns the whole app
489 metadata, but limiting the builds list to the builds matching the
490 appid_versionCode_pairs and vercodes specified. If no appid_versionCode_pairs are specified, then
491 all App and Build instances are returned.
495 vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes)
501 for appid, app in allapps.items():
502 if appid in vercodes:
505 if len(apps) != len(vercodes):
508 logging.critical(_("No such package: %s") % p)
509 raise FDroidException(_("Found invalid appids in arguments"))
511 raise FDroidException(_("No packages specified"))
514 for appid, app in apps.items():
518 app.builds = [b for b in app.builds if b.versionCode in vc]
519 if len(app.builds) != len(vercodes[appid]):
521 allvcs = [b.versionCode for b in app.builds]
522 for v in vercodes[appid]:
524 logging.critical(_("No such versionCode {versionCode} for app {appid}")
525 .format(versionCode=v, appid=appid))
528 raise FDroidException(_("Found invalid versionCodes for some apps"))
533 def get_extension(filename):
534 base, ext = os.path.splitext(filename)
537 return base, ext.lower()[1:]
540 def has_extension(filename, ext):
541 _ignored, f_ext = get_extension(filename)
545 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
548 def clean_description(description):
549 'Remove unneeded newlines and spaces from a block of description text'
551 # this is split up by paragraph to make removing the newlines easier
552 for paragraph in re.split(r'\n\n', description):
553 paragraph = re.sub('\r', '', paragraph)
554 paragraph = re.sub('\n', ' ', paragraph)
555 paragraph = re.sub(' {2,}', ' ', paragraph)
556 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
557 returnstring += paragraph + '\n\n'
558 return returnstring.rstrip('\n')
561 def publishednameinfo(filename):
562 filename = os.path.basename(filename)
563 m = publish_name_regex.match(filename)
565 result = (m.group(1), m.group(2))
566 except AttributeError:
567 raise FDroidException(_("Invalid name for published file: %s") % filename)
571 apk_release_filename = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)\.apk')
572 apk_release_filename_with_sigfp = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)_(?P<sigfp>[0-9a-f]{7})\.apk')
575 def apk_parse_release_filename(apkname):
576 """Parses the name of an APK file according the F-Droids APK naming
577 scheme and returns the tokens.
579 WARNING: Returned values don't necessarily represent the APKs actual
580 properties, the are just paresed from the file name.
582 :returns: A triplet containing (appid, versionCode, signer), where appid
583 should be the package name, versionCode should be the integer
584 represion of the APKs version and signer should be the first 7 hex
585 digists of the sha256 signing key fingerprint which was used to sign
588 m = apk_release_filename_with_sigfp.match(apkname)
590 return m.group('appid'), m.group('vercode'), m.group('sigfp')
591 m = apk_release_filename.match(apkname)
593 return m.group('appid'), m.group('vercode'), None
594 return None, None, None
597 def get_release_filename(app, build):
599 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
601 return "%s_%s.apk" % (app.id, build.versionCode)
604 def get_toolsversion_logname(app, build):
605 return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
608 def getsrcname(app, build):
609 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
621 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
624 def get_build_dir(app):
625 '''get the dir that this app will be built in'''
627 if app.RepoType == 'srclib':
628 return os.path.join('build', 'srclib', app.Repo)
630 return os.path.join('build', app.id)
634 '''checkout code from VCS and return instance of vcs and the build dir'''
635 build_dir = get_build_dir(app)
637 # Set up vcs interface and make sure we have the latest code...
638 logging.debug("Getting {0} vcs interface for {1}"
639 .format(app.RepoType, app.Repo))
640 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
644 vcs = getvcs(app.RepoType, remote, build_dir)
646 return vcs, build_dir
649 def getvcs(vcstype, remote, local):
651 return vcs_git(remote, local)
652 if vcstype == 'git-svn':
653 return vcs_gitsvn(remote, local)
655 return vcs_hg(remote, local)
657 return vcs_bzr(remote, local)
658 if vcstype == 'srclib':
659 if local != os.path.join('build', 'srclib', remote):
660 raise VCSException("Error: srclib paths are hard-coded!")
661 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
663 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
664 raise VCSException("Invalid vcs type " + vcstype)
667 def getsrclibvcs(name):
668 if name not in fdroidserver.metadata.srclibs:
669 raise VCSException("Missing srclib " + name)
670 return fdroidserver.metadata.srclibs[name]['Repo Type']
675 def __init__(self, remote, local):
677 # svn, git-svn and bzr may require auth
679 if self.repotype() in ('git-svn', 'bzr'):
681 if self.repotype == 'git-svn':
682 raise VCSException("Authentication is not supported for git-svn")
683 self.username, remote = remote.split('@')
684 if ':' not in self.username:
685 raise VCSException(_("Password required with username"))
686 self.username, self.password = self.username.split(':')
690 self.clone_failed = False
691 self.refreshed = False
697 def clientversion(self):
698 versionstr = FDroidPopen(self.clientversioncmd()).output
699 return versionstr[0:versionstr.find('\n')]
701 def clientversioncmd(self):
704 def gotorevision(self, rev, refresh=True):
705 """Take the local repository to a clean version of the given
706 revision, which is specificed in the VCS's native
707 format. Beforehand, the repository can be dirty, or even
708 non-existent. If the repository does already exist locally, it
709 will be updated from the origin, but only once in the lifetime
710 of the vcs object. None is acceptable for 'rev' if you know
711 you are cloning a clean copy of the repo - otherwise it must
712 specify a valid revision.
715 if self.clone_failed:
716 raise VCSException(_("Downloading the repository already failed once, not trying again."))
718 # The .fdroidvcs-id file for a repo tells us what VCS type
719 # and remote that directory was created from, allowing us to drop it
720 # automatically if either of those things changes.
721 fdpath = os.path.join(self.local, '..',
722 '.fdroidvcs-' + os.path.basename(self.local))
723 fdpath = os.path.normpath(fdpath)
724 cdata = self.repotype() + ' ' + self.remote
727 if os.path.exists(self.local):
728 if os.path.exists(fdpath):
729 with open(fdpath, 'r') as f:
730 fsdata = f.read().strip()
735 logging.info("Repository details for %s changed - deleting" % (
739 logging.info("Repository details for %s missing - deleting" % (
742 shutil.rmtree(self.local)
746 self.refreshed = True
749 self.gotorevisionx(rev)
750 except FDroidException as e:
753 # If necessary, write the .fdroidvcs file.
754 if writeback and not self.clone_failed:
755 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
756 with open(fdpath, 'w+') as f:
762 def gotorevisionx(self, rev): # pylint: disable=unused-argument
763 """Derived classes need to implement this.
765 It's called once basic checking has been performed.
767 raise VCSException("This VCS type doesn't define gotorevisionx")
769 # Initialise and update submodules
770 def initsubmodules(self):
771 raise VCSException('Submodules not supported for this vcs type')
773 # Get a list of all known tags
775 if not self._gettags:
776 raise VCSException('gettags not supported for this vcs type')
778 for tag in self._gettags():
779 if re.match('[-A-Za-z0-9_. /]+$', tag):
783 def latesttags(self):
784 """Get a list of all the known tags, sorted from newest to oldest"""
785 raise VCSException('latesttags not supported for this vcs type')
788 """Get current commit reference (hash, revision, etc)"""
789 raise VCSException('getref not supported for this vcs type')
792 """Returns the srclib (name, path) used in setting up the current revision, or None."""
801 def clientversioncmd(self):
802 return ['git', '--version']
804 def git(self, args, envs=dict(), cwd=None, output=True):
805 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
807 While fetch/pull/clone respect the command line option flags,
808 it seems that submodule commands do not. They do seem to
809 follow whatever is in env vars, if the version of git is new
810 enough. So we just throw the kitchen sink at it to see what
813 Also, because of CVE-2017-1000117, block all SSH URLs.
816 # supported in git >= 2.3
818 '-c', 'core.askpass=/bin/true',
819 '-c', 'core.sshCommand=/bin/false',
820 '-c', 'url.https://.insteadOf=ssh://',
822 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
823 git_config.append('-c')
824 git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':')
825 git_config.append('-c')
826 git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain)
827 git_config.append('-c')
828 git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain)
830 'GIT_TERMINAL_PROMPT': '0',
831 'GIT_ASKPASS': '/bin/true',
832 'SSH_ASKPASS': '/bin/true',
833 'GIT_SSH': '/bin/false', # for git < 2.3
835 return FDroidPopen(['git', ] + git_config + args,
836 envs=envs, cwd=cwd, output=output)
839 """If the local directory exists, but is somehow not a git repository,
840 git will traverse up the directory tree until it finds one
841 that is (i.e. fdroidserver) and then we'll proceed to destroy
842 it! This is called as a safety check.
846 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
847 result = p.output.rstrip()
848 if not result.endswith(self.local):
849 raise VCSException('Repository mismatch')
851 def gotorevisionx(self, rev):
852 if not os.path.exists(self.local):
854 p = self.git(['clone', '--', self.remote, self.local])
855 if p.returncode != 0:
856 self.clone_failed = True
857 raise VCSException("Git clone failed", p.output)
861 # Discard any working tree changes
862 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
863 'git', 'reset', '--hard'], cwd=self.local, output=False)
864 if p.returncode != 0:
865 raise VCSException(_("Git reset failed"), p.output)
866 # Remove untracked files now, in case they're tracked in the target
867 # revision (it happens!)
868 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
869 'git', 'clean', '-dffx'], cwd=self.local, output=False)
870 if p.returncode != 0:
871 raise VCSException(_("Git clean failed"), p.output)
872 if not self.refreshed:
873 # Get latest commits and tags from remote
874 p = self.git(['fetch', 'origin'], cwd=self.local)
875 if p.returncode != 0:
876 raise VCSException(_("Git fetch failed"), p.output)
877 p = self.git(['fetch', '--prune', '--tags', 'origin'], output=False, cwd=self.local)
878 if p.returncode != 0:
879 raise VCSException(_("Git fetch failed"), p.output)
880 # Recreate origin/HEAD as git clone would do it, in case it disappeared
881 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
882 if p.returncode != 0:
883 lines = p.output.splitlines()
884 if 'Multiple remote HEAD branches' not in lines[0]:
885 raise VCSException(_("Git remote set-head failed"), p.output)
886 branch = lines[1].split(' ')[-1]
887 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--', branch],
888 cwd=self.local, output=False)
889 if p2.returncode != 0:
890 raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output)
891 self.refreshed = True
892 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
893 # a github repo. Most of the time this is the same as origin/master.
894 rev = rev or 'origin/HEAD'
895 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
896 if p.returncode != 0:
897 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
898 # Get rid of any uncontrolled files left behind
899 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
900 if p.returncode != 0:
901 raise VCSException(_("Git clean failed"), p.output)
903 def initsubmodules(self):
905 submfile = os.path.join(self.local, '.gitmodules')
906 if not os.path.isfile(submfile):
907 raise NoSubmodulesException(_("No git submodules available"))
909 # fix submodules not accessible without an account and public key auth
910 with open(submfile, 'r') as f:
911 lines = f.readlines()
912 with open(submfile, 'w') as f:
914 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
915 line = re.sub('git@' + domain + ':', 'https://u:p@' + domain + '/', line)
918 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
919 if p.returncode != 0:
920 raise VCSException(_("Git submodule sync failed"), p.output)
921 p = self.git(['submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
922 if p.returncode != 0:
923 raise VCSException(_("Git submodule update failed"), p.output)
927 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
928 return p.output.splitlines()
930 tag_format = re.compile(r'tag: ([^),]*)')
932 def latesttags(self):
934 p = FDroidPopen(['git', 'log', '--tags',
935 '--simplify-by-decoration', '--pretty=format:%d'],
936 cwd=self.local, output=False)
938 for line in p.output.splitlines():
939 for tag in self.tag_format.findall(line):
944 class vcs_gitsvn(vcs):
949 def clientversioncmd(self):
950 return ['git', 'svn', '--version']
953 """If the local directory exists, but is somehow not a git repository,
954 git will traverse up the directory tree until it finds one that
955 is (i.e. fdroidserver) and then we'll proceed to destory it!
956 This is called as a safety check.
959 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
960 result = p.output.rstrip()
961 if not result.endswith(self.local):
962 raise VCSException('Repository mismatch')
964 def git(self, args, envs=dict(), cwd=None, output=True):
965 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
967 AskPass is set to /bin/true to let the process try to connect
968 without a username/password.
970 The SSH command is set to /bin/false to block all SSH URLs
971 (supported in git >= 2.3). This protects against
976 '-c', 'core.askpass=/bin/true',
977 '-c', 'core.sshCommand=/bin/false',
980 'GIT_TERMINAL_PROMPT': '0',
981 'GIT_ASKPASS': '/bin/true',
982 'SSH_ASKPASS': '/bin/true',
983 'GIT_SSH': '/bin/false', # for git < 2.3
984 'SVN_SSH': '/bin/false',
986 return FDroidPopen(['git', ] + git_config + args,
987 envs=envs, cwd=cwd, output=output)
989 def gotorevisionx(self, rev):
990 if not os.path.exists(self.local):
992 gitsvn_args = ['svn', 'clone']
994 if ';' in self.remote:
995 remote_split = self.remote.split(';')
996 for i in remote_split[1:]:
997 if i.startswith('trunk='):
998 gitsvn_args.extend(['-T', i[6:]])
999 elif i.startswith('tags='):
1000 gitsvn_args.extend(['-t', i[5:]])
1001 elif i.startswith('branches='):
1002 gitsvn_args.extend(['-b', i[9:]])
1003 remote = remote_split[0]
1005 remote = self.remote
1007 if not remote.startswith('https://'):
1008 raise VCSException(_('HTTPS must be used with Subversion URLs!'))
1010 # git-svn sucks at certificate validation, this throws useful errors:
1012 r = requests.head(remote)
1013 r.raise_for_status()
1014 location = r.headers.get('location')
1015 if location and not location.startswith('https://'):
1016 raise VCSException(_('Invalid redirect to non-HTTPS: {before} -> {after} ')
1017 .format(before=remote, after=location))
1019 gitsvn_args.extend(['--', remote, self.local])
1020 p = self.git(gitsvn_args)
1021 if p.returncode != 0:
1022 self.clone_failed = True
1023 raise VCSException(_('git svn clone failed'), p.output)
1027 # Discard any working tree changes
1028 p = self.git(['reset', '--hard'], cwd=self.local, output=False)
1029 if p.returncode != 0:
1030 raise VCSException("Git reset failed", p.output)
1031 # Remove untracked files now, in case they're tracked in the target
1032 # revision (it happens!)
1033 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1034 if p.returncode != 0:
1035 raise VCSException("Git clean failed", p.output)
1036 if not self.refreshed:
1037 # Get new commits, branches and tags from repo
1038 p = self.git(['svn', 'fetch'], cwd=self.local, output=False)
1039 if p.returncode != 0:
1040 raise VCSException("Git svn fetch failed")
1041 p = self.git(['svn', 'rebase'], cwd=self.local, output=False)
1042 if p.returncode != 0:
1043 raise VCSException("Git svn rebase failed", p.output)
1044 self.refreshed = True
1046 rev = rev or 'master'
1048 nospaces_rev = rev.replace(' ', '%20')
1049 # Try finding a svn tag
1050 for treeish in ['origin/', '']:
1051 p = self.git(['checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
1052 if p.returncode == 0:
1054 if p.returncode != 0:
1055 # No tag found, normal svn rev translation
1056 # Translate svn rev into git format
1057 rev_split = rev.split('/')
1060 for treeish in ['origin/', '']:
1061 if len(rev_split) > 1:
1062 treeish += rev_split[0]
1063 svn_rev = rev_split[1]
1066 # if no branch is specified, then assume trunk (i.e. 'master' branch):
1070 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
1072 p = self.git(['svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
1073 git_rev = p.output.rstrip()
1075 if p.returncode == 0 and git_rev:
1078 if p.returncode != 0 or not git_rev:
1079 # Try a plain git checkout as a last resort
1080 p = self.git(['checkout', rev], cwd=self.local, output=False)
1081 if p.returncode != 0:
1082 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
1084 # Check out the git rev equivalent to the svn rev
1085 p = self.git(['checkout', git_rev], cwd=self.local, output=False)
1086 if p.returncode != 0:
1087 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
1089 # Get rid of any uncontrolled files left behind
1090 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1091 if p.returncode != 0:
1092 raise VCSException(_("Git clean failed"), p.output)
1096 for treeish in ['origin/', '']:
1097 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
1098 if os.path.isdir(d):
1099 return os.listdir(d)
1103 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
1104 if p.returncode != 0:
1106 return p.output.strip()
1114 def clientversioncmd(self):
1115 return ['hg', '--version']
1117 def gotorevisionx(self, rev):
1118 if not os.path.exists(self.local):
1119 p = FDroidPopen(['hg', 'clone', '--ssh', '/bin/false', '--', self.remote, self.local],
1121 if p.returncode != 0:
1122 self.clone_failed = True
1123 raise VCSException("Hg clone failed", p.output)
1125 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
1126 if p.returncode != 0:
1127 raise VCSException("Hg status failed", p.output)
1128 for line in p.output.splitlines():
1129 if not line.startswith('? '):
1130 raise VCSException("Unexpected output from hg status -uS: " + line)
1131 FDroidPopen(['rm', '-rf', '--', line[2:]], cwd=self.local, output=False)
1132 if not self.refreshed:
1133 p = FDroidPopen(['hg', 'pull', '--ssh', '/bin/false'], cwd=self.local, output=False)
1134 if p.returncode != 0:
1135 raise VCSException("Hg pull failed", p.output)
1136 self.refreshed = True
1138 rev = rev or 'default'
1141 p = FDroidPopen(['hg', 'update', '-C', '--', rev], cwd=self.local, output=False)
1142 if p.returncode != 0:
1143 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
1144 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1145 # Also delete untracked files, we have to enable purge extension for that:
1146 if "'purge' is provided by the following extension" in p.output:
1147 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
1148 myfile.write("\n[extensions]\nhgext.purge=\n")
1149 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1150 if p.returncode != 0:
1151 raise VCSException("HG purge failed", p.output)
1152 elif p.returncode != 0:
1153 raise VCSException("HG purge failed", p.output)
1156 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
1157 return p.output.splitlines()[1:]
1165 def clientversioncmd(self):
1166 return ['bzr', '--version']
1168 def bzr(self, args, envs=dict(), cwd=None, output=True):
1169 '''Prevent bzr from ever using SSH to avoid security vulns'''
1173 return FDroidPopen(['bzr', ] + args, envs=envs, cwd=cwd, output=output)
1175 def gotorevisionx(self, rev):
1176 if not os.path.exists(self.local):
1177 p = self.bzr(['branch', self.remote, self.local], output=False)
1178 if p.returncode != 0:
1179 self.clone_failed = True
1180 raise VCSException("Bzr branch failed", p.output)
1182 p = self.bzr(['clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
1183 if p.returncode != 0:
1184 raise VCSException("Bzr revert failed", p.output)
1185 if not self.refreshed:
1186 p = self.bzr(['pull'], cwd=self.local, output=False)
1187 if p.returncode != 0:
1188 raise VCSException("Bzr update failed", p.output)
1189 self.refreshed = True
1191 revargs = list(['-r', rev] if rev else [])
1192 p = self.bzr(['revert'] + revargs, cwd=self.local, output=False)
1193 if p.returncode != 0:
1194 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1197 p = self.bzr(['tags'], cwd=self.local, output=False)
1198 return [tag.split(' ')[0].strip() for tag in
1199 p.output.splitlines()]
1202 def unescape_string(string):
1205 if string[0] == '"' and string[-1] == '"':
1208 return string.replace("\\'", "'")
1211 def retrieve_string(app_dir, string, xmlfiles=None):
1213 if not string.startswith('@string/'):
1214 return unescape_string(string)
1216 if xmlfiles is None:
1219 os.path.join(app_dir, 'res'),
1220 os.path.join(app_dir, 'src', 'main', 'res'),
1222 for root, dirs, files in os.walk(res_dir):
1223 if os.path.basename(root) == 'values':
1224 xmlfiles += [os.path.join(root, x) for x in files if x.endswith('.xml')]
1226 name = string[len('@string/'):]
1228 def element_content(element):
1229 if element.text is None:
1231 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1232 return s.decode('utf-8').strip()
1234 for path in xmlfiles:
1235 if not os.path.isfile(path):
1237 xml = parse_xml(path)
1238 element = xml.find('string[@name="' + name + '"]')
1239 if element is not None:
1240 content = element_content(element)
1241 return retrieve_string(app_dir, content, xmlfiles)
1246 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1247 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1250 def manifest_paths(app_dir, flavours):
1251 '''Return list of existing files that will be used to find the highest vercode'''
1253 possible_manifests = \
1254 [os.path.join(app_dir, 'AndroidManifest.xml'),
1255 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1256 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1257 os.path.join(app_dir, 'build.gradle')]
1259 for flavour in flavours:
1260 if flavour == 'yes':
1262 possible_manifests.append(
1263 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1265 return [path for path in possible_manifests if os.path.isfile(path)]
1268 def fetch_real_name(app_dir, flavours):
1269 '''Retrieve the package name. Returns the name, or None if not found.'''
1270 for path in manifest_paths(app_dir, flavours):
1271 if not has_extension(path, 'xml') or not os.path.isfile(path):
1273 logging.debug("fetch_real_name: Checking manifest at " + path)
1274 xml = parse_xml(path)
1275 app = xml.find('application')
1278 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1280 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1281 result = retrieve_string_singleline(app_dir, label)
1283 result = result.strip()
1288 def get_library_references(root_dir):
1290 proppath = os.path.join(root_dir, 'project.properties')
1291 if not os.path.isfile(proppath):
1293 with open(proppath, 'r', encoding='iso-8859-1') as f:
1295 if not line.startswith('android.library.reference.'):
1297 path = line.split('=')[1].strip()
1298 relpath = os.path.join(root_dir, path)
1299 if not os.path.isdir(relpath):
1301 logging.debug("Found subproject at %s" % path)
1302 libraries.append(path)
1306 def ant_subprojects(root_dir):
1307 subprojects = get_library_references(root_dir)
1308 for subpath in subprojects:
1309 subrelpath = os.path.join(root_dir, subpath)
1310 for p in get_library_references(subrelpath):
1311 relp = os.path.normpath(os.path.join(subpath, p))
1312 if relp not in subprojects:
1313 subprojects.insert(0, relp)
1317 def remove_debuggable_flags(root_dir):
1318 # Remove forced debuggable flags
1319 logging.debug("Removing debuggable flags from %s" % root_dir)
1320 for root, dirs, files in os.walk(root_dir):
1321 if 'AndroidManifest.xml' in files and os.path.isfile(os.path.join(root, 'AndroidManifest.xml')):
1322 regsub_file(r'android:debuggable="[^"]*"',
1324 os.path.join(root, 'AndroidManifest.xml'))
1327 vcsearch_g = re.compile(r'''.*[Vv]ersionCode\s*=?\s*["']*([0-9]+)["']*''').search
1328 vnsearch_g = re.compile(r'''.*[Vv]ersionName\s*=?\s*(["'])((?:(?=(\\?))\3.)*?)\1.*''').search
1329 psearch_g = re.compile(r'''.*(packageName|applicationId)\s*=*\s*["']([^"']+)["'].*''').search
1332 def app_matches_packagename(app, package):
1335 appid = app.UpdateCheckName or app.id
1336 if appid is None or appid == "Ignore":
1338 return appid == package
1341 def parse_androidmanifests(paths, app):
1343 Extract some information from the AndroidManifest.xml at the given path.
1344 Returns (version, vercode, package), any or all of which might be None.
1345 All values returned are strings.
1348 ignoreversions = app.UpdateCheckIgnore
1349 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1352 return (None, None, None)
1360 if not os.path.isfile(path):
1363 logging.debug(_("Parsing manifest at '{path}'").format(path=path))
1369 if app.builds and 'gradle' in app.builds[-1] and app.builds[-1].gradle:
1370 flavour = app.builds[-1].gradle[-1]
1372 if has_extension(path, 'gradle'):
1373 with open(path, 'r') as f:
1374 inside_flavour_group = 0
1375 inside_required_flavour = 0
1377 if gradle_comment.match(line):
1380 if inside_flavour_group > 0:
1381 if inside_required_flavour > 0:
1382 matches = psearch_g(line)
1384 s = matches.group(2)
1385 if app_matches_packagename(app, s):
1388 matches = vnsearch_g(line)
1390 version = matches.group(2)
1392 matches = vcsearch_g(line)
1394 vercode = matches.group(1)
1397 inside_required_flavour += 1
1399 inside_required_flavour -= 1
1401 if flavour and (flavour in line):
1402 inside_required_flavour = 1
1405 inside_flavour_group += 1
1407 inside_flavour_group -= 1
1409 if "productFlavors" in line:
1410 inside_flavour_group = 1
1412 matches = psearch_g(line)
1414 s = matches.group(2)
1415 if app_matches_packagename(app, s):
1418 matches = vnsearch_g(line)
1420 version = matches.group(2)
1422 matches = vcsearch_g(line)
1424 vercode = matches.group(1)
1427 xml = parse_xml(path)
1428 if "package" in xml.attrib:
1429 s = xml.attrib["package"]
1430 if app_matches_packagename(app, s):
1432 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1433 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1434 base_dir = os.path.dirname(path)
1435 version = retrieve_string_singleline(base_dir, version)
1436 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1437 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1438 if string_is_integer(a):
1441 logging.warning(_("Problem with xml at '{path}'").format(path=path))
1443 # Remember package name, may be defined separately from version+vercode
1445 package = max_package
1447 logging.debug("..got package={0}, version={1}, vercode={2}"
1448 .format(package, version, vercode))
1450 # Always grab the package name and version name in case they are not
1451 # together with the highest version code
1452 if max_package is None and package is not None:
1453 max_package = package
1454 if max_version is None and version is not None:
1455 max_version = version
1457 if vercode is not None \
1458 and (max_vercode is None or vercode > max_vercode):
1459 if not ignoresearch or not ignoresearch(version):
1460 if version is not None:
1461 max_version = version
1462 if vercode is not None:
1463 max_vercode = vercode
1464 if package is not None:
1465 max_package = package
1467 max_version = "Ignore"
1469 if max_version is None:
1470 max_version = "Unknown"
1472 if max_package and not is_valid_package_name(max_package):
1473 raise FDroidException(_("Invalid package name {0}").format(max_package))
1475 return (max_version, max_vercode, max_package)
1478 def is_valid_package_name(name):
1479 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1482 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1483 raw=False, prepare=True, preponly=False, refresh=True,
1485 """Get the specified source library.
1487 Returns the path to it. Normally this is the path to be used when
1488 referencing it, which may be a subdirectory of the actual project. If
1489 you want the base directory of the project, pass 'basepath=True'.
1498 name, ref = spec.split('@')
1500 number, name = name.split(':', 1)
1502 name, subdir = name.split('/', 1)
1504 if name not in fdroidserver.metadata.srclibs:
1505 raise VCSException('srclib ' + name + ' not found.')
1507 srclib = fdroidserver.metadata.srclibs[name]
1509 sdir = os.path.join(srclib_dir, name)
1512 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1513 vcs.srclib = (name, number, sdir)
1515 vcs.gotorevision(ref, refresh)
1522 libdir = os.path.join(sdir, subdir)
1523 elif srclib["Subdir"]:
1524 for subdir in srclib["Subdir"]:
1525 libdir_candidate = os.path.join(sdir, subdir)
1526 if os.path.exists(libdir_candidate):
1527 libdir = libdir_candidate
1533 remove_signing_keys(sdir)
1534 remove_debuggable_flags(sdir)
1538 if srclib["Prepare"]:
1539 cmd = replace_config_vars(srclib["Prepare"], build)
1541 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=libdir)
1542 if p.returncode != 0:
1543 raise BuildException("Error running prepare command for srclib %s"
1549 return (name, number, libdir)
1552 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1555 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1556 """ Prepare the source code for a particular build
1558 :param vcs: the appropriate vcs object for the application
1559 :param app: the application details from the metadata
1560 :param build: the build details from the metadata
1561 :param build_dir: the path to the build directory, usually 'build/app.id'
1562 :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
1563 :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
1565 Returns the (root, srclibpaths) where:
1566 :param root: is the root directory, which may be the same as 'build_dir' or may
1567 be a subdirectory of it.
1568 :param srclibpaths: is information on the srclibs being used
1571 # Optionally, the actual app source can be in a subdirectory
1573 root_dir = os.path.join(build_dir, build.subdir)
1575 root_dir = build_dir
1577 # Get a working copy of the right revision
1578 logging.info("Getting source for revision " + build.commit)
1579 vcs.gotorevision(build.commit, refresh)
1581 # Initialise submodules if required
1582 if build.submodules:
1583 logging.info(_("Initialising submodules"))
1584 vcs.initsubmodules()
1586 # Check that a subdir (if we're using one) exists. This has to happen
1587 # after the checkout, since it might not exist elsewhere
1588 if not os.path.exists(root_dir):
1589 raise BuildException('Missing subdir ' + root_dir)
1591 # Run an init command if one is required
1593 cmd = replace_config_vars(build.init, build)
1594 logging.info("Running 'init' commands in %s" % root_dir)
1596 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir)
1597 if p.returncode != 0:
1598 raise BuildException("Error running init command for %s:%s" %
1599 (app.id, build.versionName), p.output)
1601 # Apply patches if any
1603 logging.info("Applying patches")
1604 for patch in build.patch:
1605 patch = patch.strip()
1606 logging.info("Applying " + patch)
1607 patch_path = os.path.join('metadata', app.id, patch)
1608 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1609 if p.returncode != 0:
1610 raise BuildException("Failed to apply patch %s" % patch_path)
1612 # Get required source libraries
1615 logging.info("Collecting source libraries")
1616 for lib in build.srclibs:
1617 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1618 refresh=refresh, build=build))
1620 for name, number, libpath in srclibpaths:
1621 place_srclib(root_dir, int(number) if number else None, libpath)
1623 basesrclib = vcs.getsrclib()
1624 # If one was used for the main source, add that too.
1626 srclibpaths.append(basesrclib)
1628 # Update the local.properties file
1629 localprops = [os.path.join(build_dir, 'local.properties')]
1631 parts = build.subdir.split(os.sep)
1634 cur = os.path.join(cur, d)
1635 localprops += [os.path.join(cur, 'local.properties')]
1636 for path in localprops:
1638 if os.path.isfile(path):
1639 logging.info("Updating local.properties file at %s" % path)
1640 with open(path, 'r', encoding='iso-8859-1') as f:
1644 logging.info("Creating local.properties file at %s" % path)
1645 # Fix old-fashioned 'sdk-location' by copying
1646 # from sdk.dir, if necessary
1648 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1649 re.S | re.M).group(1)
1650 props += "sdk-location=%s\n" % sdkloc
1652 props += "sdk.dir=%s\n" % config['sdk_path']
1653 props += "sdk-location=%s\n" % config['sdk_path']
1654 ndk_path = build.ndk_path()
1655 # if for any reason the path isn't valid or the directory
1656 # doesn't exist, some versions of Gradle will error with a
1657 # cryptic message (even if the NDK is not even necessary).
1658 # https://gitlab.com/fdroid/fdroidserver/issues/171
1659 if ndk_path and os.path.exists(ndk_path):
1661 props += "ndk.dir=%s\n" % ndk_path
1662 props += "ndk-location=%s\n" % ndk_path
1663 # Add java.encoding if necessary
1665 props += "java.encoding=%s\n" % build.encoding
1666 with open(path, 'w', encoding='iso-8859-1') as f:
1670 if build.build_method() == 'gradle':
1671 flavours = build.gradle
1674 n = build.target.split('-')[1]
1675 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1676 r'compileSdkVersion %s' % n,
1677 os.path.join(root_dir, 'build.gradle'))
1679 # Remove forced debuggable flags
1680 remove_debuggable_flags(root_dir)
1682 # Insert version code and number into the manifest if necessary
1683 if build.forceversion:
1684 logging.info("Changing the version name")
1685 for path in manifest_paths(root_dir, flavours):
1686 if not os.path.isfile(path):
1688 if has_extension(path, 'xml'):
1689 regsub_file(r'android:versionName="[^"]*"',
1690 r'android:versionName="%s"' % build.versionName,
1692 elif has_extension(path, 'gradle'):
1693 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1694 r"""\1versionName '%s'""" % build.versionName,
1697 if build.forcevercode:
1698 logging.info("Changing the version code")
1699 for path in manifest_paths(root_dir, flavours):
1700 if not os.path.isfile(path):
1702 if has_extension(path, 'xml'):
1703 regsub_file(r'android:versionCode="[^"]*"',
1704 r'android:versionCode="%s"' % build.versionCode,
1706 elif has_extension(path, 'gradle'):
1707 regsub_file(r'versionCode[ =]+[0-9]+',
1708 r'versionCode %s' % build.versionCode,
1711 # Delete unwanted files
1713 logging.info(_("Removing specified files"))
1714 for part in getpaths(build_dir, build.rm):
1715 dest = os.path.join(build_dir, part)
1716 logging.info("Removing {0}".format(part))
1717 if os.path.lexists(dest):
1718 # rmtree can only handle directories that are not symlinks, so catch anything else
1719 if not os.path.isdir(dest) or os.path.islink(dest):
1724 logging.info("...but it didn't exist")
1726 remove_signing_keys(build_dir)
1728 # Add required external libraries
1730 logging.info("Collecting prebuilt libraries")
1731 libsdir = os.path.join(root_dir, 'libs')
1732 if not os.path.exists(libsdir):
1734 for lib in build.extlibs:
1736 logging.info("...installing extlib {0}".format(lib))
1737 libf = os.path.basename(lib)
1738 libsrc = os.path.join(extlib_dir, lib)
1739 if not os.path.exists(libsrc):
1740 raise BuildException("Missing extlib file {0}".format(libsrc))
1741 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1743 # Run a pre-build command if one is required
1745 logging.info("Running 'prebuild' commands in %s" % root_dir)
1747 cmd = replace_config_vars(build.prebuild, build)
1749 # Substitute source library paths into prebuild commands
1750 for name, number, libpath in srclibpaths:
1751 libpath = os.path.relpath(libpath, root_dir)
1752 cmd = cmd.replace('$$' + name + '$$', libpath)
1754 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir)
1755 if p.returncode != 0:
1756 raise BuildException("Error running prebuild command for %s:%s" %
1757 (app.id, build.versionName), p.output)
1759 # Generate (or update) the ant build file, build.xml...
1760 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1761 parms = ['android', 'update', 'lib-project']
1762 lparms = ['android', 'update', 'project']
1765 parms += ['-t', build.target]
1766 lparms += ['-t', build.target]
1767 if build.androidupdate:
1768 update_dirs = build.androidupdate
1770 update_dirs = ant_subprojects(root_dir) + ['.']
1772 for d in update_dirs:
1773 subdir = os.path.join(root_dir, d)
1775 logging.debug("Updating main project")
1776 cmd = parms + ['-p', d]
1778 logging.debug("Updating subproject %s" % d)
1779 cmd = lparms + ['-p', d]
1780 p = SdkToolsPopen(cmd, cwd=root_dir)
1781 # Check to see whether an error was returned without a proper exit
1782 # code (this is the case for the 'no target set or target invalid'
1784 if p.returncode != 0 or p.output.startswith("Error: "):
1785 raise BuildException("Failed to update project at %s" % d, p.output)
1786 # Clean update dirs via ant
1788 logging.info("Cleaning subproject %s" % d)
1789 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1791 return (root_dir, srclibpaths)
1794 def getpaths_map(build_dir, globpaths):
1795 """Extend via globbing the paths from a field and return them as a map from original path to resulting paths"""
1799 full_path = os.path.join(build_dir, p)
1800 full_path = os.path.normpath(full_path)
1801 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1803 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1807 def getpaths(build_dir, globpaths):
1808 """Extend via globbing the paths from a field and return them as a set"""
1809 paths_map = getpaths_map(build_dir, globpaths)
1811 for k, v in paths_map.items():
1818 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1821 def check_system_clock(dt_obj, path):
1822 """Check if system clock is updated based on provided date
1824 If an APK has files newer than the system time, suggest updating
1825 the system clock. This is useful for offline systems, used for
1826 signing, which do not have another source of clock sync info. It
1827 has to be more than 24 hours newer because ZIP/APK files do not
1831 checkdt = dt_obj - timedelta(1)
1832 if datetime.today() < checkdt:
1833 logging.warning(_('System clock is older than date in {path}!').format(path=path)
1834 + '\n' + _('Set clock to that time using:') + '\n'
1835 + 'sudo date -s "' + str(dt_obj) + '"')
1839 """permanent store of existing APKs with the date they were added
1841 This is currently the only way to permanently store the "updated"
1846 '''Load filename/date info about previously seen APKs
1848 Since the appid and date strings both will never have spaces,
1849 this is parsed as a list from the end to allow the filename to
1850 have any combo of spaces.
1853 self.path = os.path.join('stats', 'known_apks.txt')
1855 if os.path.isfile(self.path):
1856 with open(self.path, 'r', encoding='utf8') as f:
1858 t = line.rstrip().split(' ')
1860 self.apks[t[0]] = (t[1], None)
1863 date = datetime.strptime(t[-1], '%Y-%m-%d')
1864 filename = line[0:line.rfind(appid) - 1]
1865 self.apks[filename] = (appid, date)
1866 check_system_clock(date, self.path)
1867 self.changed = False
1869 def writeifchanged(self):
1870 if not self.changed:
1873 if not os.path.exists('stats'):
1877 for apk, app in self.apks.items():
1879 line = apk + ' ' + appid
1881 line += ' ' + added.strftime('%Y-%m-%d')
1884 with open(self.path, 'w', encoding='utf8') as f:
1885 for line in sorted(lst, key=natural_key):
1886 f.write(line + '\n')
1888 def recordapk(self, apkName, app, default_date=None):
1890 Record an apk (if it's new, otherwise does nothing)
1891 Returns the date it was added as a datetime instance
1893 if apkName not in self.apks:
1894 if default_date is None:
1895 default_date = datetime.utcnow()
1896 self.apks[apkName] = (app, default_date)
1898 _ignored, added = self.apks[apkName]
1901 def getapp(self, apkname):
1902 """Look up information - given the 'apkname', returns (app id, date added/None).
1904 Or returns None for an unknown apk.
1906 if apkname in self.apks:
1907 return self.apks[apkname]
1910 def getlatest(self, num):
1911 """Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first"""
1913 for apk, app in self.apks.items():
1917 if apps[appid] > added:
1921 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1922 lst = [app for app, _ignored in sortedapps]
1927 def get_file_extension(filename):
1928 """get the normalized file extension, can be blank string but never None"""
1929 if isinstance(filename, bytes):
1930 filename = filename.decode('utf-8')
1931 return os.path.splitext(filename)[1].lower()[1:]
1934 def use_androguard():
1935 """Report if androguard is available, and config its debug logging"""
1939 if use_androguard.show_path:
1940 logging.debug(_('Using androguard from "{path}"').format(path=androguard.__file__))
1941 use_androguard.show_path = False
1942 if options and options.verbose:
1943 logging.getLogger("androguard.axml").setLevel(logging.INFO)
1949 use_androguard.show_path = True
1952 def is_apk_and_debuggable_aapt(apkfile):
1953 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1955 if p.returncode != 0:
1956 raise FDroidException(_("Failed to get APK manifest information"))
1957 for line in p.output.splitlines():
1958 if 'android:debuggable' in line and not line.endswith('0x0'):
1963 def is_apk_and_debuggable_androguard(apkfile):
1965 from androguard.core.bytecodes.apk import APK
1967 raise FDroidException("androguard library is not installed and aapt not present")
1969 apkobject = APK(apkfile)
1970 if apkobject.is_valid_APK():
1971 debuggable = apkobject.get_element("application", "debuggable")
1972 if debuggable is not None:
1973 return bool(strtobool(debuggable))
1977 def is_apk_and_debuggable(apkfile):
1978 """Returns True if the given file is an APK and is debuggable
1980 :param apkfile: full path to the apk to check"""
1982 if get_file_extension(apkfile) != 'apk':
1985 if use_androguard():
1986 return is_apk_and_debuggable_androguard(apkfile)
1988 return is_apk_and_debuggable_aapt(apkfile)
1991 def get_apk_id_aapt(apkfile):
1992 """Extrat identification information from APK using aapt.
1994 :param apkfile: path to an APK file.
1995 :returns: triplet (appid, version code, version name)
1997 r = re.compile("^package: name='(?P<appid>.*)' versionCode='(?P<vercode>.*)' versionName='(?P<vername>.*)'.*")
1998 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1999 for line in p.output.splitlines():
2002 return m.group('appid'), m.group('vercode'), m.group('vername')
2003 raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'")
2004 .format(apkfilename=apkfile))
2007 def get_minSdkVersion_aapt(apkfile):
2008 """Extract the minimum supported Android SDK from an APK using aapt
2010 :param apkfile: path to an APK file.
2011 :returns: the integer representing the SDK version
2013 r = re.compile(r"^sdkVersion:'([0-9]+)'")
2014 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
2015 for line in p.output.splitlines():
2018 return int(m.group(1))
2019 raise FDroidException(_('Reading minSdkVersion failed: "{apkfilename}"')
2020 .format(apkfilename=apkfile))
2025 self.returncode = None
2029 def SdkToolsPopen(commands, cwd=None, output=True):
2031 if cmd not in config:
2032 config[cmd] = find_sdk_tools_cmd(commands[0])
2033 abscmd = config[cmd]
2035 raise FDroidException(_("Could not find '{command}' on your system").format(command=cmd))
2037 test_aapt_version(config['aapt'])
2038 return FDroidPopen([abscmd] + commands[1:],
2039 cwd=cwd, output=output)
2042 def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2044 Run a command and capture the possibly huge output as bytes.
2046 :param commands: command and argument list like in subprocess.Popen
2047 :param cwd: optionally specifies a working directory
2048 :param envs: a optional dictionary of environment variables and their values
2049 :returns: A PopenResult.
2054 set_FDroidPopen_env()
2056 process_env = env.copy()
2057 if envs is not None and len(envs) > 0:
2058 process_env.update(envs)
2061 cwd = os.path.normpath(cwd)
2062 logging.debug("Directory: %s" % cwd)
2063 logging.debug("> %s" % ' '.join(commands))
2065 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
2066 result = PopenResult()
2069 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
2070 stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
2071 stderr=stderr_param)
2072 except OSError as e:
2073 raise BuildException("OSError while trying to execute " +
2074 ' '.join(commands) + ': ' + str(e))
2076 # TODO are these AsynchronousFileReader threads always exiting?
2077 if not stderr_to_stdout and options.verbose:
2078 stderr_queue = Queue()
2079 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
2081 while not stderr_reader.eof():
2082 while not stderr_queue.empty():
2083 line = stderr_queue.get()
2084 sys.stderr.buffer.write(line)
2089 stdout_queue = Queue()
2090 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
2093 # Check the queue for output (until there is no more to get)
2094 while not stdout_reader.eof():
2095 while not stdout_queue.empty():
2096 line = stdout_queue.get()
2097 if output and options.verbose:
2098 # Output directly to console
2099 sys.stderr.buffer.write(line)
2105 result.returncode = p.wait()
2106 result.output = buf.getvalue()
2108 # make sure all filestreams of the subprocess are closed
2109 for streamvar in ['stdin', 'stdout', 'stderr']:
2110 if hasattr(p, streamvar):
2111 stream = getattr(p, streamvar)
2117 def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2119 Run a command and capture the possibly huge output as a str.
2121 :param commands: command and argument list like in subprocess.Popen
2122 :param cwd: optionally specifies a working directory
2123 :param envs: a optional dictionary of environment variables and their values
2124 :returns: A PopenResult.
2126 result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
2127 result.output = result.output.decode('utf-8', 'ignore')
2131 gradle_comment = re.compile(r'[ ]*//')
2132 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
2133 gradle_line_matches = [
2134 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
2135 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
2136 re.compile(r'.*\.readLine\(.*'),
2140 def remove_signing_keys(build_dir):
2141 for root, dirs, files in os.walk(build_dir):
2142 if 'build.gradle' in files:
2143 path = os.path.join(root, 'build.gradle')
2145 with open(path, "r", encoding='utf8') as o:
2146 lines = o.readlines()
2152 with open(path, "w", encoding='utf8') as o:
2153 while i < len(lines):
2156 while line.endswith('\\\n'):
2157 line = line.rstrip('\\\n') + lines[i]
2160 if gradle_comment.match(line):
2165 opened += line.count('{')
2166 opened -= line.count('}')
2169 if gradle_signing_configs.match(line):
2174 if any(s.match(line) for s in gradle_line_matches):
2182 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
2185 'project.properties',
2187 'default.properties',
2188 'ant.properties', ]:
2189 if propfile in files:
2190 path = os.path.join(root, propfile)
2192 with open(path, "r", encoding='iso-8859-1') as o:
2193 lines = o.readlines()
2197 with open(path, "w", encoding='iso-8859-1') as o:
2199 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
2206 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
2209 def set_FDroidPopen_env(build=None):
2211 set up the environment variables for the build environment
2213 There is only a weak standard, the variables used by gradle, so also set
2214 up the most commonly used environment variables for SDK and NDK. Also, if
2215 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
2217 global env, orig_path
2221 orig_path = env['PATH']
2222 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
2223 env[n] = config['sdk_path']
2224 for k, v in config['java_paths'].items():
2225 env['JAVA%s_HOME' % k] = v
2227 missinglocale = True
2228 for k, v in env.items():
2229 if k == 'LANG' and v != 'C':
2230 missinglocale = False
2232 missinglocale = False
2234 env['LANG'] = 'en_US.UTF-8'
2236 if build is not None:
2237 path = build.ndk_path()
2238 paths = orig_path.split(os.pathsep)
2239 if path not in paths:
2240 paths = [path] + paths
2241 env['PATH'] = os.pathsep.join(paths)
2242 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
2243 env[n] = build.ndk_path()
2246 def replace_build_vars(cmd, build):
2247 cmd = cmd.replace('$$COMMIT$$', build.commit)
2248 cmd = cmd.replace('$$VERSION$$', build.versionName)
2249 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
2253 def replace_config_vars(cmd, build):
2254 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
2255 cmd = cmd.replace('$$NDK$$', build.ndk_path())
2256 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
2257 if build is not None:
2258 cmd = replace_build_vars(cmd, build)
2262 def place_srclib(root_dir, number, libpath):
2265 relpath = os.path.relpath(libpath, root_dir)
2266 proppath = os.path.join(root_dir, 'project.properties')
2269 if os.path.isfile(proppath):
2270 with open(proppath, "r", encoding='iso-8859-1') as o:
2271 lines = o.readlines()
2273 with open(proppath, "w", encoding='iso-8859-1') as o:
2276 if line.startswith('android.library.reference.%d=' % number):
2277 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2282 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2285 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z_\-]+\.(SF|RSA|DSA|EC)')
2288 def signer_fingerprint_short(sig):
2289 """Obtain shortened sha256 signing-key fingerprint for pkcs7 signature.
2291 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2292 for a given pkcs7 signature.
2294 :param sig: Contents of an APK signing certificate.
2295 :returns: shortened signing-key fingerprint.
2297 return signer_fingerprint(sig)[:7]
2300 def signer_fingerprint(sig):
2301 """Obtain sha256 signing-key fingerprint for pkcs7 signature.
2303 Extracts hexadecimal sha256 signing-key fingerprint string
2304 for a given pkcs7 signature.
2306 :param: Contents of an APK signature.
2307 :returns: shortened signature fingerprint.
2309 cert_encoded = get_certificate(sig)
2310 return hashlib.sha256(cert_encoded).hexdigest()
2313 def apk_signer_fingerprint(apk_path):
2314 """Obtain sha256 signing-key fingerprint for APK.
2316 Extracts hexadecimal sha256 signing-key fingerprint string
2319 :param apkpath: path to APK
2320 :returns: signature fingerprint
2323 with zipfile.ZipFile(apk_path, 'r') as apk:
2324 certs = [n for n in apk.namelist() if CERT_PATH_REGEX.match(n)]
2327 logging.error("Found no signing certificates on %s" % apk_path)
2330 logging.error("Found multiple signing certificates on %s" % apk_path)
2333 cert = apk.read(certs[0])
2334 return signer_fingerprint(cert)
2337 def apk_signer_fingerprint_short(apk_path):
2338 """Obtain shortened sha256 signing-key fingerprint for APK.
2340 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2341 for a given pkcs7 APK.
2343 :param apk_path: path to APK
2344 :returns: shortened signing-key fingerprint
2346 return apk_signer_fingerprint(apk_path)[:7]
2349 def metadata_get_sigdir(appid, vercode=None):
2350 """Get signature directory for app"""
2352 return os.path.join('metadata', appid, 'signatures', vercode)
2354 return os.path.join('metadata', appid, 'signatures')
2357 def metadata_find_developer_signature(appid, vercode=None):
2358 """Tires to find the developer signature for given appid.
2360 This picks the first signature file found in metadata an returns its
2363 :returns: sha256 signing key fingerprint of the developer signing key.
2364 None in case no signature can not be found."""
2366 # fetch list of dirs for all versions of signatures
2369 appversigdirs.append(metadata_get_sigdir(appid, vercode))
2371 appsigdir = metadata_get_sigdir(appid)
2372 if os.path.isdir(appsigdir):
2373 numre = re.compile('[0-9]+')
2374 for ver in os.listdir(appsigdir):
2375 if numre.match(ver):
2376 appversigdir = os.path.join(appsigdir, ver)
2377 appversigdirs.append(appversigdir)
2379 for sigdir in appversigdirs:
2380 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2381 glob.glob(os.path.join(sigdir, '*.EC')) + \
2382 glob.glob(os.path.join(sigdir, '*.RSA'))
2384 raise FDroidException('ambiguous signatures, please make sure there is only one signature in \'{}\'. (The signature has to be the App maintainers signature for version of the APK.)'.format(sigdir))
2386 with open(sig, 'rb') as f:
2387 return signer_fingerprint(f.read())
2391 def metadata_find_signing_files(appid, vercode):
2392 """Gets a list of singed manifests and signatures.
2394 :param appid: app id string
2395 :param vercode: app version code
2396 :returns: a list of triplets for each signing key with following paths:
2397 (signature_file, singed_file, manifest_file)
2400 sigdir = metadata_get_sigdir(appid, vercode)
2401 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2402 glob.glob(os.path.join(sigdir, '*.EC')) + \
2403 glob.glob(os.path.join(sigdir, '*.RSA'))
2404 extre = re.compile('(\.DSA|\.EC|\.RSA)$')
2406 sf = extre.sub('.SF', sig)
2407 if os.path.isfile(sf):
2408 mf = os.path.join(sigdir, 'MANIFEST.MF')
2409 if os.path.isfile(mf):
2410 ret.append((sig, sf, mf))
2414 def metadata_find_developer_signing_files(appid, vercode):
2415 """Get developer signature files for specified app from metadata.
2417 :returns: A triplet of paths for signing files from metadata:
2418 (signature_file, singed_file, manifest_file)
2420 allsigningfiles = metadata_find_signing_files(appid, vercode)
2421 if allsigningfiles and len(allsigningfiles) == 1:
2422 return allsigningfiles[0]
2427 def apk_strip_signatures(signed_apk, strip_manifest=False):
2428 """Removes signatures from APK.
2430 :param signed_apk: path to apk file.
2431 :param strip_manifest: when set to True also the manifest file will
2432 be removed from the APK.
2434 with tempfile.TemporaryDirectory() as tmpdir:
2435 tmp_apk = os.path.join(tmpdir, 'tmp.apk')
2436 shutil.move(signed_apk, tmp_apk)
2437 with ZipFile(tmp_apk, 'r') as in_apk:
2438 with ZipFile(signed_apk, 'w') as out_apk:
2439 for info in in_apk.infolist():
2440 if not apk_sigfile.match(info.filename):
2442 if info.filename != 'META-INF/MANIFEST.MF':
2443 buf = in_apk.read(info.filename)
2444 out_apk.writestr(info, buf)
2446 buf = in_apk.read(info.filename)
2447 out_apk.writestr(info, buf)
2450 def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest):
2451 """Implats a signature from metadata into an APK.
2453 Note: this changes there supplied APK in place. So copy it if you
2454 need the original to be preserved.
2456 :param apkpath: location of the apk
2458 # get list of available signature files in metadata
2459 with tempfile.TemporaryDirectory() as tmpdir:
2460 apkwithnewsig = os.path.join(tmpdir, 'newsig.apk')
2461 with ZipFile(apkpath, 'r') as in_apk:
2462 with ZipFile(apkwithnewsig, 'w') as out_apk:
2463 for sig_file in [signaturefile, signedfile, manifest]:
2464 with open(sig_file, 'rb') as fp:
2466 info = zipfile.ZipInfo('META-INF/' + os.path.basename(sig_file))
2467 info.compress_type = zipfile.ZIP_DEFLATED
2468 info.create_system = 0 # "Windows" aka "FAT", what Android SDK uses
2469 out_apk.writestr(info, buf)
2470 for info in in_apk.infolist():
2471 if not apk_sigfile.match(info.filename):
2472 if info.filename != 'META-INF/MANIFEST.MF':
2473 buf = in_apk.read(info.filename)
2474 out_apk.writestr(info, buf)
2476 p = SdkToolsPopen(['zipalign', '-v', '4', apkwithnewsig, apkpath])
2477 if p.returncode != 0:
2478 raise BuildException("Failed to align application")
2481 def apk_extract_signatures(apkpath, outdir, manifest=True):
2482 """Extracts a signature files from APK and puts them into target directory.
2484 :param apkpath: location of the apk
2485 :param outdir: folder where the extracted signature files will be stored
2486 :param manifest: (optionally) disable extracting manifest file
2488 with ZipFile(apkpath, 'r') as in_apk:
2489 for f in in_apk.infolist():
2490 if apk_sigfile.match(f.filename) or \
2491 (manifest and f.filename == 'META-INF/MANIFEST.MF'):
2492 newpath = os.path.join(outdir, os.path.basename(f.filename))
2493 with open(newpath, 'wb') as out_file:
2494 out_file.write(in_apk.read(f.filename))
2497 def sign_apk(unsigned_path, signed_path, keyalias):
2498 """Sign and zipalign an unsigned APK, then save to a new file, deleting the unsigned
2500 android-18 (4.3) finally added support for reasonable hash
2501 algorithms, like SHA-256, before then, the only options were MD5
2502 and SHA1 :-/ This aims to use SHA-256 when the APK does not target
2503 older Android versions, and is therefore safe to do so.
2505 https://issuetracker.google.com/issues/36956587
2506 https://android-review.googlesource.com/c/platform/libcore/+/44491
2510 if get_minSdkVersion_aapt(unsigned_path) < 18:
2511 signature_algorithm = ['-sigalg', 'SHA1withRSA', '-digestalg', 'SHA1']
2513 signature_algorithm = ['-sigalg', 'SHA256withRSA', '-digestalg', 'SHA-256']
2515 p = FDroidPopen([config['jarsigner'], '-keystore', config['keystore'],
2516 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2517 '-keypass:env', 'FDROID_KEY_PASS']
2518 + signature_algorithm + [unsigned_path, keyalias],
2520 'FDROID_KEY_STORE_PASS': config['keystorepass'],
2521 'FDROID_KEY_PASS': config['keypass'], })
2522 if p.returncode != 0:
2523 raise BuildException(_("Failed to sign application"), p.output)
2525 p = SdkToolsPopen(['zipalign', '-v', '4', unsigned_path, signed_path])
2526 if p.returncode != 0:
2527 raise BuildException(_("Failed to zipalign application"))
2528 os.remove(unsigned_path)
2531 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
2532 """Verify that two apks are the same
2534 One of the inputs is signed, the other is unsigned. The signature metadata
2535 is transferred from the signed to the unsigned apk, and then jarsigner is
2536 used to verify that the signature from the signed apk is also varlid for
2537 the unsigned one. If the APK given as unsigned actually does have a
2538 signature, it will be stripped out and ignored.
2540 There are two SHA1 git commit IDs that fdroidserver includes in the builds
2541 it makes: fdroidserverid and buildserverid. Originally, these were inserted
2542 into AndroidManifest.xml, but that makes the build not reproducible. So
2543 instead they are included as separate files in the APK's META-INF/ folder.
2544 If those files exist in the signed APK, they will be part of the signature
2545 and need to also be included in the unsigned APK for it to validate.
2547 :param signed_apk: Path to a signed apk file
2548 :param unsigned_apk: Path to an unsigned apk file expected to match it
2549 :param tmp_dir: Path to directory for temporary files
2550 :returns: None if the verification is successful, otherwise a string
2551 describing what went wrong.
2554 if not os.path.isfile(signed_apk):
2555 return 'can not verify: file does not exists: {}'.format(signed_apk)
2557 if not os.path.isfile(unsigned_apk):
2558 return 'can not verify: file does not exists: {}'.format(unsigned_apk)
2560 with ZipFile(signed_apk, 'r') as signed:
2561 meta_inf_files = ['META-INF/MANIFEST.MF']
2562 for f in signed.namelist():
2563 if apk_sigfile.match(f) \
2564 or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
2565 meta_inf_files.append(f)
2566 if len(meta_inf_files) < 3:
2567 return "Signature files missing from {0}".format(signed_apk)
2569 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2570 with ZipFile(unsigned_apk, 'r') as unsigned:
2571 # only read the signature from the signed APK, everything else from unsigned
2572 with ZipFile(tmp_apk, 'w') as tmp:
2573 for filename in meta_inf_files:
2574 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2575 for info in unsigned.infolist():
2576 if info.filename in meta_inf_files:
2577 logging.warning('Ignoring %s from %s',
2578 info.filename, unsigned_apk)
2580 if info.filename in tmp.namelist():
2581 return "duplicate filename found: " + info.filename
2582 tmp.writestr(info, unsigned.read(info.filename))
2584 verified = verify_apk_signature(tmp_apk)
2587 logging.info("...NOT verified - {0}".format(tmp_apk))
2588 return compare_apks(signed_apk, tmp_apk, tmp_dir,
2589 os.path.dirname(unsigned_apk))
2591 logging.info("...successfully verified")
2595 def verify_jar_signature(jar):
2596 """Verifies the signature of a given JAR file.
2598 jarsigner is very shitty: unsigned JARs pass as "verified"! So
2599 this has to turn on -strict then check for result 4, since this
2600 does not expect the signature to be from a CA-signed certificate.
2602 :raises: VerificationException() if the JAR's signature could not be verified
2606 error = _('JAR signature failed to verify: {path}').format(path=jar)
2608 output = subprocess.check_output([config['jarsigner'], '-strict', '-verify', jar],
2609 stderr=subprocess.STDOUT)
2610 raise VerificationException(error + '\n' + output.decode('utf-8'))
2611 except subprocess.CalledProcessError as e:
2612 if e.returncode == 4:
2613 logging.debug(_('JAR signature verified: {path}').format(path=jar))
2615 raise VerificationException(error + '\n' + e.output.decode('utf-8'))
2618 def verify_apk_signature(apk, min_sdk_version=None):
2619 """verify the signature on an APK
2621 Try to use apksigner whenever possible since jarsigner is very
2622 shitty: unsigned APKs pass as "verified"! Warning, this does
2623 not work on JARs with apksigner >= 0.7 (build-tools 26.0.1)
2625 :returns: boolean whether the APK was verified
2627 if set_command_in_config('apksigner'):
2628 args = [config['apksigner'], 'verify']
2630 args += ['--min-sdk-version=' + min_sdk_version]
2632 args += ['--verbose']
2634 output = subprocess.check_output(args + [apk])
2636 logging.debug(apk + ': ' + output.decode('utf-8'))
2638 except subprocess.CalledProcessError as e:
2639 logging.error('\n' + apk + ': ' + e.output.decode('utf-8'))
2641 if not config.get('jarsigner_warning_displayed'):
2642 config['jarsigner_warning_displayed'] = True
2643 logging.warning(_("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner"))
2645 verify_jar_signature(apk)
2647 except Exception as e:
2652 def verify_old_apk_signature(apk):
2653 """verify the signature on an archived APK, supporting deprecated algorithms
2655 F-Droid aims to keep every single binary that it ever published. Therefore,
2656 it needs to be able to verify APK signatures that include deprecated/removed
2657 algorithms. For example, jarsigner treats an MD5 signature as unsigned.
2659 jarsigner passes unsigned APKs as "verified"! So this has to turn
2660 on -strict then check for result 4.
2662 :returns: boolean whether the APK was verified
2665 _java_security = os.path.join(os.getcwd(), '.java.security')
2666 with open(_java_security, 'w') as fp:
2667 fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
2671 config['jarsigner'],
2672 '-J-Djava.security.properties=' + _java_security,
2673 '-strict', '-verify', apk
2675 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
2676 except subprocess.CalledProcessError as e:
2677 if e.returncode != 4:
2680 logging.debug(_('JAR signature verified: {path}').format(path=apk))
2683 logging.error(_('Old APK signature failed to verify: {path}').format(path=apk)
2684 + '\n' + output.decode('utf-8'))
2688 apk_badchars = re.compile('''[/ :;'"]''')
2691 def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
2694 Returns None if the apk content is the same (apart from the signing key),
2695 otherwise a string describing what's different, or what went wrong when
2696 trying to do the comparison.
2702 absapk1 = os.path.abspath(apk1)
2703 absapk2 = os.path.abspath(apk2)
2705 if set_command_in_config('diffoscope'):
2706 logfilename = os.path.join(log_dir, os.path.basename(absapk1))
2707 htmlfile = logfilename + '.diffoscope.html'
2708 textfile = logfilename + '.diffoscope.txt'
2709 if subprocess.call([config['diffoscope'],
2710 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2711 '--html', htmlfile, '--text', textfile,
2712 absapk1, absapk2]) != 0:
2713 return("Failed to unpack " + apk1)
2715 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2716 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2717 for d in [apk1dir, apk2dir]:
2718 if os.path.exists(d):
2721 os.mkdir(os.path.join(d, 'jar-xf'))
2723 if subprocess.call(['jar', 'xf',
2724 os.path.abspath(apk1)],
2725 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2726 return("Failed to unpack " + apk1)
2727 if subprocess.call(['jar', 'xf',
2728 os.path.abspath(apk2)],
2729 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2730 return("Failed to unpack " + apk2)
2732 if set_command_in_config('apktool'):
2733 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2735 return("Failed to unpack " + apk1)
2736 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2738 return("Failed to unpack " + apk2)
2740 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2741 lines = p.output.splitlines()
2742 if len(lines) != 1 or 'META-INF' not in lines[0]:
2743 if set_command_in_config('meld'):
2744 p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
2745 return("Unexpected diff output - " + p.output)
2747 # since everything verifies, delete the comparison to keep cruft down
2748 shutil.rmtree(apk1dir)
2749 shutil.rmtree(apk2dir)
2751 # If we get here, it seems like they're the same!
2755 def set_command_in_config(command):
2756 '''Try to find specified command in the path, if it hasn't been
2757 manually set in config.py. If found, it is added to the config
2758 dict. The return value says whether the command is available.
2761 if command in config:
2764 tmp = find_command(command)
2766 config[command] = tmp
2771 def find_command(command):
2772 '''find the full path of a command, or None if it can't be found in the PATH'''
2775 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2777 fpath, fname = os.path.split(command)
2782 for path in os.environ["PATH"].split(os.pathsep):
2783 path = path.strip('"')
2784 exe_file = os.path.join(path, command)
2785 if is_exe(exe_file):
2792 '''generate a random password for when generating keys'''
2793 h = hashlib.sha256()
2794 h.update(os.urandom(16)) # salt
2795 h.update(socket.getfqdn().encode('utf-8'))
2796 passwd = base64.b64encode(h.digest()).strip()
2797 return passwd.decode('utf-8')
2800 def genkeystore(localconfig):
2802 Generate a new key with password provided in :param localconfig and add it to new keystore
2803 :return: hexed public key, public key fingerprint
2805 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2806 keystoredir = os.path.dirname(localconfig['keystore'])
2807 if keystoredir is None or keystoredir == '':
2808 keystoredir = os.path.join(os.getcwd(), keystoredir)
2809 if not os.path.exists(keystoredir):
2810 os.makedirs(keystoredir, mode=0o700)
2813 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
2814 'FDROID_KEY_PASS': localconfig['keypass'],
2816 p = FDroidPopen([config['keytool'], '-genkey',
2817 '-keystore', localconfig['keystore'],
2818 '-alias', localconfig['repo_keyalias'],
2819 '-keyalg', 'RSA', '-keysize', '4096',
2820 '-sigalg', 'SHA256withRSA',
2821 '-validity', '10000',
2822 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2823 '-keypass:env', 'FDROID_KEY_PASS',
2824 '-dname', localconfig['keydname']], envs=env_vars)
2825 if p.returncode != 0:
2826 raise BuildException("Failed to generate key", p.output)
2827 os.chmod(localconfig['keystore'], 0o0600)
2828 if not options.quiet:
2829 # now show the lovely key that was just generated
2830 p = FDroidPopen([config['keytool'], '-list', '-v',
2831 '-keystore', localconfig['keystore'],
2832 '-alias', localconfig['repo_keyalias'],
2833 '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
2834 logging.info(p.output.strip() + '\n\n')
2835 # get the public key
2836 p = FDroidPopenBytes([config['keytool'], '-exportcert',
2837 '-keystore', localconfig['keystore'],
2838 '-alias', localconfig['repo_keyalias'],
2839 '-storepass:env', 'FDROID_KEY_STORE_PASS']
2840 + config['smartcardoptions'],
2841 envs=env_vars, output=False, stderr_to_stdout=False)
2842 if p.returncode != 0 or len(p.output) < 20:
2843 raise BuildException("Failed to get public key", p.output)
2845 fingerprint = get_cert_fingerprint(pubkey)
2846 return hexlify(pubkey), fingerprint
2849 def get_cert_fingerprint(pubkey):
2851 Generate a certificate fingerprint the same way keytool does it
2852 (but with slightly different formatting)
2854 digest = hashlib.sha256(pubkey).digest()
2855 ret = [' '.join("%02X" % b for b in bytearray(digest))]
2856 return " ".join(ret)
2859 def get_certificate(certificate_file):
2861 Extracts a certificate from the given file.
2862 :param certificate_file: file bytes (as string) representing the certificate
2863 :return: A binary representation of the certificate's public key, or None in case of error
2865 content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
2866 if content.getComponentByName('contentType') != rfc2315.signedData:
2868 content = decoder.decode(content.getComponentByName('content'),
2869 asn1Spec=rfc2315.SignedData())[0]
2871 certificates = content.getComponentByName('certificates')
2872 cert = certificates[0].getComponentByName('certificate')
2874 logging.error("Certificates not found.")
2876 return encoder.encode(cert)
2879 def load_stats_fdroid_signing_key_fingerprints():
2880 """Load list of signing-key fingerprints stored by fdroid publish from file.
2882 :returns: list of dictionanryies containing the singing-key fingerprints.
2884 jar_file = os.path.join('stats', 'publishsigkeys.jar')
2885 if not os.path.isfile(jar_file):
2887 cmd = [config['jarsigner'], '-strict', '-verify', jar_file]
2888 p = FDroidPopen(cmd, output=False)
2889 if p.returncode != 4:
2890 raise FDroidException("Signature validation of '{}' failed! "
2891 "Please run publish again to rebuild this file.".format(jar_file))
2893 jar_sigkey = apk_signer_fingerprint(jar_file)
2894 repo_key_sig = config.get('repo_key_sha256')
2896 if jar_sigkey != repo_key_sig:
2897 raise FDroidException("Signature key fingerprint of file '{}' does not match repo_key_sha256 in config.py (found fingerprint: '{}')".format(jar_file, jar_sigkey))
2899 logging.warning("repo_key_sha256 not in config.py, setting it to the signature key fingerprint of '{}'".format(jar_file))
2900 config['repo_key_sha256'] = jar_sigkey
2901 write_to_config(config, 'repo_key_sha256')
2903 with zipfile.ZipFile(jar_file, 'r') as f:
2904 return json.loads(str(f.read('publishsigkeys.json'), 'utf-8'))
2907 def write_to_config(thisconfig, key, value=None, config_file=None):
2908 '''write a key/value to the local config.py
2910 NOTE: only supports writing string variables.
2912 :param thisconfig: config dictionary
2913 :param key: variable name in config.py to be overwritten/added
2914 :param value: optional value to be written, instead of fetched
2915 from 'thisconfig' dictionary.
2918 origkey = key + '_orig'
2919 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2920 cfg = config_file if config_file else 'config.py'
2922 # load config file, create one if it doesn't exist
2923 if not os.path.exists(cfg):
2924 open(cfg, 'a').close()
2925 logging.info("Creating empty " + cfg)
2926 with open(cfg, 'r', encoding="utf-8") as f:
2927 lines = f.readlines()
2929 # make sure the file ends with a carraige return
2931 if not lines[-1].endswith('\n'):
2934 # regex for finding and replacing python string variable
2935 # definitions/initializations
2936 pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
2937 repl = key + ' = "' + value + '"'
2938 pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
2939 repl2 = key + " = '" + value + "'"
2941 # If we replaced this line once, we make sure won't be a
2942 # second instance of this line for this key in the document.
2945 with open(cfg, 'w', encoding="utf-8") as f:
2947 if pattern.match(line) or pattern2.match(line):
2949 line = pattern.sub(repl, line)
2950 line = pattern2.sub(repl2, line)
2961 def parse_xml(path):
2962 return XMLElementTree.parse(path).getroot()
2965 def string_is_integer(string):
2973 def local_rsync(options, fromdir, todir):
2974 '''Rsync method for local to local copying of things
2976 This is an rsync wrapper with all the settings for safe use within
2977 the various fdroidserver use cases. This uses stricter rsync
2978 checking on all files since people using offline mode are already
2979 prioritizing security above ease and speed.
2982 rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
2983 '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
2984 if not options.no_checksum:
2985 rsyncargs.append('--checksum')
2987 rsyncargs += ['--verbose']
2989 rsyncargs += ['--quiet']
2990 logging.debug(' '.join(rsyncargs + [fromdir, todir]))
2991 if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
2992 raise FDroidException()
2995 def get_per_app_repos():
2996 '''per-app repos are dirs named with the packageName of a single app'''
2998 # Android packageNames are Java packages, they may contain uppercase or
2999 # lowercase letters ('A' through 'Z'), numbers, and underscores
3000 # ('_'). However, individual package name parts may only start with
3001 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
3002 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
3005 for root, dirs, files in os.walk(os.getcwd()):
3007 print('checking', root, 'for', d)
3008 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
3009 # standard parts of an fdroid repo, so never packageNames
3012 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
3018 def is_repo_file(filename):
3019 '''Whether the file in a repo is a build product to be delivered to users'''
3020 if isinstance(filename, str):
3021 filename = filename.encode('utf-8', errors="surrogateescape")
3022 return os.path.isfile(filename) \
3023 and not filename.endswith(b'.asc') \
3024 and not filename.endswith(b'.sig') \
3025 and os.path.basename(filename) not in [
3027 b'index_unsigned.jar',
3036 def get_examples_dir():
3037 '''Return the dir where the fdroidserver example files are available'''
3039 tmp = os.path.dirname(sys.argv[0])
3040 if os.path.basename(tmp) == 'bin':
3041 egg_links = glob.glob(os.path.join(tmp, '..',
3042 'local/lib/python3.*/site-packages/fdroidserver.egg-link'))
3044 # installed from local git repo
3045 examplesdir = os.path.join(open(egg_links[0]).readline().rstrip(), 'examples')
3048 examplesdir = os.path.dirname(os.path.dirname(__file__)) + '/share/doc/fdroidserver/examples'
3049 if not os.path.exists(examplesdir): # use UNIX layout
3050 examplesdir = os.path.dirname(tmp) + '/share/doc/fdroidserver/examples'
3052 # we're running straight out of the git repo
3053 prefix = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
3054 examplesdir = prefix + '/examples'
3059 def get_wiki_timestamp(timestamp=None):
3060 """Return current time in the standard format for posting to the wiki"""
3062 if timestamp is None:
3063 timestamp = time.gmtime()
3064 return time.strftime("%Y-%m-%d %H:%M:%SZ", timestamp)
3067 def get_android_tools_versions(ndk_path=None):
3068 '''get a list of the versions of all installed Android SDK/NDK components'''
3071 sdk_path = config['sdk_path']
3072 if sdk_path[-1] != '/':
3076 ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT')
3077 if os.path.isfile(ndk_release_txt):
3078 with open(ndk_release_txt, 'r') as fp:
3079 components.append((os.path.basename(ndk_path), fp.read()[:-1]))
3081 pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE)
3082 for root, dirs, files in os.walk(sdk_path):
3083 if 'source.properties' in files:
3084 source_properties = os.path.join(root, 'source.properties')
3085 with open(source_properties, 'r') as fp:
3086 m = pattern.search(fp.read())
3088 components.append((root[len(sdk_path):], m.group(1)))
3093 def get_android_tools_version_log(ndk_path=None):
3094 '''get a list of the versions of all installed Android SDK/NDK components'''
3095 log = '== Installed Android Tools ==\n\n'
3096 components = get_android_tools_versions(ndk_path)
3097 for name, version in sorted(components):
3098 log += '* ' + name + ' (' + version + ')\n'