3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
40 import xml.etree.ElementTree as XMLElementTree
42 from binascii import hexlify
43 from datetime import datetime, timedelta
44 from distutils.version import LooseVersion
45 from queue import Queue
46 from zipfile import ZipFile
48 from pyasn1.codec.der import decoder, encoder
49 from pyasn1_modules import rfc2315
50 from pyasn1.error import PyAsn1Error
52 from distutils.util import strtobool
54 import fdroidserver.metadata
55 from fdroidserver import _
56 from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException,\
57 BuildException, VerificationException
58 from .asynchronousfilereader import AsynchronousFileReader
60 # this is the build-tools version, aapt has a separate version that
61 # has to be manually set in test_aapt_version()
62 MINIMUM_AAPT_VERSION = '26.0.0'
64 # A signature block file with a .DSA, .RSA, or .EC extension
65 CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
66 APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
67 STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
69 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
78 'sdk_path': "$ANDROID_HOME",
82 'r12b': "$ANDROID_NDK",
88 'build_tools': MINIMUM_AAPT_VERSION,
89 'force_build_tools': False,
94 'accepted_formats': ['txt', 'yml'],
95 'sync_from_local_copy_dir': False,
96 'allow_disabled_algorithms': False,
97 'per_app_repos': False,
98 'make_current_version_link': True,
99 'current_version_name_source': 'Name',
100 'update_stats': False,
102 'stats_server': None,
104 'stats_to_carbon': False,
106 'build_server_always': False,
107 'keystore': 'keystore.jks',
108 'smartcardoptions': [],
118 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
119 'repo_name': "My First FDroid Repo Demo",
120 'repo_icon': "fdroid-icon.png",
121 'repo_description': '''
122 This is a repository of apps to be used with FDroid. Applications in this
123 repository are either official binaries built by the original application
124 developers, or are binaries built from source by the admin of f-droid.org
125 using the tools on https://gitlab.com/u/fdroid.
131 def setup_global_opts(parser):
132 try: # the buildserver VM might not have PIL installed
133 from PIL import PngImagePlugin
134 logger = logging.getLogger(PngImagePlugin.__name__)
135 logger.setLevel(logging.INFO) # tame the "STREAM" debug messages
139 parser.add_argument("-v", "--verbose", action="store_true", default=False,
140 help=_("Spew out even more information than normal"))
141 parser.add_argument("-q", "--quiet", action="store_true", default=False,
142 help=_("Restrict output to warnings and errors"))
145 def _add_java_paths_to_config(pathlist, thisconfig):
146 def path_version_key(s):
148 for u in re.split('[^0-9]+', s):
150 versionlist.append(int(u))
155 for d in sorted(pathlist, key=path_version_key):
156 if os.path.islink(d):
158 j = os.path.basename(d)
159 # the last one found will be the canonical one, so order appropriately
161 r'^1\.([6-9])\.0\.jdk$', # OSX
162 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
163 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
164 r'^jdk([6-9])-openjdk$', # Arch
165 r'^java-([6-9])-openjdk$', # Arch
166 r'^java-([6-9])-jdk$', # Arch (oracle)
167 r'^java-1\.([6-9])\.0-.*$', # RedHat
168 r'^java-([6-9])-oracle$', # Debian WebUpd8
169 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
170 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
172 m = re.match(regex, j)
175 for p in [d, os.path.join(d, 'Contents', 'Home')]:
176 if os.path.exists(os.path.join(p, 'bin', 'javac')):
177 thisconfig['java_paths'][m.group(1)] = p
180 def fill_config_defaults(thisconfig):
181 for k, v in default_config.items():
182 if k not in thisconfig:
185 # Expand paths (~users and $vars)
186 def expand_path(path):
190 path = os.path.expanduser(path)
191 path = os.path.expandvars(path)
196 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
201 thisconfig[k + '_orig'] = v
203 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
204 if thisconfig['java_paths'] is None:
205 thisconfig['java_paths'] = dict()
207 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
208 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
209 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
210 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
211 if os.getenv('JAVA_HOME') is not None:
212 pathlist.append(os.getenv('JAVA_HOME'))
213 if os.getenv('PROGRAMFILES') is not None:
214 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
215 _add_java_paths_to_config(pathlist, thisconfig)
217 for java_version in ('7', '8', '9'):
218 if java_version not in thisconfig['java_paths']:
220 java_home = thisconfig['java_paths'][java_version]
221 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
222 if os.path.exists(jarsigner):
223 thisconfig['jarsigner'] = jarsigner
224 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
225 break # Java7 is preferred, so quit if found
227 for k in ['ndk_paths', 'java_paths']:
233 thisconfig[k][k2] = exp
234 thisconfig[k][k2 + '_orig'] = v
237 def regsub_file(pattern, repl, path):
238 with open(path, 'rb') as f:
240 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
241 with open(path, 'wb') as f:
245 def read_config(opts, config_file='config.py'):
246 """Read the repository config
248 The config is read from config_file, which is in the current
249 directory when any of the repo management commands are used. If
250 there is a local metadata file in the git repo, then config.py is
251 not required, just use defaults.
254 global config, options
256 if config is not None:
263 if os.path.isfile(config_file):
264 logging.debug(_("Reading '{config_file}'").format(config_file=config_file))
265 with io.open(config_file, "rb") as f:
266 code = compile(f.read(), config_file, 'exec')
267 exec(code, None, config)
269 logging.warning(_("No 'config.py' found, using defaults."))
271 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
273 if not type(config[k]) in (str, list, tuple):
275 _("'{field}' will be in random order! Use () or [] brackets if order is important!")
278 # smartcardoptions must be a list since its command line args for Popen
279 if 'smartcardoptions' in config:
280 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
281 elif 'keystore' in config and config['keystore'] == 'NONE':
282 # keystore='NONE' means use smartcard, these are required defaults
283 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
284 'SunPKCS11-OpenSC', '-providerClass',
285 'sun.security.pkcs11.SunPKCS11',
286 '-providerArg', 'opensc-fdroid.cfg']
288 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
289 st = os.stat(config_file)
290 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
291 logging.warning(_("unsafe permissions on '{config_file}' (should be 0600)!")
292 .format(config_file=config_file))
294 fill_config_defaults(config)
296 for k in ["repo_description", "archive_description"]:
298 config[k] = clean_description(config[k])
300 if 'serverwebroot' in config:
301 if isinstance(config['serverwebroot'], str):
302 roots = [config['serverwebroot']]
303 elif all(isinstance(item, str) for item in config['serverwebroot']):
304 roots = config['serverwebroot']
306 raise TypeError(_('only accepts strings, lists, and tuples'))
308 for rootstr in roots:
309 # since this is used with rsync, where trailing slashes have
310 # meaning, ensure there is always a trailing slash
311 if rootstr[-1] != '/':
313 rootlist.append(rootstr.replace('//', '/'))
314 config['serverwebroot'] = rootlist
316 if 'servergitmirrors' in config:
317 if isinstance(config['servergitmirrors'], str):
318 roots = [config['servergitmirrors']]
319 elif all(isinstance(item, str) for item in config['servergitmirrors']):
320 roots = config['servergitmirrors']
322 raise TypeError(_('only accepts strings, lists, and tuples'))
323 config['servergitmirrors'] = roots
328 def assert_config_keystore(config):
329 """Check weather keystore is configured correctly and raise exception if not."""
332 if 'repo_keyalias' not in config:
334 logging.critical(_("'repo_keyalias' not found in config.py!"))
335 if 'keystore' not in config:
337 logging.critical(_("'keystore' not found in config.py!"))
338 elif not os.path.exists(config['keystore']):
340 logging.critical("'" + config['keystore'] + "' does not exist!")
341 if 'keystorepass' not in config:
343 logging.critical(_("'keystorepass' not found in config.py!"))
344 if 'keypass' not in config:
346 logging.critical(_("'keypass' not found in config.py!"))
348 raise FDroidException("This command requires a signing key, " +
349 "you can create one using: fdroid update --create-key")
352 def find_sdk_tools_cmd(cmd):
353 '''find a working path to a tool from the Android SDK'''
356 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
357 # try to find a working path to this command, in all the recent possible paths
358 if 'build_tools' in config:
359 build_tools = os.path.join(config['sdk_path'], 'build-tools')
360 # if 'build_tools' was manually set and exists, check only that one
361 configed_build_tools = os.path.join(build_tools, config['build_tools'])
362 if os.path.exists(configed_build_tools):
363 tooldirs.append(configed_build_tools)
365 # no configed version, so hunt known paths for it
366 for f in sorted(os.listdir(build_tools), reverse=True):
367 if os.path.isdir(os.path.join(build_tools, f)):
368 tooldirs.append(os.path.join(build_tools, f))
369 tooldirs.append(build_tools)
370 sdk_tools = os.path.join(config['sdk_path'], 'tools')
371 if os.path.exists(sdk_tools):
372 tooldirs.append(sdk_tools)
373 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
374 if os.path.exists(sdk_platform_tools):
375 tooldirs.append(sdk_platform_tools)
376 tooldirs.append('/usr/bin')
378 path = os.path.join(d, cmd)
379 if os.path.isfile(path):
381 test_aapt_version(path)
383 # did not find the command, exit with error message
384 ensure_build_tools_exists(config)
387 def test_aapt_version(aapt):
388 '''Check whether the version of aapt is new enough'''
389 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
390 if output is None or output == '':
391 logging.error(_("'{path}' failed to execute!").format(path=aapt))
393 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
398 # the Debian package has the version string like "v0.2-23.0.2"
401 if LooseVersion(bugfix) < LooseVersion(MINIMUM_AAPT_VERSION):
403 elif LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.4062713'):
406 logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-{version} or newer!")
407 .format(aapt=aapt, version=MINIMUM_AAPT_VERSION))
409 logging.warning(_('Unknown version of aapt, might cause problems: ') + output)
412 def test_sdk_exists(thisconfig):
413 if 'sdk_path' not in thisconfig:
414 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
415 test_aapt_version(thisconfig['aapt'])
418 logging.error(_("'sdk_path' not set in 'config.py'!"))
420 if thisconfig['sdk_path'] == default_config['sdk_path']:
421 logging.error(_('No Android SDK found!'))
422 logging.error(_('You can use ANDROID_HOME to set the path to your SDK, i.e.:'))
423 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
425 if not os.path.exists(thisconfig['sdk_path']):
426 logging.critical(_("Android SDK path '{path}' does not exist!")
427 .format(path=thisconfig['sdk_path']))
429 if not os.path.isdir(thisconfig['sdk_path']):
430 logging.critical(_("Android SDK path '{path}' is not a directory!")
431 .format(path=thisconfig['sdk_path']))
433 for d in ['build-tools', 'platform-tools', 'tools']:
434 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
435 logging.critical(_("Android SDK '{path}' does not have '{dirname}' installed!")
436 .format(path=thisconfig['sdk_path'], dirname=d))
441 def ensure_build_tools_exists(thisconfig):
442 if not test_sdk_exists(thisconfig):
443 raise FDroidException(_("Android SDK not found!"))
444 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
445 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
446 if not os.path.isdir(versioned_build_tools):
447 raise FDroidException(
448 _("Android build-tools path '{path}' does not exist!")
449 .format(path=versioned_build_tools))
452 def get_local_metadata_files():
453 '''get any metadata files local to an app's source repo
455 This tries to ignore anything that does not count as app metdata,
456 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
459 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
462 def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False):
464 :param appids: arguments in the form of multiple appid:[vc] strings
465 :returns: a dictionary with the set of vercodes specified for each package
468 if not appid_versionCode_pairs:
471 for p in appid_versionCode_pairs:
472 if allow_vercodes and ':' in p:
473 package, vercode = p.split(':')
475 package, vercode = p, None
476 if package not in vercodes:
477 vercodes[package] = [vercode] if vercode else []
479 elif vercode and vercode not in vercodes[package]:
480 vercodes[package] += [vercode] if vercode else []
485 def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
486 """Build a list of App instances for processing
488 On top of what read_pkg_args does, this returns the whole app
489 metadata, but limiting the builds list to the builds matching the
490 appid_versionCode_pairs and vercodes specified. If no appid_versionCode_pairs are specified, then
491 all App and Build instances are returned.
495 vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes)
501 for appid, app in allapps.items():
502 if appid in vercodes:
505 if len(apps) != len(vercodes):
508 logging.critical(_("No such package: %s") % p)
509 raise FDroidException(_("Found invalid appids in arguments"))
511 raise FDroidException(_("No packages specified"))
514 for appid, app in apps.items():
518 app.builds = [b for b in app.builds if b.versionCode in vc]
519 if len(app.builds) != len(vercodes[appid]):
521 allvcs = [b.versionCode for b in app.builds]
522 for v in vercodes[appid]:
524 logging.critical(_("No such versionCode {versionCode} for app {appid}")
525 .format(versionCode=v, appid=appid))
528 raise FDroidException(_("Found invalid versionCodes for some apps"))
533 def get_extension(filename):
534 base, ext = os.path.splitext(filename)
537 return base, ext.lower()[1:]
540 def has_extension(filename, ext):
541 _ignored, f_ext = get_extension(filename)
545 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
548 def clean_description(description):
549 'Remove unneeded newlines and spaces from a block of description text'
551 # this is split up by paragraph to make removing the newlines easier
552 for paragraph in re.split(r'\n\n', description):
553 paragraph = re.sub('\r', '', paragraph)
554 paragraph = re.sub('\n', ' ', paragraph)
555 paragraph = re.sub(' {2,}', ' ', paragraph)
556 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
557 returnstring += paragraph + '\n\n'
558 return returnstring.rstrip('\n')
561 def publishednameinfo(filename):
562 filename = os.path.basename(filename)
563 m = publish_name_regex.match(filename)
565 result = (m.group(1), m.group(2))
566 except AttributeError:
567 raise FDroidException(_("Invalid name for published file: %s") % filename)
571 apk_release_filename = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)\.apk')
572 apk_release_filename_with_sigfp = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)_(?P<sigfp>[0-9a-f]{7})\.apk')
575 def apk_parse_release_filename(apkname):
576 """Parses the name of an APK file according the F-Droids APK naming
577 scheme and returns the tokens.
579 WARNING: Returned values don't necessarily represent the APKs actual
580 properties, the are just paresed from the file name.
582 :returns: A triplet containing (appid, versionCode, signer), where appid
583 should be the package name, versionCode should be the integer
584 represion of the APKs version and signer should be the first 7 hex
585 digists of the sha256 signing key fingerprint which was used to sign
588 m = apk_release_filename_with_sigfp.match(apkname)
590 return m.group('appid'), m.group('vercode'), m.group('sigfp')
591 m = apk_release_filename.match(apkname)
593 return m.group('appid'), m.group('vercode'), None
594 return None, None, None
597 def get_release_filename(app, build):
599 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
601 return "%s_%s.apk" % (app.id, build.versionCode)
604 def get_toolsversion_logname(app, build):
605 return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
608 def getsrcname(app, build):
609 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
621 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
624 def get_build_dir(app):
625 '''get the dir that this app will be built in'''
627 if app.RepoType == 'srclib':
628 return os.path.join('build', 'srclib', app.Repo)
630 return os.path.join('build', app.id)
634 '''checkout code from VCS and return instance of vcs and the build dir'''
635 build_dir = get_build_dir(app)
637 # Set up vcs interface and make sure we have the latest code...
638 logging.debug("Getting {0} vcs interface for {1}"
639 .format(app.RepoType, app.Repo))
640 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
644 vcs = getvcs(app.RepoType, remote, build_dir)
646 return vcs, build_dir
649 def getvcs(vcstype, remote, local):
651 return vcs_git(remote, local)
652 if vcstype == 'git-svn':
653 return vcs_gitsvn(remote, local)
655 return vcs_hg(remote, local)
657 return vcs_bzr(remote, local)
658 if vcstype == 'srclib':
659 if local != os.path.join('build', 'srclib', remote):
660 raise VCSException("Error: srclib paths are hard-coded!")
661 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
663 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
664 raise VCSException("Invalid vcs type " + vcstype)
667 def getsrclibvcs(name):
668 if name not in fdroidserver.metadata.srclibs:
669 raise VCSException("Missing srclib " + name)
670 return fdroidserver.metadata.srclibs[name]['Repo Type']
675 def __init__(self, remote, local):
677 # svn, git-svn and bzr may require auth
679 if self.repotype() in ('git-svn', 'bzr'):
681 if self.repotype == 'git-svn':
682 raise VCSException("Authentication is not supported for git-svn")
683 self.username, remote = remote.split('@')
684 if ':' not in self.username:
685 raise VCSException(_("Password required with username"))
686 self.username, self.password = self.username.split(':')
690 self.clone_failed = False
691 self.refreshed = False
697 def clientversion(self):
698 versionstr = FDroidPopen(self.clientversioncmd()).output
699 return versionstr[0:versionstr.find('\n')]
701 def clientversioncmd(self):
704 def gotorevision(self, rev, refresh=True):
705 """Take the local repository to a clean version of the given
706 revision, which is specificed in the VCS's native
707 format. Beforehand, the repository can be dirty, or even
708 non-existent. If the repository does already exist locally, it
709 will be updated from the origin, but only once in the lifetime
710 of the vcs object. None is acceptable for 'rev' if you know
711 you are cloning a clean copy of the repo - otherwise it must
712 specify a valid revision.
715 if self.clone_failed:
716 raise VCSException(_("Downloading the repository already failed once, not trying again."))
718 # The .fdroidvcs-id file for a repo tells us what VCS type
719 # and remote that directory was created from, allowing us to drop it
720 # automatically if either of those things changes.
721 fdpath = os.path.join(self.local, '..',
722 '.fdroidvcs-' + os.path.basename(self.local))
723 fdpath = os.path.normpath(fdpath)
724 cdata = self.repotype() + ' ' + self.remote
727 if os.path.exists(self.local):
728 if os.path.exists(fdpath):
729 with open(fdpath, 'r') as f:
730 fsdata = f.read().strip()
735 logging.info("Repository details for %s changed - deleting" % (
739 logging.info("Repository details for %s missing - deleting" % (
742 shutil.rmtree(self.local)
746 self.refreshed = True
749 self.gotorevisionx(rev)
750 except FDroidException as e:
753 # If necessary, write the .fdroidvcs file.
754 if writeback and not self.clone_failed:
755 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
756 with open(fdpath, 'w+') as f:
762 def gotorevisionx(self, rev): # pylint: disable=unused-argument
763 """Derived classes need to implement this.
765 It's called once basic checking has been performed.
767 raise VCSException("This VCS type doesn't define gotorevisionx")
769 # Initialise and update submodules
770 def initsubmodules(self):
771 raise VCSException('Submodules not supported for this vcs type')
773 # Get a list of all known tags
775 if not self._gettags:
776 raise VCSException('gettags not supported for this vcs type')
778 for tag in self._gettags():
779 if re.match('[-A-Za-z0-9_. /]+$', tag):
783 def latesttags(self):
784 """Get a list of all the known tags, sorted from newest to oldest"""
785 raise VCSException('latesttags not supported for this vcs type')
788 """Get current commit reference (hash, revision, etc)"""
789 raise VCSException('getref not supported for this vcs type')
792 """Returns the srclib (name, path) used in setting up the current revision, or None."""
801 def clientversioncmd(self):
802 return ['git', '--version']
804 def git(self, args, envs=dict(), cwd=None, output=True):
805 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
807 While fetch/pull/clone respect the command line option flags,
808 it seems that submodule commands do not. They do seem to
809 follow whatever is in env vars, if the version of git is new
810 enough. So we just throw the kitchen sink at it to see what
813 Also, because of CVE-2017-1000117, block all SSH URLs.
816 # supported in git >= 2.3
818 '-c', 'core.askpass=/bin/true',
819 '-c', 'core.sshCommand=/bin/false',
820 '-c', 'url.https://.insteadOf=ssh://',
822 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
823 git_config.append('-c')
824 git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':')
825 git_config.append('-c')
826 git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain)
827 git_config.append('-c')
828 git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain)
830 'GIT_TERMINAL_PROMPT': '0',
831 'GIT_ASKPASS': '/bin/true',
832 'SSH_ASKPASS': '/bin/true',
833 'GIT_SSH': '/bin/false', # for git < 2.3
835 return FDroidPopen(['git', ] + git_config + args,
836 envs=envs, cwd=cwd, output=output)
839 """If the local directory exists, but is somehow not a git repository,
840 git will traverse up the directory tree until it finds one
841 that is (i.e. fdroidserver) and then we'll proceed to destroy
842 it! This is called as a safety check.
846 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
847 result = p.output.rstrip()
848 if not result.endswith(self.local):
849 raise VCSException('Repository mismatch')
851 def gotorevisionx(self, rev):
852 if not os.path.exists(self.local):
854 p = self.git(['clone', '--', self.remote, self.local])
855 if p.returncode != 0:
856 self.clone_failed = True
857 raise VCSException("Git clone failed", p.output)
861 # Discard any working tree changes
862 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
863 'git', 'reset', '--hard'], cwd=self.local, output=False)
864 if p.returncode != 0:
865 raise VCSException(_("Git reset failed"), p.output)
866 # Remove untracked files now, in case they're tracked in the target
867 # revision (it happens!)
868 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
869 'git', 'clean', '-dffx'], cwd=self.local, output=False)
870 if p.returncode != 0:
871 raise VCSException(_("Git clean failed"), p.output)
872 if not self.refreshed:
873 # Get latest commits and tags from remote
874 p = self.git(['fetch', 'origin'], cwd=self.local)
875 if p.returncode != 0:
876 raise VCSException(_("Git fetch failed"), p.output)
877 p = self.git(['fetch', '--prune', '--tags', 'origin'], output=False, cwd=self.local)
878 if p.returncode != 0:
879 raise VCSException(_("Git fetch failed"), p.output)
880 # Recreate origin/HEAD as git clone would do it, in case it disappeared
881 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
882 if p.returncode != 0:
883 lines = p.output.splitlines()
884 if 'Multiple remote HEAD branches' not in lines[0]:
885 raise VCSException(_("Git remote set-head failed"), p.output)
886 branch = lines[1].split(' ')[-1]
887 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--', branch],
888 cwd=self.local, output=False)
889 if p2.returncode != 0:
890 raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output)
891 self.refreshed = True
892 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
893 # a github repo. Most of the time this is the same as origin/master.
894 rev = rev or 'origin/HEAD'
895 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
896 if p.returncode != 0:
897 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
898 # Get rid of any uncontrolled files left behind
899 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
900 if p.returncode != 0:
901 raise VCSException(_("Git clean failed"), p.output)
903 def initsubmodules(self):
905 submfile = os.path.join(self.local, '.gitmodules')
906 if not os.path.isfile(submfile):
907 raise NoSubmodulesException(_("No git submodules available"))
909 # fix submodules not accessible without an account and public key auth
910 with open(submfile, 'r') as f:
911 lines = f.readlines()
912 with open(submfile, 'w') as f:
914 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
915 line = re.sub('git@' + domain + ':', 'https://u:p@' + domain + '/', line)
918 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
919 if p.returncode != 0:
920 raise VCSException(_("Git submodule sync failed"), p.output)
921 p = self.git(['submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
922 if p.returncode != 0:
923 raise VCSException(_("Git submodule update failed"), p.output)
927 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
928 return p.output.splitlines()
930 tag_format = re.compile(r'tag: ([^),]*)')
932 def latesttags(self):
934 p = FDroidPopen(['git', 'log', '--tags',
935 '--simplify-by-decoration', '--pretty=format:%d'],
936 cwd=self.local, output=False)
938 for line in p.output.splitlines():
939 for tag in self.tag_format.findall(line):
944 class vcs_gitsvn(vcs):
949 def clientversioncmd(self):
950 return ['git', 'svn', '--version']
953 """If the local directory exists, but is somehow not a git repository,
954 git will traverse up the directory tree until it finds one that
955 is (i.e. fdroidserver) and then we'll proceed to destory it!
956 This is called as a safety check.
959 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
960 result = p.output.rstrip()
961 if not result.endswith(self.local):
962 raise VCSException('Repository mismatch')
964 def git(self, args, envs=dict(), cwd=None, output=True):
965 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
967 AskPass is set to /bin/true to let the process try to connect
968 without a username/password.
970 The SSH command is set to /bin/false to block all SSH URLs
971 (supported in git >= 2.3). This protects against
976 '-c', 'core.askpass=/bin/true',
977 '-c', 'core.sshCommand=/bin/false',
980 'GIT_TERMINAL_PROMPT': '0',
981 'GIT_ASKPASS': '/bin/true',
982 'SSH_ASKPASS': '/bin/true',
983 'GIT_SSH': '/bin/false', # for git < 2.3
984 'SVN_SSH': '/bin/false',
986 return FDroidPopen(['git', ] + git_config + args,
987 envs=envs, cwd=cwd, output=output)
989 def gotorevisionx(self, rev):
990 if not os.path.exists(self.local):
992 gitsvn_args = ['svn', 'clone']
994 if ';' in self.remote:
995 remote_split = self.remote.split(';')
996 for i in remote_split[1:]:
997 if i.startswith('trunk='):
998 gitsvn_args.extend(['-T', i[6:]])
999 elif i.startswith('tags='):
1000 gitsvn_args.extend(['-t', i[5:]])
1001 elif i.startswith('branches='):
1002 gitsvn_args.extend(['-b', i[9:]])
1003 remote = remote_split[0]
1005 remote = self.remote
1007 if not remote.startswith('https://'):
1008 raise VCSException(_('HTTPS must be used with Subversion URLs!'))
1010 # git-svn sucks at certificate validation, this throws useful errors:
1012 r = requests.head(remote)
1013 r.raise_for_status()
1015 gitsvn_args.extend(['--', remote, self.local])
1016 p = self.git(gitsvn_args)
1017 if p.returncode != 0:
1018 self.clone_failed = True
1019 raise VCSException(_('git svn clone failed'), p.output)
1023 # Discard any working tree changes
1024 p = self.git(['reset', '--hard'], cwd=self.local, output=False)
1025 if p.returncode != 0:
1026 raise VCSException("Git reset failed", p.output)
1027 # Remove untracked files now, in case they're tracked in the target
1028 # revision (it happens!)
1029 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1030 if p.returncode != 0:
1031 raise VCSException("Git clean failed", p.output)
1032 if not self.refreshed:
1033 # Get new commits, branches and tags from repo
1034 p = self.git(['svn', 'fetch'], cwd=self.local, output=False)
1035 if p.returncode != 0:
1036 raise VCSException("Git svn fetch failed")
1037 p = self.git(['svn', 'rebase'], cwd=self.local, output=False)
1038 if p.returncode != 0:
1039 raise VCSException("Git svn rebase failed", p.output)
1040 self.refreshed = True
1042 rev = rev or 'master'
1044 nospaces_rev = rev.replace(' ', '%20')
1045 # Try finding a svn tag
1046 for treeish in ['origin/', '']:
1047 p = self.git(['checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
1048 if p.returncode == 0:
1050 if p.returncode != 0:
1051 # No tag found, normal svn rev translation
1052 # Translate svn rev into git format
1053 rev_split = rev.split('/')
1056 for treeish in ['origin/', '']:
1057 if len(rev_split) > 1:
1058 treeish += rev_split[0]
1059 svn_rev = rev_split[1]
1062 # if no branch is specified, then assume trunk (i.e. 'master' branch):
1066 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
1068 p = self.git(['svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
1069 git_rev = p.output.rstrip()
1071 if p.returncode == 0 and git_rev:
1074 if p.returncode != 0 or not git_rev:
1075 # Try a plain git checkout as a last resort
1076 p = self.git(['checkout', rev], cwd=self.local, output=False)
1077 if p.returncode != 0:
1078 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
1080 # Check out the git rev equivalent to the svn rev
1081 p = self.git(['checkout', git_rev], cwd=self.local, output=False)
1082 if p.returncode != 0:
1083 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
1085 # Get rid of any uncontrolled files left behind
1086 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1087 if p.returncode != 0:
1088 raise VCSException(_("Git clean failed"), p.output)
1092 for treeish in ['origin/', '']:
1093 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
1094 if os.path.isdir(d):
1095 return os.listdir(d)
1099 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
1100 if p.returncode != 0:
1102 return p.output.strip()
1110 def clientversioncmd(self):
1111 return ['hg', '--version']
1113 def gotorevisionx(self, rev):
1114 if not os.path.exists(self.local):
1115 p = FDroidPopen(['hg', 'clone', '--ssh', 'false', '--', self.remote, self.local],
1117 if p.returncode != 0:
1118 self.clone_failed = True
1119 raise VCSException("Hg clone failed", p.output)
1121 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
1122 if p.returncode != 0:
1123 raise VCSException("Hg status failed", p.output)
1124 for line in p.output.splitlines():
1125 if not line.startswith('? '):
1126 raise VCSException("Unexpected output from hg status -uS: " + line)
1127 FDroidPopen(['rm', '-rf', '--', line[2:]], cwd=self.local, output=False)
1128 if not self.refreshed:
1129 p = FDroidPopen(['hg', 'pull', '--ssh', 'false'], cwd=self.local, output=False)
1130 if p.returncode != 0:
1131 raise VCSException("Hg pull failed", p.output)
1132 self.refreshed = True
1134 rev = rev or 'default'
1137 p = FDroidPopen(['hg', 'update', '-C', '--', rev], cwd=self.local, output=False)
1138 if p.returncode != 0:
1139 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
1140 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1141 # Also delete untracked files, we have to enable purge extension for that:
1142 if "'purge' is provided by the following extension" in p.output:
1143 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
1144 myfile.write("\n[extensions]\nhgext.purge=\n")
1145 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1146 if p.returncode != 0:
1147 raise VCSException("HG purge failed", p.output)
1148 elif p.returncode != 0:
1149 raise VCSException("HG purge failed", p.output)
1152 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
1153 return p.output.splitlines()[1:]
1161 def clientversioncmd(self):
1162 return ['bzr', '--version']
1164 def bzr(self, args, envs=dict(), cwd=None, output=True):
1165 '''Prevent bzr from ever using SSH to avoid security vulns'''
1169 return FDroidPopen(['bzr', ] + args, envs=envs, cwd=cwd, output=output)
1171 def gotorevisionx(self, rev):
1172 if not os.path.exists(self.local):
1173 p = self.bzr(['branch', self.remote, self.local], output=False)
1174 if p.returncode != 0:
1175 self.clone_failed = True
1176 raise VCSException("Bzr branch failed", p.output)
1178 p = self.bzr(['clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
1179 if p.returncode != 0:
1180 raise VCSException("Bzr revert failed", p.output)
1181 if not self.refreshed:
1182 p = self.bzr(['pull'], cwd=self.local, output=False)
1183 if p.returncode != 0:
1184 raise VCSException("Bzr update failed", p.output)
1185 self.refreshed = True
1187 revargs = list(['-r', rev] if rev else [])
1188 p = self.bzr(['revert'] + revargs, cwd=self.local, output=False)
1189 if p.returncode != 0:
1190 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1193 p = self.bzr(['tags'], cwd=self.local, output=False)
1194 return [tag.split(' ')[0].strip() for tag in
1195 p.output.splitlines()]
1198 def unescape_string(string):
1201 if string[0] == '"' and string[-1] == '"':
1204 return string.replace("\\'", "'")
1207 def retrieve_string(app_dir, string, xmlfiles=None):
1209 if not string.startswith('@string/'):
1210 return unescape_string(string)
1212 if xmlfiles is None:
1215 os.path.join(app_dir, 'res'),
1216 os.path.join(app_dir, 'src', 'main', 'res'),
1218 for root, dirs, files in os.walk(res_dir):
1219 if os.path.basename(root) == 'values':
1220 xmlfiles += [os.path.join(root, x) for x in files if x.endswith('.xml')]
1222 name = string[len('@string/'):]
1224 def element_content(element):
1225 if element.text is None:
1227 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1228 return s.decode('utf-8').strip()
1230 for path in xmlfiles:
1231 if not os.path.isfile(path):
1233 xml = parse_xml(path)
1234 element = xml.find('string[@name="' + name + '"]')
1235 if element is not None:
1236 content = element_content(element)
1237 return retrieve_string(app_dir, content, xmlfiles)
1242 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1243 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1246 def manifest_paths(app_dir, flavours):
1247 '''Return list of existing files that will be used to find the highest vercode'''
1249 possible_manifests = \
1250 [os.path.join(app_dir, 'AndroidManifest.xml'),
1251 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1252 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1253 os.path.join(app_dir, 'build.gradle')]
1255 for flavour in flavours:
1256 if flavour == 'yes':
1258 possible_manifests.append(
1259 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1261 return [path for path in possible_manifests if os.path.isfile(path)]
1264 def fetch_real_name(app_dir, flavours):
1265 '''Retrieve the package name. Returns the name, or None if not found.'''
1266 for path in manifest_paths(app_dir, flavours):
1267 if not has_extension(path, 'xml') or not os.path.isfile(path):
1269 logging.debug("fetch_real_name: Checking manifest at " + path)
1270 xml = parse_xml(path)
1271 app = xml.find('application')
1274 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1276 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1277 result = retrieve_string_singleline(app_dir, label)
1279 result = result.strip()
1284 def get_library_references(root_dir):
1286 proppath = os.path.join(root_dir, 'project.properties')
1287 if not os.path.isfile(proppath):
1289 with open(proppath, 'r', encoding='iso-8859-1') as f:
1291 if not line.startswith('android.library.reference.'):
1293 path = line.split('=')[1].strip()
1294 relpath = os.path.join(root_dir, path)
1295 if not os.path.isdir(relpath):
1297 logging.debug("Found subproject at %s" % path)
1298 libraries.append(path)
1302 def ant_subprojects(root_dir):
1303 subprojects = get_library_references(root_dir)
1304 for subpath in subprojects:
1305 subrelpath = os.path.join(root_dir, subpath)
1306 for p in get_library_references(subrelpath):
1307 relp = os.path.normpath(os.path.join(subpath, p))
1308 if relp not in subprojects:
1309 subprojects.insert(0, relp)
1313 def remove_debuggable_flags(root_dir):
1314 # Remove forced debuggable flags
1315 logging.debug("Removing debuggable flags from %s" % root_dir)
1316 for root, dirs, files in os.walk(root_dir):
1317 if 'AndroidManifest.xml' in files and os.path.isfile(os.path.join(root, 'AndroidManifest.xml')):
1318 regsub_file(r'android:debuggable="[^"]*"',
1320 os.path.join(root, 'AndroidManifest.xml'))
1323 vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search
1324 vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1325 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1328 def app_matches_packagename(app, package):
1331 appid = app.UpdateCheckName or app.id
1332 if appid is None or appid == "Ignore":
1334 return appid == package
1337 def parse_androidmanifests(paths, app):
1339 Extract some information from the AndroidManifest.xml at the given path.
1340 Returns (version, vercode, package), any or all of which might be None.
1341 All values returned are strings.
1344 ignoreversions = app.UpdateCheckIgnore
1345 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1348 return (None, None, None)
1356 if not os.path.isfile(path):
1359 logging.debug(_("Parsing manifest at '{path}'").format(path=path))
1365 if app.builds and 'gradle' in app.builds[-1] and app.builds[-1].gradle:
1366 flavour = app.builds[-1].gradle[-1]
1368 if has_extension(path, 'gradle'):
1369 with open(path, 'r') as f:
1370 inside_flavour_group = 0
1371 inside_required_flavour = 0
1373 if gradle_comment.match(line):
1376 if inside_flavour_group > 0:
1377 if inside_required_flavour > 0:
1378 matches = psearch_g(line)
1380 s = matches.group(2)
1381 if app_matches_packagename(app, s):
1384 matches = vnsearch_g(line)
1386 version = matches.group(2)
1388 matches = vcsearch_g(line)
1390 vercode = matches.group(1)
1393 inside_required_flavour += 1
1395 inside_required_flavour -= 1
1397 if flavour and (flavour in line):
1398 inside_required_flavour = 1
1401 inside_flavour_group += 1
1403 inside_flavour_group -= 1
1405 if "productFlavors" in line:
1406 inside_flavour_group = 1
1408 matches = psearch_g(line)
1410 s = matches.group(2)
1411 if app_matches_packagename(app, s):
1414 matches = vnsearch_g(line)
1416 version = matches.group(2)
1418 matches = vcsearch_g(line)
1420 vercode = matches.group(1)
1423 xml = parse_xml(path)
1424 if "package" in xml.attrib:
1425 s = xml.attrib["package"]
1426 if app_matches_packagename(app, s):
1428 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1429 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1430 base_dir = os.path.dirname(path)
1431 version = retrieve_string_singleline(base_dir, version)
1432 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1433 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1434 if string_is_integer(a):
1437 logging.warning(_("Problem with xml at '{path}'").format(path=path))
1439 # Remember package name, may be defined separately from version+vercode
1441 package = max_package
1443 logging.debug("..got package={0}, version={1}, vercode={2}"
1444 .format(package, version, vercode))
1446 # Always grab the package name and version name in case they are not
1447 # together with the highest version code
1448 if max_package is None and package is not None:
1449 max_package = package
1450 if max_version is None and version is not None:
1451 max_version = version
1453 if vercode is not None \
1454 and (max_vercode is None or vercode > max_vercode):
1455 if not ignoresearch or not ignoresearch(version):
1456 if version is not None:
1457 max_version = version
1458 if vercode is not None:
1459 max_vercode = vercode
1460 if package is not None:
1461 max_package = package
1463 max_version = "Ignore"
1465 if max_version is None:
1466 max_version = "Unknown"
1468 if max_package and not is_valid_package_name(max_package):
1469 raise FDroidException(_("Invalid package name {0}").format(max_package))
1471 return (max_version, max_vercode, max_package)
1474 def is_valid_package_name(name):
1475 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1478 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1479 raw=False, prepare=True, preponly=False, refresh=True,
1481 """Get the specified source library.
1483 Returns the path to it. Normally this is the path to be used when
1484 referencing it, which may be a subdirectory of the actual project. If
1485 you want the base directory of the project, pass 'basepath=True'.
1494 name, ref = spec.split('@')
1496 number, name = name.split(':', 1)
1498 name, subdir = name.split('/', 1)
1500 if name not in fdroidserver.metadata.srclibs:
1501 raise VCSException('srclib ' + name + ' not found.')
1503 srclib = fdroidserver.metadata.srclibs[name]
1505 sdir = os.path.join(srclib_dir, name)
1508 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1509 vcs.srclib = (name, number, sdir)
1511 vcs.gotorevision(ref, refresh)
1518 libdir = os.path.join(sdir, subdir)
1519 elif srclib["Subdir"]:
1520 for subdir in srclib["Subdir"]:
1521 libdir_candidate = os.path.join(sdir, subdir)
1522 if os.path.exists(libdir_candidate):
1523 libdir = libdir_candidate
1529 remove_signing_keys(sdir)
1530 remove_debuggable_flags(sdir)
1534 if srclib["Prepare"]:
1535 cmd = replace_config_vars(srclib["Prepare"], build)
1537 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=libdir)
1538 if p.returncode != 0:
1539 raise BuildException("Error running prepare command for srclib %s"
1545 return (name, number, libdir)
1548 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1551 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1552 """ Prepare the source code for a particular build
1554 :param vcs: the appropriate vcs object for the application
1555 :param app: the application details from the metadata
1556 :param build: the build details from the metadata
1557 :param build_dir: the path to the build directory, usually 'build/app.id'
1558 :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
1559 :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
1561 Returns the (root, srclibpaths) where:
1562 :param root: is the root directory, which may be the same as 'build_dir' or may
1563 be a subdirectory of it.
1564 :param srclibpaths: is information on the srclibs being used
1567 # Optionally, the actual app source can be in a subdirectory
1569 root_dir = os.path.join(build_dir, build.subdir)
1571 root_dir = build_dir
1573 # Get a working copy of the right revision
1574 logging.info("Getting source for revision " + build.commit)
1575 vcs.gotorevision(build.commit, refresh)
1577 # Initialise submodules if required
1578 if build.submodules:
1579 logging.info(_("Initialising submodules"))
1580 vcs.initsubmodules()
1582 # Check that a subdir (if we're using one) exists. This has to happen
1583 # after the checkout, since it might not exist elsewhere
1584 if not os.path.exists(root_dir):
1585 raise BuildException('Missing subdir ' + root_dir)
1587 # Run an init command if one is required
1589 cmd = replace_config_vars(build.init, build)
1590 logging.info("Running 'init' commands in %s" % root_dir)
1592 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir)
1593 if p.returncode != 0:
1594 raise BuildException("Error running init command for %s:%s" %
1595 (app.id, build.versionName), p.output)
1597 # Apply patches if any
1599 logging.info("Applying patches")
1600 for patch in build.patch:
1601 patch = patch.strip()
1602 logging.info("Applying " + patch)
1603 patch_path = os.path.join('metadata', app.id, patch)
1604 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1605 if p.returncode != 0:
1606 raise BuildException("Failed to apply patch %s" % patch_path)
1608 # Get required source libraries
1611 logging.info("Collecting source libraries")
1612 for lib in build.srclibs:
1613 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1614 refresh=refresh, build=build))
1616 for name, number, libpath in srclibpaths:
1617 place_srclib(root_dir, int(number) if number else None, libpath)
1619 basesrclib = vcs.getsrclib()
1620 # If one was used for the main source, add that too.
1622 srclibpaths.append(basesrclib)
1624 # Update the local.properties file
1625 localprops = [os.path.join(build_dir, 'local.properties')]
1627 parts = build.subdir.split(os.sep)
1630 cur = os.path.join(cur, d)
1631 localprops += [os.path.join(cur, 'local.properties')]
1632 for path in localprops:
1634 if os.path.isfile(path):
1635 logging.info("Updating local.properties file at %s" % path)
1636 with open(path, 'r', encoding='iso-8859-1') as f:
1640 logging.info("Creating local.properties file at %s" % path)
1641 # Fix old-fashioned 'sdk-location' by copying
1642 # from sdk.dir, if necessary
1644 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1645 re.S | re.M).group(1)
1646 props += "sdk-location=%s\n" % sdkloc
1648 props += "sdk.dir=%s\n" % config['sdk_path']
1649 props += "sdk-location=%s\n" % config['sdk_path']
1650 ndk_path = build.ndk_path()
1651 # if for any reason the path isn't valid or the directory
1652 # doesn't exist, some versions of Gradle will error with a
1653 # cryptic message (even if the NDK is not even necessary).
1654 # https://gitlab.com/fdroid/fdroidserver/issues/171
1655 if ndk_path and os.path.exists(ndk_path):
1657 props += "ndk.dir=%s\n" % ndk_path
1658 props += "ndk-location=%s\n" % ndk_path
1659 # Add java.encoding if necessary
1661 props += "java.encoding=%s\n" % build.encoding
1662 with open(path, 'w', encoding='iso-8859-1') as f:
1666 if build.build_method() == 'gradle':
1667 flavours = build.gradle
1670 n = build.target.split('-')[1]
1671 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1672 r'compileSdkVersion %s' % n,
1673 os.path.join(root_dir, 'build.gradle'))
1675 # Remove forced debuggable flags
1676 remove_debuggable_flags(root_dir)
1678 # Insert version code and number into the manifest if necessary
1679 if build.forceversion:
1680 logging.info("Changing the version name")
1681 for path in manifest_paths(root_dir, flavours):
1682 if not os.path.isfile(path):
1684 if has_extension(path, 'xml'):
1685 regsub_file(r'android:versionName="[^"]*"',
1686 r'android:versionName="%s"' % build.versionName,
1688 elif has_extension(path, 'gradle'):
1689 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1690 r"""\1versionName '%s'""" % build.versionName,
1693 if build.forcevercode:
1694 logging.info("Changing the version code")
1695 for path in manifest_paths(root_dir, flavours):
1696 if not os.path.isfile(path):
1698 if has_extension(path, 'xml'):
1699 regsub_file(r'android:versionCode="[^"]*"',
1700 r'android:versionCode="%s"' % build.versionCode,
1702 elif has_extension(path, 'gradle'):
1703 regsub_file(r'versionCode[ =]+[0-9]+',
1704 r'versionCode %s' % build.versionCode,
1707 # Delete unwanted files
1709 logging.info(_("Removing specified files"))
1710 for part in getpaths(build_dir, build.rm):
1711 dest = os.path.join(build_dir, part)
1712 logging.info("Removing {0}".format(part))
1713 if os.path.lexists(dest):
1714 # rmtree can only handle directories that are not symlinks, so catch anything else
1715 if not os.path.isdir(dest) or os.path.islink(dest):
1720 logging.info("...but it didn't exist")
1722 remove_signing_keys(build_dir)
1724 # Add required external libraries
1726 logging.info("Collecting prebuilt libraries")
1727 libsdir = os.path.join(root_dir, 'libs')
1728 if not os.path.exists(libsdir):
1730 for lib in build.extlibs:
1732 logging.info("...installing extlib {0}".format(lib))
1733 libf = os.path.basename(lib)
1734 libsrc = os.path.join(extlib_dir, lib)
1735 if not os.path.exists(libsrc):
1736 raise BuildException("Missing extlib file {0}".format(libsrc))
1737 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1739 # Run a pre-build command if one is required
1741 logging.info("Running 'prebuild' commands in %s" % root_dir)
1743 cmd = replace_config_vars(build.prebuild, build)
1745 # Substitute source library paths into prebuild commands
1746 for name, number, libpath in srclibpaths:
1747 libpath = os.path.relpath(libpath, root_dir)
1748 cmd = cmd.replace('$$' + name + '$$', libpath)
1750 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir)
1751 if p.returncode != 0:
1752 raise BuildException("Error running prebuild command for %s:%s" %
1753 (app.id, build.versionName), p.output)
1755 # Generate (or update) the ant build file, build.xml...
1756 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1757 parms = ['android', 'update', 'lib-project']
1758 lparms = ['android', 'update', 'project']
1761 parms += ['-t', build.target]
1762 lparms += ['-t', build.target]
1763 if build.androidupdate:
1764 update_dirs = build.androidupdate
1766 update_dirs = ant_subprojects(root_dir) + ['.']
1768 for d in update_dirs:
1769 subdir = os.path.join(root_dir, d)
1771 logging.debug("Updating main project")
1772 cmd = parms + ['-p', d]
1774 logging.debug("Updating subproject %s" % d)
1775 cmd = lparms + ['-p', d]
1776 p = SdkToolsPopen(cmd, cwd=root_dir)
1777 # Check to see whether an error was returned without a proper exit
1778 # code (this is the case for the 'no target set or target invalid'
1780 if p.returncode != 0 or p.output.startswith("Error: "):
1781 raise BuildException("Failed to update project at %s" % d, p.output)
1782 # Clean update dirs via ant
1784 logging.info("Cleaning subproject %s" % d)
1785 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1787 return (root_dir, srclibpaths)
1790 def getpaths_map(build_dir, globpaths):
1791 """Extend via globbing the paths from a field and return them as a map from original path to resulting paths"""
1795 full_path = os.path.join(build_dir, p)
1796 full_path = os.path.normpath(full_path)
1797 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1799 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1803 def getpaths(build_dir, globpaths):
1804 """Extend via globbing the paths from a field and return them as a set"""
1805 paths_map = getpaths_map(build_dir, globpaths)
1807 for k, v in paths_map.items():
1814 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1817 def check_system_clock(dt_obj, path):
1818 """Check if system clock is updated based on provided date
1820 If an APK has files newer than the system time, suggest updating
1821 the system clock. This is useful for offline systems, used for
1822 signing, which do not have another source of clock sync info. It
1823 has to be more than 24 hours newer because ZIP/APK files do not
1827 checkdt = dt_obj - timedelta(1)
1828 if datetime.today() < checkdt:
1829 logging.warning(_('System clock is older than date in {path}!').format(path=path)
1830 + '\n' + _('Set clock to that time using:') + '\n'
1831 + 'sudo date -s "' + str(dt_obj) + '"')
1835 """permanent store of existing APKs with the date they were added
1837 This is currently the only way to permanently store the "updated"
1842 '''Load filename/date info about previously seen APKs
1844 Since the appid and date strings both will never have spaces,
1845 this is parsed as a list from the end to allow the filename to
1846 have any combo of spaces.
1849 self.path = os.path.join('stats', 'known_apks.txt')
1851 if os.path.isfile(self.path):
1852 with open(self.path, 'r', encoding='utf8') as f:
1854 t = line.rstrip().split(' ')
1856 self.apks[t[0]] = (t[1], None)
1859 date = datetime.strptime(t[-1], '%Y-%m-%d')
1860 filename = line[0:line.rfind(appid) - 1]
1861 self.apks[filename] = (appid, date)
1862 check_system_clock(date, self.path)
1863 self.changed = False
1865 def writeifchanged(self):
1866 if not self.changed:
1869 if not os.path.exists('stats'):
1873 for apk, app in self.apks.items():
1875 line = apk + ' ' + appid
1877 line += ' ' + added.strftime('%Y-%m-%d')
1880 with open(self.path, 'w', encoding='utf8') as f:
1881 for line in sorted(lst, key=natural_key):
1882 f.write(line + '\n')
1884 def recordapk(self, apkName, app, default_date=None):
1886 Record an apk (if it's new, otherwise does nothing)
1887 Returns the date it was added as a datetime instance
1889 if apkName not in self.apks:
1890 if default_date is None:
1891 default_date = datetime.utcnow()
1892 self.apks[apkName] = (app, default_date)
1894 _ignored, added = self.apks[apkName]
1897 def getapp(self, apkname):
1898 """Look up information - given the 'apkname', returns (app id, date added/None).
1900 Or returns None for an unknown apk.
1902 if apkname in self.apks:
1903 return self.apks[apkname]
1906 def getlatest(self, num):
1907 """Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first"""
1909 for apk, app in self.apks.items():
1913 if apps[appid] > added:
1917 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1918 lst = [app for app, _ignored in sortedapps]
1923 def get_file_extension(filename):
1924 """get the normalized file extension, can be blank string but never None"""
1925 if isinstance(filename, bytes):
1926 filename = filename.decode('utf-8')
1927 return os.path.splitext(filename)[1].lower()[1:]
1930 def use_androguard():
1931 """Report if androguard is available, and config its debug logging"""
1935 if use_androguard.show_path:
1936 logging.debug(_('Using androguard from "{path}"').format(path=androguard.__file__))
1937 use_androguard.show_path = False
1938 if options and options.verbose:
1939 logging.getLogger("androguard.axml").setLevel(logging.INFO)
1945 use_androguard.show_path = True
1948 def is_apk_and_debuggable_aapt(apkfile):
1949 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1951 if p.returncode != 0:
1952 raise FDroidException(_("Failed to get APK manifest information"))
1953 for line in p.output.splitlines():
1954 if 'android:debuggable' in line and not line.endswith('0x0'):
1959 def is_apk_and_debuggable_androguard(apkfile):
1961 from androguard.core.bytecodes.apk import APK
1963 raise FDroidException("androguard library is not installed and aapt not present")
1965 apkobject = APK(apkfile)
1966 if apkobject.is_valid_APK():
1967 debuggable = apkobject.get_element("application", "debuggable")
1968 if debuggable is not None:
1969 return bool(strtobool(debuggable))
1973 def is_apk_and_debuggable(apkfile):
1974 """Returns True if the given file is an APK and is debuggable
1976 :param apkfile: full path to the apk to check"""
1978 if get_file_extension(apkfile) != 'apk':
1981 if use_androguard():
1982 return is_apk_and_debuggable_androguard(apkfile)
1984 return is_apk_and_debuggable_aapt(apkfile)
1987 def get_apk_id_aapt(apkfile):
1988 """Extrat identification information from APK using aapt.
1990 :param apkfile: path to an APK file.
1991 :returns: triplet (appid, version code, version name)
1993 r = re.compile("package: name='(?P<appid>.*)' versionCode='(?P<vercode>.*)' versionName='(?P<vername>.*)' platformBuildVersionName='.*'")
1994 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1995 for line in p.output.splitlines():
1998 return m.group('appid'), m.group('vercode'), m.group('vername')
1999 raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'")
2000 .format(apkfilename=apkfile))
2003 def get_minSdkVersion_aapt(apkfile):
2004 """Extract the minimum supported Android SDK from an APK using aapt
2006 :param apkfile: path to an APK file.
2007 :returns: the integer representing the SDK version
2009 r = re.compile(r"^sdkVersion:'([0-9]+)'")
2010 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
2011 for line in p.output.splitlines():
2014 return int(m.group(1))
2015 raise FDroidException(_('Reading minSdkVersion failed: "{apkfilename}"')
2016 .format(apkfilename=apkfile))
2021 self.returncode = None
2025 def SdkToolsPopen(commands, cwd=None, output=True):
2027 if cmd not in config:
2028 config[cmd] = find_sdk_tools_cmd(commands[0])
2029 abscmd = config[cmd]
2031 raise FDroidException(_("Could not find '{command}' on your system").format(command=cmd))
2033 test_aapt_version(config['aapt'])
2034 return FDroidPopen([abscmd] + commands[1:],
2035 cwd=cwd, output=output)
2038 def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2040 Run a command and capture the possibly huge output as bytes.
2042 :param commands: command and argument list like in subprocess.Popen
2043 :param cwd: optionally specifies a working directory
2044 :param envs: a optional dictionary of environment variables and their values
2045 :returns: A PopenResult.
2050 set_FDroidPopen_env()
2052 process_env = env.copy()
2053 if envs is not None and len(envs) > 0:
2054 process_env.update(envs)
2057 cwd = os.path.normpath(cwd)
2058 logging.debug("Directory: %s" % cwd)
2059 logging.debug("> %s" % ' '.join(commands))
2061 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
2062 result = PopenResult()
2065 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
2066 stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
2067 stderr=stderr_param)
2068 except OSError as e:
2069 raise BuildException("OSError while trying to execute " +
2070 ' '.join(commands) + ': ' + str(e))
2072 # TODO are these AsynchronousFileReader threads always exiting?
2073 if not stderr_to_stdout and options.verbose:
2074 stderr_queue = Queue()
2075 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
2077 while not stderr_reader.eof():
2078 while not stderr_queue.empty():
2079 line = stderr_queue.get()
2080 sys.stderr.buffer.write(line)
2085 stdout_queue = Queue()
2086 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
2089 # Check the queue for output (until there is no more to get)
2090 while not stdout_reader.eof():
2091 while not stdout_queue.empty():
2092 line = stdout_queue.get()
2093 if output and options.verbose:
2094 # Output directly to console
2095 sys.stderr.buffer.write(line)
2101 result.returncode = p.wait()
2102 result.output = buf.getvalue()
2104 # make sure all filestreams of the subprocess are closed
2105 for streamvar in ['stdin', 'stdout', 'stderr']:
2106 if hasattr(p, streamvar):
2107 stream = getattr(p, streamvar)
2113 def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2115 Run a command and capture the possibly huge output as a str.
2117 :param commands: command and argument list like in subprocess.Popen
2118 :param cwd: optionally specifies a working directory
2119 :param envs: a optional dictionary of environment variables and their values
2120 :returns: A PopenResult.
2122 result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
2123 result.output = result.output.decode('utf-8', 'ignore')
2127 gradle_comment = re.compile(r'[ ]*//')
2128 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
2129 gradle_line_matches = [
2130 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
2131 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
2132 re.compile(r'.*\.readLine\(.*'),
2136 def remove_signing_keys(build_dir):
2137 for root, dirs, files in os.walk(build_dir):
2138 if 'build.gradle' in files:
2139 path = os.path.join(root, 'build.gradle')
2141 with open(path, "r", encoding='utf8') as o:
2142 lines = o.readlines()
2148 with open(path, "w", encoding='utf8') as o:
2149 while i < len(lines):
2152 while line.endswith('\\\n'):
2153 line = line.rstrip('\\\n') + lines[i]
2156 if gradle_comment.match(line):
2161 opened += line.count('{')
2162 opened -= line.count('}')
2165 if gradle_signing_configs.match(line):
2170 if any(s.match(line) for s in gradle_line_matches):
2178 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
2181 'project.properties',
2183 'default.properties',
2184 'ant.properties', ]:
2185 if propfile in files:
2186 path = os.path.join(root, propfile)
2188 with open(path, "r", encoding='iso-8859-1') as o:
2189 lines = o.readlines()
2193 with open(path, "w", encoding='iso-8859-1') as o:
2195 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
2202 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
2205 def set_FDroidPopen_env(build=None):
2207 set up the environment variables for the build environment
2209 There is only a weak standard, the variables used by gradle, so also set
2210 up the most commonly used environment variables for SDK and NDK. Also, if
2211 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
2213 global env, orig_path
2217 orig_path = env['PATH']
2218 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
2219 env[n] = config['sdk_path']
2220 for k, v in config['java_paths'].items():
2221 env['JAVA%s_HOME' % k] = v
2223 missinglocale = True
2224 for k, v in env.items():
2225 if k == 'LANG' and v != 'C':
2226 missinglocale = False
2228 missinglocale = False
2230 env['LANG'] = 'en_US.UTF-8'
2232 if build is not None:
2233 path = build.ndk_path()
2234 paths = orig_path.split(os.pathsep)
2235 if path not in paths:
2236 paths = [path] + paths
2237 env['PATH'] = os.pathsep.join(paths)
2238 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
2239 env[n] = build.ndk_path()
2242 def replace_build_vars(cmd, build):
2243 cmd = cmd.replace('$$COMMIT$$', build.commit)
2244 cmd = cmd.replace('$$VERSION$$', build.versionName)
2245 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
2249 def replace_config_vars(cmd, build):
2250 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
2251 cmd = cmd.replace('$$NDK$$', build.ndk_path())
2252 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
2253 if build is not None:
2254 cmd = replace_build_vars(cmd, build)
2258 def place_srclib(root_dir, number, libpath):
2261 relpath = os.path.relpath(libpath, root_dir)
2262 proppath = os.path.join(root_dir, 'project.properties')
2265 if os.path.isfile(proppath):
2266 with open(proppath, "r", encoding='iso-8859-1') as o:
2267 lines = o.readlines()
2269 with open(proppath, "w", encoding='iso-8859-1') as o:
2272 if line.startswith('android.library.reference.%d=' % number):
2273 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2278 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2281 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
2284 def signer_fingerprint_short(sig):
2285 """Obtain shortened sha256 signing-key fingerprint for pkcs7 signature.
2287 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2288 for a given pkcs7 signature.
2290 :param sig: Contents of an APK signing certificate.
2291 :returns: shortened signing-key fingerprint.
2293 return signer_fingerprint(sig)[:7]
2296 def signer_fingerprint(sig):
2297 """Obtain sha256 signing-key fingerprint for pkcs7 signature.
2299 Extracts hexadecimal sha256 signing-key fingerprint string
2300 for a given pkcs7 signature.
2302 :param: Contents of an APK signature.
2303 :returns: shortened signature fingerprint.
2305 cert_encoded = get_certificate(sig)
2306 return hashlib.sha256(cert_encoded).hexdigest()
2309 def apk_signer_fingerprint(apk_path):
2310 """Obtain sha256 signing-key fingerprint for APK.
2312 Extracts hexadecimal sha256 signing-key fingerprint string
2315 :param apkpath: path to APK
2316 :returns: signature fingerprint
2319 with zipfile.ZipFile(apk_path, 'r') as apk:
2320 certs = [n for n in apk.namelist() if CERT_PATH_REGEX.match(n)]
2323 logging.error("Found no signing certificates on %s" % apk_path)
2326 logging.error("Found multiple signing certificates on %s" % apk_path)
2329 cert = apk.read(certs[0])
2330 return signer_fingerprint(cert)
2333 def apk_signer_fingerprint_short(apk_path):
2334 """Obtain shortened sha256 signing-key fingerprint for APK.
2336 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2337 for a given pkcs7 APK.
2339 :param apk_path: path to APK
2340 :returns: shortened signing-key fingerprint
2342 return apk_signer_fingerprint(apk_path)[:7]
2345 def metadata_get_sigdir(appid, vercode=None):
2346 """Get signature directory for app"""
2348 return os.path.join('metadata', appid, 'signatures', vercode)
2350 return os.path.join('metadata', appid, 'signatures')
2353 def metadata_find_developer_signature(appid, vercode=None):
2354 """Tires to find the developer signature for given appid.
2356 This picks the first signature file found in metadata an returns its
2359 :returns: sha256 signing key fingerprint of the developer signing key.
2360 None in case no signature can not be found."""
2362 # fetch list of dirs for all versions of signatures
2365 appversigdirs.append(metadata_get_sigdir(appid, vercode))
2367 appsigdir = metadata_get_sigdir(appid)
2368 if os.path.isdir(appsigdir):
2369 numre = re.compile('[0-9]+')
2370 for ver in os.listdir(appsigdir):
2371 if numre.match(ver):
2372 appversigdir = os.path.join(appsigdir, ver)
2373 appversigdirs.append(appversigdir)
2375 for sigdir in appversigdirs:
2376 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2377 glob.glob(os.path.join(sigdir, '*.EC')) + \
2378 glob.glob(os.path.join(sigdir, '*.RSA'))
2380 raise FDroidException('ambiguous signatures, please make sure there is only one signature in \'{}\'. (The signature has to be the App maintainers signature for version of the APK.)'.format(sigdir))
2382 with open(sig, 'rb') as f:
2383 return signer_fingerprint(f.read())
2387 def metadata_find_signing_files(appid, vercode):
2388 """Gets a list of singed manifests and signatures.
2390 :param appid: app id string
2391 :param vercode: app version code
2392 :returns: a list of triplets for each signing key with following paths:
2393 (signature_file, singed_file, manifest_file)
2396 sigdir = metadata_get_sigdir(appid, vercode)
2397 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2398 glob.glob(os.path.join(sigdir, '*.EC')) + \
2399 glob.glob(os.path.join(sigdir, '*.RSA'))
2400 extre = re.compile('(\.DSA|\.EC|\.RSA)$')
2402 sf = extre.sub('.SF', sig)
2403 if os.path.isfile(sf):
2404 mf = os.path.join(sigdir, 'MANIFEST.MF')
2405 if os.path.isfile(mf):
2406 ret.append((sig, sf, mf))
2410 def metadata_find_developer_signing_files(appid, vercode):
2411 """Get developer signature files for specified app from metadata.
2413 :returns: A triplet of paths for signing files from metadata:
2414 (signature_file, singed_file, manifest_file)
2416 allsigningfiles = metadata_find_signing_files(appid, vercode)
2417 if allsigningfiles and len(allsigningfiles) == 1:
2418 return allsigningfiles[0]
2423 def apk_strip_signatures(signed_apk, strip_manifest=False):
2424 """Removes signatures from APK.
2426 :param signed_apk: path to apk file.
2427 :param strip_manifest: when set to True also the manifest file will
2428 be removed from the APK.
2430 with tempfile.TemporaryDirectory() as tmpdir:
2431 tmp_apk = os.path.join(tmpdir, 'tmp.apk')
2432 shutil.move(signed_apk, tmp_apk)
2433 with ZipFile(tmp_apk, 'r') as in_apk:
2434 with ZipFile(signed_apk, 'w') as out_apk:
2435 for info in in_apk.infolist():
2436 if not apk_sigfile.match(info.filename):
2438 if info.filename != 'META-INF/MANIFEST.MF':
2439 buf = in_apk.read(info.filename)
2440 out_apk.writestr(info, buf)
2442 buf = in_apk.read(info.filename)
2443 out_apk.writestr(info, buf)
2446 def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest):
2447 """Implats a signature from metadata into an APK.
2449 Note: this changes there supplied APK in place. So copy it if you
2450 need the original to be preserved.
2452 :param apkpath: location of the apk
2454 # get list of available signature files in metadata
2455 with tempfile.TemporaryDirectory() as tmpdir:
2456 apkwithnewsig = os.path.join(tmpdir, 'newsig.apk')
2457 with ZipFile(apkpath, 'r') as in_apk:
2458 with ZipFile(apkwithnewsig, 'w') as out_apk:
2459 for sig_file in [signaturefile, signedfile, manifest]:
2460 with open(sig_file, 'rb') as fp:
2462 info = zipfile.ZipInfo('META-INF/' + os.path.basename(sig_file))
2463 info.compress_type = zipfile.ZIP_DEFLATED
2464 info.create_system = 0 # "Windows" aka "FAT", what Android SDK uses
2465 out_apk.writestr(info, buf)
2466 for info in in_apk.infolist():
2467 if not apk_sigfile.match(info.filename):
2468 if info.filename != 'META-INF/MANIFEST.MF':
2469 buf = in_apk.read(info.filename)
2470 out_apk.writestr(info, buf)
2472 p = SdkToolsPopen(['zipalign', '-v', '4', apkwithnewsig, apkpath])
2473 if p.returncode != 0:
2474 raise BuildException("Failed to align application")
2477 def apk_extract_signatures(apkpath, outdir, manifest=True):
2478 """Extracts a signature files from APK and puts them into target directory.
2480 :param apkpath: location of the apk
2481 :param outdir: folder where the extracted signature files will be stored
2482 :param manifest: (optionally) disable extracting manifest file
2484 with ZipFile(apkpath, 'r') as in_apk:
2485 for f in in_apk.infolist():
2486 if apk_sigfile.match(f.filename) or \
2487 (manifest and f.filename == 'META-INF/MANIFEST.MF'):
2488 newpath = os.path.join(outdir, os.path.basename(f.filename))
2489 with open(newpath, 'wb') as out_file:
2490 out_file.write(in_apk.read(f.filename))
2493 def sign_apk(unsigned_path, signed_path, keyalias):
2494 """Sign and zipalign an unsigned APK, then save to a new file, deleting the unsigned
2496 android-18 (4.3) finally added support for reasonable hash
2497 algorithms, like SHA-256, before then, the only options were MD5
2498 and SHA1 :-/ This aims to use SHA-256 when the APK does not target
2499 older Android versions, and is therefore safe to do so.
2501 https://issuetracker.google.com/issues/36956587
2502 https://android-review.googlesource.com/c/platform/libcore/+/44491
2506 if get_minSdkVersion_aapt(unsigned_path) < 18:
2507 signature_algorithm = ['-sigalg', 'SHA1withRSA', '-digestalg', 'SHA1']
2509 signature_algorithm = ['-sigalg', 'SHA256withRSA', '-digestalg', 'SHA-256']
2511 p = FDroidPopen([config['jarsigner'], '-keystore', config['keystore'],
2512 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2513 '-keypass:env', 'FDROID_KEY_PASS']
2514 + signature_algorithm + [unsigned_path, keyalias],
2516 'FDROID_KEY_STORE_PASS': config['keystorepass'],
2517 'FDROID_KEY_PASS': config['keypass'], })
2518 if p.returncode != 0:
2519 raise BuildException(_("Failed to sign application"), p.output)
2521 p = SdkToolsPopen(['zipalign', '-v', '4', unsigned_path, signed_path])
2522 if p.returncode != 0:
2523 raise BuildException(_("Failed to zipalign application"))
2524 os.remove(unsigned_path)
2527 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
2528 """Verify that two apks are the same
2530 One of the inputs is signed, the other is unsigned. The signature metadata
2531 is transferred from the signed to the unsigned apk, and then jarsigner is
2532 used to verify that the signature from the signed apk is also varlid for
2533 the unsigned one. If the APK given as unsigned actually does have a
2534 signature, it will be stripped out and ignored.
2536 There are two SHA1 git commit IDs that fdroidserver includes in the builds
2537 it makes: fdroidserverid and buildserverid. Originally, these were inserted
2538 into AndroidManifest.xml, but that makes the build not reproducible. So
2539 instead they are included as separate files in the APK's META-INF/ folder.
2540 If those files exist in the signed APK, they will be part of the signature
2541 and need to also be included in the unsigned APK for it to validate.
2543 :param signed_apk: Path to a signed apk file
2544 :param unsigned_apk: Path to an unsigned apk file expected to match it
2545 :param tmp_dir: Path to directory for temporary files
2546 :returns: None if the verification is successful, otherwise a string
2547 describing what went wrong.
2550 if not os.path.isfile(signed_apk):
2551 return 'can not verify: file does not exists: {}'.format(signed_apk)
2553 if not os.path.isfile(unsigned_apk):
2554 return 'can not verify: file does not exists: {}'.format(unsigned_apk)
2556 with ZipFile(signed_apk, 'r') as signed:
2557 meta_inf_files = ['META-INF/MANIFEST.MF']
2558 for f in signed.namelist():
2559 if apk_sigfile.match(f) \
2560 or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
2561 meta_inf_files.append(f)
2562 if len(meta_inf_files) < 3:
2563 return "Signature files missing from {0}".format(signed_apk)
2565 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2566 with ZipFile(unsigned_apk, 'r') as unsigned:
2567 # only read the signature from the signed APK, everything else from unsigned
2568 with ZipFile(tmp_apk, 'w') as tmp:
2569 for filename in meta_inf_files:
2570 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2571 for info in unsigned.infolist():
2572 if info.filename in meta_inf_files:
2573 logging.warning('Ignoring %s from %s',
2574 info.filename, unsigned_apk)
2576 if info.filename in tmp.namelist():
2577 return "duplicate filename found: " + info.filename
2578 tmp.writestr(info, unsigned.read(info.filename))
2580 verified = verify_apk_signature(tmp_apk)
2583 logging.info("...NOT verified - {0}".format(tmp_apk))
2584 return compare_apks(signed_apk, tmp_apk, tmp_dir,
2585 os.path.dirname(unsigned_apk))
2587 logging.info("...successfully verified")
2591 def verify_jar_signature(jar):
2592 """Verifies the signature of a given JAR file.
2594 jarsigner is very shitty: unsigned JARs pass as "verified"! So
2595 this has to turn on -strict then check for result 4, since this
2596 does not expect the signature to be from a CA-signed certificate.
2598 :raises: VerificationException() if the JAR's signature could not be verified
2602 error = _('JAR signature failed to verify: {path}').format(path=jar)
2604 output = subprocess.check_output([config['jarsigner'], '-strict', '-verify', jar],
2605 stderr=subprocess.STDOUT)
2606 raise VerificationException(error + '\n' + output.decode('utf-8'))
2607 except subprocess.CalledProcessError as e:
2608 if e.returncode == 4:
2609 logging.debug(_('JAR signature verified: {path}').format(path=jar))
2611 raise VerificationException(error + '\n' + e.output.decode('utf-8'))
2614 def verify_apk_signature(apk, min_sdk_version=None):
2615 """verify the signature on an APK
2617 Try to use apksigner whenever possible since jarsigner is very
2618 shitty: unsigned APKs pass as "verified"! Warning, this does
2619 not work on JARs with apksigner >= 0.7 (build-tools 26.0.1)
2621 :returns: boolean whether the APK was verified
2623 if set_command_in_config('apksigner'):
2624 args = [config['apksigner'], 'verify']
2626 args += ['--min-sdk-version=' + min_sdk_version]
2628 args += ['--verbose']
2630 output = subprocess.check_output(args + [apk])
2632 logging.debug(apk + ': ' + output.decode('utf-8'))
2634 except subprocess.CalledProcessError as e:
2635 logging.error('\n' + apk + ': ' + e.output.decode('utf-8'))
2637 if not config.get('jarsigner_warning_displayed'):
2638 config['jarsigner_warning_displayed'] = True
2639 logging.warning(_("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner"))
2641 verify_jar_signature(apk)
2643 except Exception as e:
2648 def verify_old_apk_signature(apk):
2649 """verify the signature on an archived APK, supporting deprecated algorithms
2651 F-Droid aims to keep every single binary that it ever published. Therefore,
2652 it needs to be able to verify APK signatures that include deprecated/removed
2653 algorithms. For example, jarsigner treats an MD5 signature as unsigned.
2655 jarsigner passes unsigned APKs as "verified"! So this has to turn
2656 on -strict then check for result 4.
2658 :returns: boolean whether the APK was verified
2661 _java_security = os.path.join(os.getcwd(), '.java.security')
2662 with open(_java_security, 'w') as fp:
2663 fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
2667 config['jarsigner'],
2668 '-J-Djava.security.properties=' + _java_security,
2669 '-strict', '-verify', apk
2671 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
2672 except subprocess.CalledProcessError as e:
2673 if e.returncode != 4:
2676 logging.debug(_('JAR signature verified: {path}').format(path=apk))
2679 logging.error(_('Old APK signature failed to verify: {path}').format(path=apk)
2680 + '\n' + output.decode('utf-8'))
2684 apk_badchars = re.compile('''[/ :;'"]''')
2687 def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
2690 Returns None if the apk content is the same (apart from the signing key),
2691 otherwise a string describing what's different, or what went wrong when
2692 trying to do the comparison.
2698 absapk1 = os.path.abspath(apk1)
2699 absapk2 = os.path.abspath(apk2)
2701 if set_command_in_config('diffoscope'):
2702 logfilename = os.path.join(log_dir, os.path.basename(absapk1))
2703 htmlfile = logfilename + '.diffoscope.html'
2704 textfile = logfilename + '.diffoscope.txt'
2705 if subprocess.call([config['diffoscope'],
2706 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2707 '--html', htmlfile, '--text', textfile,
2708 absapk1, absapk2]) != 0:
2709 return("Failed to unpack " + apk1)
2711 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2712 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2713 for d in [apk1dir, apk2dir]:
2714 if os.path.exists(d):
2717 os.mkdir(os.path.join(d, 'jar-xf'))
2719 if subprocess.call(['jar', 'xf',
2720 os.path.abspath(apk1)],
2721 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2722 return("Failed to unpack " + apk1)
2723 if subprocess.call(['jar', 'xf',
2724 os.path.abspath(apk2)],
2725 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2726 return("Failed to unpack " + apk2)
2728 if set_command_in_config('apktool'):
2729 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2731 return("Failed to unpack " + apk1)
2732 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2734 return("Failed to unpack " + apk2)
2736 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2737 lines = p.output.splitlines()
2738 if len(lines) != 1 or 'META-INF' not in lines[0]:
2739 if set_command_in_config('meld'):
2740 p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
2741 return("Unexpected diff output - " + p.output)
2743 # since everything verifies, delete the comparison to keep cruft down
2744 shutil.rmtree(apk1dir)
2745 shutil.rmtree(apk2dir)
2747 # If we get here, it seems like they're the same!
2751 def set_command_in_config(command):
2752 '''Try to find specified command in the path, if it hasn't been
2753 manually set in config.py. If found, it is added to the config
2754 dict. The return value says whether the command is available.
2757 if command in config:
2760 tmp = find_command(command)
2762 config[command] = tmp
2767 def find_command(command):
2768 '''find the full path of a command, or None if it can't be found in the PATH'''
2771 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2773 fpath, fname = os.path.split(command)
2778 for path in os.environ["PATH"].split(os.pathsep):
2779 path = path.strip('"')
2780 exe_file = os.path.join(path, command)
2781 if is_exe(exe_file):
2788 '''generate a random password for when generating keys'''
2789 h = hashlib.sha256()
2790 h.update(os.urandom(16)) # salt
2791 h.update(socket.getfqdn().encode('utf-8'))
2792 passwd = base64.b64encode(h.digest()).strip()
2793 return passwd.decode('utf-8')
2796 def genkeystore(localconfig):
2798 Generate a new key with password provided in :param localconfig and add it to new keystore
2799 :return: hexed public key, public key fingerprint
2801 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2802 keystoredir = os.path.dirname(localconfig['keystore'])
2803 if keystoredir is None or keystoredir == '':
2804 keystoredir = os.path.join(os.getcwd(), keystoredir)
2805 if not os.path.exists(keystoredir):
2806 os.makedirs(keystoredir, mode=0o700)
2809 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
2810 'FDROID_KEY_PASS': localconfig['keypass'],
2812 p = FDroidPopen([config['keytool'], '-genkey',
2813 '-keystore', localconfig['keystore'],
2814 '-alias', localconfig['repo_keyalias'],
2815 '-keyalg', 'RSA', '-keysize', '4096',
2816 '-sigalg', 'SHA256withRSA',
2817 '-validity', '10000',
2818 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2819 '-keypass:env', 'FDROID_KEY_PASS',
2820 '-dname', localconfig['keydname']], envs=env_vars)
2821 if p.returncode != 0:
2822 raise BuildException("Failed to generate key", p.output)
2823 os.chmod(localconfig['keystore'], 0o0600)
2824 if not options.quiet:
2825 # now show the lovely key that was just generated
2826 p = FDroidPopen([config['keytool'], '-list', '-v',
2827 '-keystore', localconfig['keystore'],
2828 '-alias', localconfig['repo_keyalias'],
2829 '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
2830 logging.info(p.output.strip() + '\n\n')
2831 # get the public key
2832 p = FDroidPopenBytes([config['keytool'], '-exportcert',
2833 '-keystore', localconfig['keystore'],
2834 '-alias', localconfig['repo_keyalias'],
2835 '-storepass:env', 'FDROID_KEY_STORE_PASS']
2836 + config['smartcardoptions'],
2837 envs=env_vars, output=False, stderr_to_stdout=False)
2838 if p.returncode != 0 or len(p.output) < 20:
2839 raise BuildException("Failed to get public key", p.output)
2841 fingerprint = get_cert_fingerprint(pubkey)
2842 return hexlify(pubkey), fingerprint
2845 def get_cert_fingerprint(pubkey):
2847 Generate a certificate fingerprint the same way keytool does it
2848 (but with slightly different formatting)
2850 digest = hashlib.sha256(pubkey).digest()
2851 ret = [' '.join("%02X" % b for b in bytearray(digest))]
2852 return " ".join(ret)
2855 def get_certificate(certificate_file):
2857 Extracts a certificate from the given file.
2858 :param certificate_file: file bytes (as string) representing the certificate
2859 :return: A binary representation of the certificate's public key, or None in case of error
2861 content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
2862 if content.getComponentByName('contentType') != rfc2315.signedData:
2864 content = decoder.decode(content.getComponentByName('content'),
2865 asn1Spec=rfc2315.SignedData())[0]
2867 certificates = content.getComponentByName('certificates')
2868 cert = certificates[0].getComponentByName('certificate')
2870 logging.error("Certificates not found.")
2872 return encoder.encode(cert)
2875 def load_stats_fdroid_signing_key_fingerprints():
2876 """Load list of signing-key fingerprints stored by fdroid publish from file.
2878 :returns: list of dictionanryies containing the singing-key fingerprints.
2880 jar_file = os.path.join('stats', 'publishsigkeys.jar')
2881 if not os.path.isfile(jar_file):
2883 cmd = [config['jarsigner'], '-strict', '-verify', jar_file]
2884 p = FDroidPopen(cmd, output=False)
2885 if p.returncode != 4:
2886 raise FDroidException("Signature validation of '{}' failed! "
2887 "Please run publish again to rebuild this file.".format(jar_file))
2889 jar_sigkey = apk_signer_fingerprint(jar_file)
2890 repo_key_sig = config.get('repo_key_sha256')
2892 if jar_sigkey != repo_key_sig:
2893 raise FDroidException("Signature key fingerprint of file '{}' does not match repo_key_sha256 in config.py (found fingerprint: '{}')".format(jar_file, jar_sigkey))
2895 logging.warning("repo_key_sha256 not in config.py, setting it to the signature key fingerprint of '{}'".format(jar_file))
2896 config['repo_key_sha256'] = jar_sigkey
2897 write_to_config(config, 'repo_key_sha256')
2899 with zipfile.ZipFile(jar_file, 'r') as f:
2900 return json.loads(str(f.read('publishsigkeys.json'), 'utf-8'))
2903 def write_to_config(thisconfig, key, value=None, config_file=None):
2904 '''write a key/value to the local config.py
2906 NOTE: only supports writing string variables.
2908 :param thisconfig: config dictionary
2909 :param key: variable name in config.py to be overwritten/added
2910 :param value: optional value to be written, instead of fetched
2911 from 'thisconfig' dictionary.
2914 origkey = key + '_orig'
2915 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2916 cfg = config_file if config_file else 'config.py'
2918 # load config file, create one if it doesn't exist
2919 if not os.path.exists(cfg):
2920 open(cfg, 'a').close()
2921 logging.info("Creating empty " + cfg)
2922 with open(cfg, 'r', encoding="utf-8") as f:
2923 lines = f.readlines()
2925 # make sure the file ends with a carraige return
2927 if not lines[-1].endswith('\n'):
2930 # regex for finding and replacing python string variable
2931 # definitions/initializations
2932 pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
2933 repl = key + ' = "' + value + '"'
2934 pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
2935 repl2 = key + " = '" + value + "'"
2937 # If we replaced this line once, we make sure won't be a
2938 # second instance of this line for this key in the document.
2941 with open(cfg, 'w', encoding="utf-8") as f:
2943 if pattern.match(line) or pattern2.match(line):
2945 line = pattern.sub(repl, line)
2946 line = pattern2.sub(repl2, line)
2957 def parse_xml(path):
2958 return XMLElementTree.parse(path).getroot()
2961 def string_is_integer(string):
2969 def local_rsync(options, fromdir, todir):
2970 '''Rsync method for local to local copying of things
2972 This is an rsync wrapper with all the settings for safe use within
2973 the various fdroidserver use cases. This uses stricter rsync
2974 checking on all files since people using offline mode are already
2975 prioritizing security above ease and speed.
2978 rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
2979 '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
2980 if not options.no_checksum:
2981 rsyncargs.append('--checksum')
2983 rsyncargs += ['--verbose']
2985 rsyncargs += ['--quiet']
2986 logging.debug(' '.join(rsyncargs + [fromdir, todir]))
2987 if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
2988 raise FDroidException()
2991 def get_per_app_repos():
2992 '''per-app repos are dirs named with the packageName of a single app'''
2994 # Android packageNames are Java packages, they may contain uppercase or
2995 # lowercase letters ('A' through 'Z'), numbers, and underscores
2996 # ('_'). However, individual package name parts may only start with
2997 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2998 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
3001 for root, dirs, files in os.walk(os.getcwd()):
3003 print('checking', root, 'for', d)
3004 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
3005 # standard parts of an fdroid repo, so never packageNames
3008 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
3014 def is_repo_file(filename):
3015 '''Whether the file in a repo is a build product to be delivered to users'''
3016 if isinstance(filename, str):
3017 filename = filename.encode('utf-8', errors="surrogateescape")
3018 return os.path.isfile(filename) \
3019 and not filename.endswith(b'.asc') \
3020 and not filename.endswith(b'.sig') \
3021 and os.path.basename(filename) not in [
3023 b'index_unsigned.jar',
3032 def get_examples_dir():
3033 '''Return the dir where the fdroidserver example files are available'''
3035 tmp = os.path.dirname(sys.argv[0])
3036 if os.path.basename(tmp) == 'bin':
3037 egg_links = glob.glob(os.path.join(tmp, '..',
3038 'local/lib/python3.*/site-packages/fdroidserver.egg-link'))
3040 # installed from local git repo
3041 examplesdir = os.path.join(open(egg_links[0]).readline().rstrip(), 'examples')
3044 examplesdir = os.path.dirname(os.path.dirname(__file__)) + '/share/doc/fdroidserver/examples'
3045 if not os.path.exists(examplesdir): # use UNIX layout
3046 examplesdir = os.path.dirname(tmp) + '/share/doc/fdroidserver/examples'
3048 # we're running straight out of the git repo
3049 prefix = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
3050 examplesdir = prefix + '/examples'
3055 def get_wiki_timestamp(timestamp=None):
3056 """Return current time in the standard format for posting to the wiki"""
3058 if timestamp is None:
3059 timestamp = time.gmtime()
3060 return time.strftime("%Y-%m-%d %H:%M:%SZ", timestamp)
3063 def get_android_tools_versions(ndk_path=None):
3064 '''get a list of the versions of all installed Android SDK/NDK components'''
3067 sdk_path = config['sdk_path']
3068 if sdk_path[-1] != '/':
3072 ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT')
3073 if os.path.isfile(ndk_release_txt):
3074 with open(ndk_release_txt, 'r') as fp:
3075 components.append((os.path.basename(ndk_path), fp.read()[:-1]))
3077 pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE)
3078 for root, dirs, files in os.walk(sdk_path):
3079 if 'source.properties' in files:
3080 source_properties = os.path.join(root, 'source.properties')
3081 with open(source_properties, 'r') as fp:
3082 m = pattern.search(fp.read())
3084 components.append((root[len(sdk_path):], m.group(1)))
3089 def get_android_tools_version_log(ndk_path=None):
3090 '''get a list of the versions of all installed Android SDK/NDK components'''
3091 log = '== Installed Android Tools ==\n\n'
3092 components = get_android_tools_versions(ndk_path)
3093 for name, version in sorted(components):
3094 log += '* ' + name + ' (' + version + ')\n'