3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
40 import xml.etree.ElementTree as XMLElementTree
42 from binascii import hexlify
43 from datetime import datetime, timedelta
44 from distutils.version import LooseVersion
45 from queue import Queue
46 from zipfile import ZipFile
48 from pyasn1.codec.der import decoder, encoder
49 from pyasn1_modules import rfc2315
50 from pyasn1.error import PyAsn1Error
52 from distutils.util import strtobool
54 import fdroidserver.metadata
55 from fdroidserver import _
56 from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException,\
57 BuildException, VerificationException
58 from .asynchronousfilereader import AsynchronousFileReader
60 # this is the build-tools version, aapt has a separate version that
61 # has to be manually set in test_aapt_version()
62 MINIMUM_AAPT_VERSION = '26.0.0'
64 # A signature block file with a .DSA, .RSA, or .EC extension
65 CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
66 APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
67 STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
69 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
78 'sdk_path': "$ANDROID_HOME",
82 'r12b': "$ANDROID_NDK",
88 'build_tools': MINIMUM_AAPT_VERSION,
89 'force_build_tools': False,
94 'accepted_formats': ['txt', 'yml'],
95 'sync_from_local_copy_dir': False,
96 'allow_disabled_algorithms': False,
97 'per_app_repos': False,
98 'make_current_version_link': True,
99 'current_version_name_source': 'Name',
100 'update_stats': False,
102 'stats_server': None,
104 'stats_to_carbon': False,
106 'build_server_always': False,
107 'keystore': 'keystore.jks',
108 'smartcardoptions': [],
118 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
119 'repo_name': "My First FDroid Repo Demo",
120 'repo_icon': "fdroid-icon.png",
121 'repo_description': '''
122 This is a repository of apps to be used with FDroid. Applications in this
123 repository are either official binaries built by the original application
124 developers, or are binaries built from source by the admin of f-droid.org
125 using the tools on https://gitlab.com/u/fdroid.
131 def setup_global_opts(parser):
132 try: # the buildserver VM might not have PIL installed
133 from PIL import PngImagePlugin
134 logger = logging.getLogger(PngImagePlugin.__name__)
135 logger.setLevel(logging.INFO) # tame the "STREAM" debug messages
139 parser.add_argument("-v", "--verbose", action="store_true", default=False,
140 help=_("Spew out even more information than normal"))
141 parser.add_argument("-q", "--quiet", action="store_true", default=False,
142 help=_("Restrict output to warnings and errors"))
145 def _add_java_paths_to_config(pathlist, thisconfig):
146 def path_version_key(s):
148 for u in re.split('[^0-9]+', s):
150 versionlist.append(int(u))
155 for d in sorted(pathlist, key=path_version_key):
156 if os.path.islink(d):
158 j = os.path.basename(d)
159 # the last one found will be the canonical one, so order appropriately
161 r'^1\.([6-9])\.0\.jdk$', # OSX
162 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
163 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
164 r'^jdk([6-9])-openjdk$', # Arch
165 r'^java-([6-9])-openjdk$', # Arch
166 r'^java-([6-9])-jdk$', # Arch (oracle)
167 r'^java-1\.([6-9])\.0-.*$', # RedHat
168 r'^java-([6-9])-oracle$', # Debian WebUpd8
169 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
170 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
172 m = re.match(regex, j)
175 for p in [d, os.path.join(d, 'Contents', 'Home')]:
176 if os.path.exists(os.path.join(p, 'bin', 'javac')):
177 thisconfig['java_paths'][m.group(1)] = p
180 def fill_config_defaults(thisconfig):
181 for k, v in default_config.items():
182 if k not in thisconfig:
185 # Expand paths (~users and $vars)
186 def expand_path(path):
190 path = os.path.expanduser(path)
191 path = os.path.expandvars(path)
196 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
201 thisconfig[k + '_orig'] = v
203 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
204 if thisconfig['java_paths'] is None:
205 thisconfig['java_paths'] = dict()
207 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
208 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
209 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
210 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
211 if os.getenv('JAVA_HOME') is not None:
212 pathlist.append(os.getenv('JAVA_HOME'))
213 if os.getenv('PROGRAMFILES') is not None:
214 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
215 _add_java_paths_to_config(pathlist, thisconfig)
217 for java_version in ('7', '8', '9'):
218 if java_version not in thisconfig['java_paths']:
220 java_home = thisconfig['java_paths'][java_version]
221 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
222 if os.path.exists(jarsigner):
223 thisconfig['jarsigner'] = jarsigner
224 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
225 break # Java7 is preferred, so quit if found
227 for k in ['ndk_paths', 'java_paths']:
233 thisconfig[k][k2] = exp
234 thisconfig[k][k2 + '_orig'] = v
237 def regsub_file(pattern, repl, path):
238 with open(path, 'rb') as f:
240 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
241 with open(path, 'wb') as f:
245 def read_config(opts, config_file='config.py'):
246 """Read the repository config
248 The config is read from config_file, which is in the current
249 directory when any of the repo management commands are used. If
250 there is a local metadata file in the git repo, then config.py is
251 not required, just use defaults.
254 global config, options
256 if config is not None:
263 if os.path.isfile(config_file):
264 logging.debug(_("Reading '{config_file}'").format(config_file=config_file))
265 with io.open(config_file, "rb") as f:
266 code = compile(f.read(), config_file, 'exec')
267 exec(code, None, config)
269 logging.warning(_("No 'config.py' found, using defaults."))
271 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
273 if not type(config[k]) in (str, list, tuple):
275 _("'{field}' will be in random order! Use () or [] brackets if order is important!")
278 # smartcardoptions must be a list since its command line args for Popen
279 if 'smartcardoptions' in config:
280 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
281 elif 'keystore' in config and config['keystore'] == 'NONE':
282 # keystore='NONE' means use smartcard, these are required defaults
283 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
284 'SunPKCS11-OpenSC', '-providerClass',
285 'sun.security.pkcs11.SunPKCS11',
286 '-providerArg', 'opensc-fdroid.cfg']
288 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
289 st = os.stat(config_file)
290 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
291 logging.warning(_("unsafe permissions on '{config_file}' (should be 0600)!")
292 .format(config_file=config_file))
294 fill_config_defaults(config)
296 for k in ["repo_description", "archive_description"]:
298 config[k] = clean_description(config[k])
300 if 'serverwebroot' in config:
301 if isinstance(config['serverwebroot'], str):
302 roots = [config['serverwebroot']]
303 elif all(isinstance(item, str) for item in config['serverwebroot']):
304 roots = config['serverwebroot']
306 raise TypeError(_('only accepts strings, lists, and tuples'))
308 for rootstr in roots:
309 # since this is used with rsync, where trailing slashes have
310 # meaning, ensure there is always a trailing slash
311 if rootstr[-1] != '/':
313 rootlist.append(rootstr.replace('//', '/'))
314 config['serverwebroot'] = rootlist
316 if 'servergitmirrors' in config:
317 if isinstance(config['servergitmirrors'], str):
318 roots = [config['servergitmirrors']]
319 elif all(isinstance(item, str) for item in config['servergitmirrors']):
320 roots = config['servergitmirrors']
322 raise TypeError(_('only accepts strings, lists, and tuples'))
323 config['servergitmirrors'] = roots
328 def assert_config_keystore(config):
329 """Check weather keystore is configured correctly and raise exception if not."""
332 if 'repo_keyalias' not in config:
334 logging.critical(_("'repo_keyalias' not found in config.py!"))
335 if 'keystore' not in config:
337 logging.critical(_("'keystore' not found in config.py!"))
338 elif not os.path.exists(config['keystore']):
340 logging.critical("'" + config['keystore'] + "' does not exist!")
341 if 'keystorepass' not in config:
343 logging.critical(_("'keystorepass' not found in config.py!"))
344 if 'keypass' not in config:
346 logging.critical(_("'keypass' not found in config.py!"))
348 raise FDroidException("This command requires a signing key, " +
349 "you can create one using: fdroid update --create-key")
352 def find_sdk_tools_cmd(cmd):
353 '''find a working path to a tool from the Android SDK'''
356 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
357 # try to find a working path to this command, in all the recent possible paths
358 if 'build_tools' in config:
359 build_tools = os.path.join(config['sdk_path'], 'build-tools')
360 # if 'build_tools' was manually set and exists, check only that one
361 configed_build_tools = os.path.join(build_tools, config['build_tools'])
362 if os.path.exists(configed_build_tools):
363 tooldirs.append(configed_build_tools)
365 # no configed version, so hunt known paths for it
366 for f in sorted(os.listdir(build_tools), reverse=True):
367 if os.path.isdir(os.path.join(build_tools, f)):
368 tooldirs.append(os.path.join(build_tools, f))
369 tooldirs.append(build_tools)
370 sdk_tools = os.path.join(config['sdk_path'], 'tools')
371 if os.path.exists(sdk_tools):
372 tooldirs.append(sdk_tools)
373 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
374 if os.path.exists(sdk_platform_tools):
375 tooldirs.append(sdk_platform_tools)
376 tooldirs.append('/usr/bin')
378 path = os.path.join(d, cmd)
379 if os.path.isfile(path):
381 test_aapt_version(path)
383 # did not find the command, exit with error message
384 ensure_build_tools_exists(config)
387 def test_aapt_version(aapt):
388 '''Check whether the version of aapt is new enough'''
389 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
390 if output is None or output == '':
391 logging.error(_("'{path}' failed to execute!").format(path=aapt))
393 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
398 # the Debian package has the version string like "v0.2-23.0.2"
401 if LooseVersion(bugfix) < LooseVersion(MINIMUM_AAPT_VERSION):
403 elif LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.4062713'):
406 logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-{version} or newer!")
407 .format(aapt=aapt, version=MINIMUM_AAPT_VERSION))
409 logging.warning(_('Unknown version of aapt, might cause problems: ') + output)
412 def test_sdk_exists(thisconfig):
413 if 'sdk_path' not in thisconfig:
414 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
415 test_aapt_version(thisconfig['aapt'])
418 logging.error(_("'sdk_path' not set in 'config.py'!"))
420 if thisconfig['sdk_path'] == default_config['sdk_path']:
421 logging.error(_('No Android SDK found!'))
422 logging.error(_('You can use ANDROID_HOME to set the path to your SDK, i.e.:'))
423 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
425 if not os.path.exists(thisconfig['sdk_path']):
426 logging.critical(_("Android SDK path '{path}' does not exist!")
427 .format(path=thisconfig['sdk_path']))
429 if not os.path.isdir(thisconfig['sdk_path']):
430 logging.critical(_("Android SDK path '{path}' is not a directory!")
431 .format(path=thisconfig['sdk_path']))
433 for d in ['build-tools', 'platform-tools', 'tools']:
434 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
435 logging.critical(_("Android SDK '{path}' does not have '{dirname}' installed!")
436 .format(path=thisconfig['sdk_path'], dirname=d))
441 def ensure_build_tools_exists(thisconfig):
442 if not test_sdk_exists(thisconfig):
443 raise FDroidException(_("Android SDK not found!"))
444 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
445 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
446 if not os.path.isdir(versioned_build_tools):
447 raise FDroidException(
448 _("Android build-tools path '{path}' does not exist!")
449 .format(path=versioned_build_tools))
452 def get_local_metadata_files():
453 '''get any metadata files local to an app's source repo
455 This tries to ignore anything that does not count as app metdata,
456 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
459 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
462 def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False):
464 :param appids: arguments in the form of multiple appid:[vc] strings
465 :returns: a dictionary with the set of vercodes specified for each package
468 if not appid_versionCode_pairs:
471 for p in appid_versionCode_pairs:
472 if allow_vercodes and ':' in p:
473 package, vercode = p.split(':')
475 package, vercode = p, None
476 if package not in vercodes:
477 vercodes[package] = [vercode] if vercode else []
479 elif vercode and vercode not in vercodes[package]:
480 vercodes[package] += [vercode] if vercode else []
485 def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
486 """Build a list of App instances for processing
488 On top of what read_pkg_args does, this returns the whole app
489 metadata, but limiting the builds list to the builds matching the
490 appid_versionCode_pairs and vercodes specified. If no appid_versionCode_pairs are specified, then
491 all App and Build instances are returned.
495 vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes)
501 for appid, app in allapps.items():
502 if appid in vercodes:
505 if len(apps) != len(vercodes):
508 logging.critical(_("No such package: %s") % p)
509 raise FDroidException(_("Found invalid appids in arguments"))
511 raise FDroidException(_("No packages specified"))
514 for appid, app in apps.items():
518 app.builds = [b for b in app.builds if b.versionCode in vc]
519 if len(app.builds) != len(vercodes[appid]):
521 allvcs = [b.versionCode for b in app.builds]
522 for v in vercodes[appid]:
524 logging.critical(_("No such versionCode {versionCode} for app {appid}")
525 .format(versionCode=v, appid=appid))
528 raise FDroidException(_("Found invalid versionCodes for some apps"))
533 def get_extension(filename):
534 base, ext = os.path.splitext(filename)
537 return base, ext.lower()[1:]
540 def has_extension(filename, ext):
541 _ignored, f_ext = get_extension(filename)
545 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
548 def clean_description(description):
549 'Remove unneeded newlines and spaces from a block of description text'
551 # this is split up by paragraph to make removing the newlines easier
552 for paragraph in re.split(r'\n\n', description):
553 paragraph = re.sub('\r', '', paragraph)
554 paragraph = re.sub('\n', ' ', paragraph)
555 paragraph = re.sub(' {2,}', ' ', paragraph)
556 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
557 returnstring += paragraph + '\n\n'
558 return returnstring.rstrip('\n')
561 def publishednameinfo(filename):
562 filename = os.path.basename(filename)
563 m = publish_name_regex.match(filename)
565 result = (m.group(1), m.group(2))
566 except AttributeError:
567 raise FDroidException(_("Invalid name for published file: %s") % filename)
571 apk_release_filename = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)\.apk')
572 apk_release_filename_with_sigfp = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)_(?P<sigfp>[0-9a-f]{7})\.apk')
575 def apk_parse_release_filename(apkname):
576 """Parses the name of an APK file according the F-Droids APK naming
577 scheme and returns the tokens.
579 WARNING: Returned values don't necessarily represent the APKs actual
580 properties, the are just paresed from the file name.
582 :returns: A triplet containing (appid, versionCode, signer), where appid
583 should be the package name, versionCode should be the integer
584 represion of the APKs version and signer should be the first 7 hex
585 digists of the sha256 signing key fingerprint which was used to sign
588 m = apk_release_filename_with_sigfp.match(apkname)
590 return m.group('appid'), m.group('vercode'), m.group('sigfp')
591 m = apk_release_filename.match(apkname)
593 return m.group('appid'), m.group('vercode'), None
594 return None, None, None
597 def get_release_filename(app, build):
599 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
601 return "%s_%s.apk" % (app.id, build.versionCode)
604 def get_toolsversion_logname(app, build):
605 return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
608 def getsrcname(app, build):
609 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
621 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
624 def get_build_dir(app):
625 '''get the dir that this app will be built in'''
627 if app.RepoType == 'srclib':
628 return os.path.join('build', 'srclib', app.Repo)
630 return os.path.join('build', app.id)
634 '''checkout code from VCS and return instance of vcs and the build dir'''
635 build_dir = get_build_dir(app)
637 # Set up vcs interface and make sure we have the latest code...
638 logging.debug("Getting {0} vcs interface for {1}"
639 .format(app.RepoType, app.Repo))
640 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
644 vcs = getvcs(app.RepoType, remote, build_dir)
646 return vcs, build_dir
649 def getvcs(vcstype, remote, local):
651 return vcs_git(remote, local)
652 if vcstype == 'git-svn':
653 return vcs_gitsvn(remote, local)
655 return vcs_hg(remote, local)
657 return vcs_bzr(remote, local)
658 if vcstype == 'srclib':
659 if local != os.path.join('build', 'srclib', remote):
660 raise VCSException("Error: srclib paths are hard-coded!")
661 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
663 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
664 raise VCSException("Invalid vcs type " + vcstype)
667 def getsrclibvcs(name):
668 if name not in fdroidserver.metadata.srclibs:
669 raise VCSException("Missing srclib " + name)
670 return fdroidserver.metadata.srclibs[name]['Repo Type']
675 def __init__(self, remote, local):
677 # svn, git-svn and bzr may require auth
679 if self.repotype() in ('git-svn', 'bzr'):
681 if self.repotype == 'git-svn':
682 raise VCSException("Authentication is not supported for git-svn")
683 self.username, remote = remote.split('@')
684 if ':' not in self.username:
685 raise VCSException(_("Password required with username"))
686 self.username, self.password = self.username.split(':')
690 self.clone_failed = False
691 self.refreshed = False
697 def clientversion(self):
698 versionstr = FDroidPopen(self.clientversioncmd()).output
699 return versionstr[0:versionstr.find('\n')]
701 def clientversioncmd(self):
704 def gotorevision(self, rev, refresh=True):
705 """Take the local repository to a clean version of the given
706 revision, which is specificed in the VCS's native
707 format. Beforehand, the repository can be dirty, or even
708 non-existent. If the repository does already exist locally, it
709 will be updated from the origin, but only once in the lifetime
710 of the vcs object. None is acceptable for 'rev' if you know
711 you are cloning a clean copy of the repo - otherwise it must
712 specify a valid revision.
715 if self.clone_failed:
716 raise VCSException(_("Downloading the repository already failed once, not trying again."))
718 # The .fdroidvcs-id file for a repo tells us what VCS type
719 # and remote that directory was created from, allowing us to drop it
720 # automatically if either of those things changes.
721 fdpath = os.path.join(self.local, '..',
722 '.fdroidvcs-' + os.path.basename(self.local))
723 fdpath = os.path.normpath(fdpath)
724 cdata = self.repotype() + ' ' + self.remote
727 if os.path.exists(self.local):
728 if os.path.exists(fdpath):
729 with open(fdpath, 'r') as f:
730 fsdata = f.read().strip()
735 logging.info("Repository details for %s changed - deleting" % (
739 logging.info("Repository details for %s missing - deleting" % (
742 shutil.rmtree(self.local)
746 self.refreshed = True
749 self.gotorevisionx(rev)
750 except FDroidException as e:
753 # If necessary, write the .fdroidvcs file.
754 if writeback and not self.clone_failed:
755 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
756 with open(fdpath, 'w+') as f:
762 def gotorevisionx(self, rev): # pylint: disable=unused-argument
763 """Derived classes need to implement this.
765 It's called once basic checking has been performed.
767 raise VCSException("This VCS type doesn't define gotorevisionx")
769 # Initialise and update submodules
770 def initsubmodules(self):
771 raise VCSException('Submodules not supported for this vcs type')
773 # Get a list of all known tags
775 if not self._gettags:
776 raise VCSException('gettags not supported for this vcs type')
778 for tag in self._gettags():
779 if re.match('[-A-Za-z0-9_. /]+$', tag):
783 def latesttags(self):
784 """Get a list of all the known tags, sorted from newest to oldest"""
785 raise VCSException('latesttags not supported for this vcs type')
788 """Get current commit reference (hash, revision, etc)"""
789 raise VCSException('getref not supported for this vcs type')
792 """Returns the srclib (name, path) used in setting up the current revision, or None."""
801 def clientversioncmd(self):
802 return ['git', '--version']
804 def git(self, args, envs=dict(), cwd=None, output=True):
805 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
807 While fetch/pull/clone respect the command line option flags,
808 it seems that submodule commands do not. They do seem to
809 follow whatever is in env vars, if the version of git is new
810 enough. So we just throw the kitchen sink at it to see what
813 Also, because of CVE-2017-1000117, block all SSH URLs.
816 # supported in git >= 2.3
818 '-c', 'core.sshCommand=/bin/false',
819 '-c', 'url.https://.insteadOf=ssh://',
821 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
822 git_config.append('-c')
823 git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':')
824 git_config.append('-c')
825 git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain)
826 git_config.append('-c')
827 git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain)
829 'GIT_TERMINAL_PROMPT': '0',
830 'GIT_SSH': '/bin/false', # for git < 2.3
832 return FDroidPopen(['git', ] + git_config + args,
833 envs=envs, cwd=cwd, output=output)
836 """If the local directory exists, but is somehow not a git repository,
837 git will traverse up the directory tree until it finds one
838 that is (i.e. fdroidserver) and then we'll proceed to destroy
839 it! This is called as a safety check.
843 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
844 result = p.output.rstrip()
845 if not result.endswith(self.local):
846 raise VCSException('Repository mismatch')
848 def gotorevisionx(self, rev):
849 if not os.path.exists(self.local):
851 p = self.git(['clone', '--', self.remote, self.local])
852 if p.returncode != 0:
853 self.clone_failed = True
854 raise VCSException("Git clone failed", p.output)
858 # Discard any working tree changes
859 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
860 'git', 'reset', '--hard'], cwd=self.local, output=False)
861 if p.returncode != 0:
862 raise VCSException(_("Git reset failed"), p.output)
863 # Remove untracked files now, in case they're tracked in the target
864 # revision (it happens!)
865 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
866 'git', 'clean', '-dffx'], cwd=self.local, output=False)
867 if p.returncode != 0:
868 raise VCSException(_("Git clean failed"), p.output)
869 if not self.refreshed:
870 # Get latest commits and tags from remote
871 p = self.git(['fetch', 'origin'], cwd=self.local)
872 if p.returncode != 0:
873 raise VCSException(_("Git fetch failed"), p.output)
874 p = self.git(['fetch', '--prune', '--tags', 'origin'], output=False, cwd=self.local)
875 if p.returncode != 0:
876 raise VCSException(_("Git fetch failed"), p.output)
877 # Recreate origin/HEAD as git clone would do it, in case it disappeared
878 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
879 if p.returncode != 0:
880 lines = p.output.splitlines()
881 if 'Multiple remote HEAD branches' not in lines[0]:
882 raise VCSException(_("Git remote set-head failed"), p.output)
883 branch = lines[1].split(' ')[-1]
884 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--', branch],
885 cwd=self.local, output=False)
886 if p2.returncode != 0:
887 raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output)
888 self.refreshed = True
889 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
890 # a github repo. Most of the time this is the same as origin/master.
891 rev = rev or 'origin/HEAD'
892 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
893 if p.returncode != 0:
894 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
895 # Get rid of any uncontrolled files left behind
896 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
897 if p.returncode != 0:
898 raise VCSException(_("Git clean failed"), p.output)
900 def initsubmodules(self):
902 submfile = os.path.join(self.local, '.gitmodules')
903 if not os.path.isfile(submfile):
904 raise NoSubmodulesException(_("No git submodules available"))
906 # fix submodules not accessible without an account and public key auth
907 with open(submfile, 'r') as f:
908 lines = f.readlines()
909 with open(submfile, 'w') as f:
911 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
912 line = re.sub('git@' + domain + ':', 'https://u:p@' + domain + '/', line)
915 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
916 if p.returncode != 0:
917 raise VCSException(_("Git submodule sync failed"), p.output)
918 p = self.git(['submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
919 if p.returncode != 0:
920 raise VCSException(_("Git submodule update failed"), p.output)
924 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
925 return p.output.splitlines()
927 tag_format = re.compile(r'tag: ([^),]*)')
929 def latesttags(self):
931 p = FDroidPopen(['git', 'log', '--tags',
932 '--simplify-by-decoration', '--pretty=format:%d'],
933 cwd=self.local, output=False)
935 for line in p.output.splitlines():
936 for tag in self.tag_format.findall(line):
941 class vcs_gitsvn(vcs):
946 def clientversioncmd(self):
947 return ['git', 'svn', '--version']
950 """If the local directory exists, but is somehow not a git repository,
951 git will traverse up the directory tree until it finds one that
952 is (i.e. fdroidserver) and then we'll proceed to destory it!
953 This is called as a safety check.
956 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
957 result = p.output.rstrip()
958 if not result.endswith(self.local):
959 raise VCSException('Repository mismatch')
961 def git(self, args, envs=dict(), cwd=None, output=True):
962 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
964 # CVE-2017-1000117 block all SSH URLs (supported in git >= 2.3)
965 config = ['-c', 'core.sshCommand=false']
967 'GIT_TERMINAL_PROMPT': '0',
968 'GIT_SSH': '/bin/false', # for git < 2.3
969 'SVN_SSH': '/bin/false',
971 return FDroidPopen(['git', ] + config + args,
972 envs=envs, cwd=cwd, output=output)
974 def gotorevisionx(self, rev):
975 if not os.path.exists(self.local):
977 gitsvn_args = ['svn', 'clone']
979 if ';' in self.remote:
980 remote_split = self.remote.split(';')
981 for i in remote_split[1:]:
982 if i.startswith('trunk='):
983 gitsvn_args.extend(['-T', i[6:]])
984 elif i.startswith('tags='):
985 gitsvn_args.extend(['-t', i[5:]])
986 elif i.startswith('branches='):
987 gitsvn_args.extend(['-b', i[9:]])
988 remote = remote_split[0]
992 gitsvn_args.extend(['--', remote, self.local])
993 p = self.git(gitsvn_args)
994 if p.returncode != 0:
995 self.clone_failed = True
996 raise VCSException(_('git svn clone failed'), p.output)
1000 # Discard any working tree changes
1001 p = self.git(['reset', '--hard'], cwd=self.local, output=False)
1002 if p.returncode != 0:
1003 raise VCSException("Git reset failed", p.output)
1004 # Remove untracked files now, in case they're tracked in the target
1005 # revision (it happens!)
1006 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1007 if p.returncode != 0:
1008 raise VCSException("Git clean failed", p.output)
1009 if not self.refreshed:
1010 # Get new commits, branches and tags from repo
1011 p = self.git(['svn', 'fetch'], cwd=self.local, output=False)
1012 if p.returncode != 0:
1013 raise VCSException("Git svn fetch failed")
1014 p = self.git(['svn', 'rebase'], cwd=self.local, output=False)
1015 if p.returncode != 0:
1016 raise VCSException("Git svn rebase failed", p.output)
1017 self.refreshed = True
1019 rev = rev or 'master'
1021 nospaces_rev = rev.replace(' ', '%20')
1022 # Try finding a svn tag
1023 for treeish in ['origin/', '']:
1024 p = self.git(['checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
1025 if p.returncode == 0:
1027 if p.returncode != 0:
1028 # No tag found, normal svn rev translation
1029 # Translate svn rev into git format
1030 rev_split = rev.split('/')
1033 for treeish in ['origin/', '']:
1034 if len(rev_split) > 1:
1035 treeish += rev_split[0]
1036 svn_rev = rev_split[1]
1039 # if no branch is specified, then assume trunk (i.e. 'master' branch):
1043 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
1045 p = self.git(['svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
1046 git_rev = p.output.rstrip()
1048 if p.returncode == 0 and git_rev:
1051 if p.returncode != 0 or not git_rev:
1052 # Try a plain git checkout as a last resort
1053 p = self.git(['checkout', rev], cwd=self.local, output=False)
1054 if p.returncode != 0:
1055 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
1057 # Check out the git rev equivalent to the svn rev
1058 p = self.git(['checkout', git_rev], cwd=self.local, output=False)
1059 if p.returncode != 0:
1060 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
1062 # Get rid of any uncontrolled files left behind
1063 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1064 if p.returncode != 0:
1065 raise VCSException(_("Git clean failed"), p.output)
1069 for treeish in ['origin/', '']:
1070 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
1071 if os.path.isdir(d):
1072 return os.listdir(d)
1076 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
1077 if p.returncode != 0:
1079 return p.output.strip()
1087 def clientversioncmd(self):
1088 return ['hg', '--version']
1090 def gotorevisionx(self, rev):
1091 if not os.path.exists(self.local):
1092 p = FDroidPopen(['hg', 'clone', '--ssh', 'false', '--', self.remote, self.local],
1094 if p.returncode != 0:
1095 self.clone_failed = True
1096 raise VCSException("Hg clone failed", p.output)
1098 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
1099 if p.returncode != 0:
1100 raise VCSException("Hg status failed", p.output)
1101 for line in p.output.splitlines():
1102 if not line.startswith('? '):
1103 raise VCSException("Unexpected output from hg status -uS: " + line)
1104 FDroidPopen(['rm', '-rf', '--', line[2:]], cwd=self.local, output=False)
1105 if not self.refreshed:
1106 p = FDroidPopen(['hg', 'pull', '--ssh', 'false'], cwd=self.local, output=False)
1107 if p.returncode != 0:
1108 raise VCSException("Hg pull failed", p.output)
1109 self.refreshed = True
1111 rev = rev or 'default'
1114 p = FDroidPopen(['hg', 'update', '-C', '--', rev], cwd=self.local, output=False)
1115 if p.returncode != 0:
1116 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
1117 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1118 # Also delete untracked files, we have to enable purge extension for that:
1119 if "'purge' is provided by the following extension" in p.output:
1120 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
1121 myfile.write("\n[extensions]\nhgext.purge=\n")
1122 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1123 if p.returncode != 0:
1124 raise VCSException("HG purge failed", p.output)
1125 elif p.returncode != 0:
1126 raise VCSException("HG purge failed", p.output)
1129 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
1130 return p.output.splitlines()[1:]
1138 def clientversioncmd(self):
1139 return ['bzr', '--version']
1141 def bzr(self, args, envs=dict(), cwd=None, output=True):
1142 '''Prevent bzr from ever using SSH to avoid security vulns'''
1146 return FDroidPopen(['bzr', ] + args, envs=envs, cwd=cwd, output=output)
1148 def gotorevisionx(self, rev):
1149 if not os.path.exists(self.local):
1150 p = self.bzr(['branch', self.remote, self.local], output=False)
1151 if p.returncode != 0:
1152 self.clone_failed = True
1153 raise VCSException("Bzr branch failed", p.output)
1155 p = self.bzr(['clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
1156 if p.returncode != 0:
1157 raise VCSException("Bzr revert failed", p.output)
1158 if not self.refreshed:
1159 p = self.bzr(['pull'], cwd=self.local, output=False)
1160 if p.returncode != 0:
1161 raise VCSException("Bzr update failed", p.output)
1162 self.refreshed = True
1164 revargs = list(['-r', rev] if rev else [])
1165 p = self.bzr(['revert'] + revargs, cwd=self.local, output=False)
1166 if p.returncode != 0:
1167 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1170 p = self.bzr(['tags'], cwd=self.local, output=False)
1171 return [tag.split(' ')[0].strip() for tag in
1172 p.output.splitlines()]
1175 def unescape_string(string):
1178 if string[0] == '"' and string[-1] == '"':
1181 return string.replace("\\'", "'")
1184 def retrieve_string(app_dir, string, xmlfiles=None):
1186 if not string.startswith('@string/'):
1187 return unescape_string(string)
1189 if xmlfiles is None:
1192 os.path.join(app_dir, 'res'),
1193 os.path.join(app_dir, 'src', 'main', 'res'),
1195 for root, dirs, files in os.walk(res_dir):
1196 if os.path.basename(root) == 'values':
1197 xmlfiles += [os.path.join(root, x) for x in files if x.endswith('.xml')]
1199 name = string[len('@string/'):]
1201 def element_content(element):
1202 if element.text is None:
1204 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1205 return s.decode('utf-8').strip()
1207 for path in xmlfiles:
1208 if not os.path.isfile(path):
1210 xml = parse_xml(path)
1211 element = xml.find('string[@name="' + name + '"]')
1212 if element is not None:
1213 content = element_content(element)
1214 return retrieve_string(app_dir, content, xmlfiles)
1219 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1220 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1223 def manifest_paths(app_dir, flavours):
1224 '''Return list of existing files that will be used to find the highest vercode'''
1226 possible_manifests = \
1227 [os.path.join(app_dir, 'AndroidManifest.xml'),
1228 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1229 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1230 os.path.join(app_dir, 'build.gradle')]
1232 for flavour in flavours:
1233 if flavour == 'yes':
1235 possible_manifests.append(
1236 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1238 return [path for path in possible_manifests if os.path.isfile(path)]
1241 def fetch_real_name(app_dir, flavours):
1242 '''Retrieve the package name. Returns the name, or None if not found.'''
1243 for path in manifest_paths(app_dir, flavours):
1244 if not has_extension(path, 'xml') or not os.path.isfile(path):
1246 logging.debug("fetch_real_name: Checking manifest at " + path)
1247 xml = parse_xml(path)
1248 app = xml.find('application')
1251 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1253 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1254 result = retrieve_string_singleline(app_dir, label)
1256 result = result.strip()
1261 def get_library_references(root_dir):
1263 proppath = os.path.join(root_dir, 'project.properties')
1264 if not os.path.isfile(proppath):
1266 with open(proppath, 'r', encoding='iso-8859-1') as f:
1268 if not line.startswith('android.library.reference.'):
1270 path = line.split('=')[1].strip()
1271 relpath = os.path.join(root_dir, path)
1272 if not os.path.isdir(relpath):
1274 logging.debug("Found subproject at %s" % path)
1275 libraries.append(path)
1279 def ant_subprojects(root_dir):
1280 subprojects = get_library_references(root_dir)
1281 for subpath in subprojects:
1282 subrelpath = os.path.join(root_dir, subpath)
1283 for p in get_library_references(subrelpath):
1284 relp = os.path.normpath(os.path.join(subpath, p))
1285 if relp not in subprojects:
1286 subprojects.insert(0, relp)
1290 def remove_debuggable_flags(root_dir):
1291 # Remove forced debuggable flags
1292 logging.debug("Removing debuggable flags from %s" % root_dir)
1293 for root, dirs, files in os.walk(root_dir):
1294 if 'AndroidManifest.xml' in files and os.path.isfile(os.path.join(root, 'AndroidManifest.xml')):
1295 regsub_file(r'android:debuggable="[^"]*"',
1297 os.path.join(root, 'AndroidManifest.xml'))
1300 vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search
1301 vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1302 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1305 def app_matches_packagename(app, package):
1308 appid = app.UpdateCheckName or app.id
1309 if appid is None or appid == "Ignore":
1311 return appid == package
1314 def parse_androidmanifests(paths, app):
1316 Extract some information from the AndroidManifest.xml at the given path.
1317 Returns (version, vercode, package), any or all of which might be None.
1318 All values returned are strings.
1321 ignoreversions = app.UpdateCheckIgnore
1322 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1325 return (None, None, None)
1333 if not os.path.isfile(path):
1336 logging.debug(_("Parsing manifest at '{path}'").format(path=path))
1342 if app.builds and 'gradle' in app.builds[-1] and app.builds[-1].gradle:
1343 flavour = app.builds[-1].gradle[-1]
1345 if has_extension(path, 'gradle'):
1346 with open(path, 'r') as f:
1347 inside_flavour_group = 0
1348 inside_required_flavour = 0
1350 if gradle_comment.match(line):
1353 if inside_flavour_group > 0:
1354 if inside_required_flavour > 0:
1355 matches = psearch_g(line)
1357 s = matches.group(2)
1358 if app_matches_packagename(app, s):
1361 matches = vnsearch_g(line)
1363 version = matches.group(2)
1365 matches = vcsearch_g(line)
1367 vercode = matches.group(1)
1370 inside_required_flavour += 1
1372 inside_required_flavour -= 1
1374 if flavour and (flavour in line):
1375 inside_required_flavour = 1
1378 inside_flavour_group += 1
1380 inside_flavour_group -= 1
1382 if "productFlavors" in line:
1383 inside_flavour_group = 1
1385 matches = psearch_g(line)
1387 s = matches.group(2)
1388 if app_matches_packagename(app, s):
1391 matches = vnsearch_g(line)
1393 version = matches.group(2)
1395 matches = vcsearch_g(line)
1397 vercode = matches.group(1)
1400 xml = parse_xml(path)
1401 if "package" in xml.attrib:
1402 s = xml.attrib["package"]
1403 if app_matches_packagename(app, s):
1405 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1406 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1407 base_dir = os.path.dirname(path)
1408 version = retrieve_string_singleline(base_dir, version)
1409 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1410 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1411 if string_is_integer(a):
1414 logging.warning(_("Problem with xml at '{path}'").format(path=path))
1416 # Remember package name, may be defined separately from version+vercode
1418 package = max_package
1420 logging.debug("..got package={0}, version={1}, vercode={2}"
1421 .format(package, version, vercode))
1423 # Always grab the package name and version name in case they are not
1424 # together with the highest version code
1425 if max_package is None and package is not None:
1426 max_package = package
1427 if max_version is None and version is not None:
1428 max_version = version
1430 if vercode is not None \
1431 and (max_vercode is None or vercode > max_vercode):
1432 if not ignoresearch or not ignoresearch(version):
1433 if version is not None:
1434 max_version = version
1435 if vercode is not None:
1436 max_vercode = vercode
1437 if package is not None:
1438 max_package = package
1440 max_version = "Ignore"
1442 if max_version is None:
1443 max_version = "Unknown"
1445 if max_package and not is_valid_package_name(max_package):
1446 raise FDroidException(_("Invalid package name {0}").format(max_package))
1448 return (max_version, max_vercode, max_package)
1451 def is_valid_package_name(name):
1452 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1455 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1456 raw=False, prepare=True, preponly=False, refresh=True,
1458 """Get the specified source library.
1460 Returns the path to it. Normally this is the path to be used when
1461 referencing it, which may be a subdirectory of the actual project. If
1462 you want the base directory of the project, pass 'basepath=True'.
1471 name, ref = spec.split('@')
1473 number, name = name.split(':', 1)
1475 name, subdir = name.split('/', 1)
1477 if name not in fdroidserver.metadata.srclibs:
1478 raise VCSException('srclib ' + name + ' not found.')
1480 srclib = fdroidserver.metadata.srclibs[name]
1482 sdir = os.path.join(srclib_dir, name)
1485 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1486 vcs.srclib = (name, number, sdir)
1488 vcs.gotorevision(ref, refresh)
1495 libdir = os.path.join(sdir, subdir)
1496 elif srclib["Subdir"]:
1497 for subdir in srclib["Subdir"]:
1498 libdir_candidate = os.path.join(sdir, subdir)
1499 if os.path.exists(libdir_candidate):
1500 libdir = libdir_candidate
1506 remove_signing_keys(sdir)
1507 remove_debuggable_flags(sdir)
1511 if srclib["Prepare"]:
1512 cmd = replace_config_vars(srclib["Prepare"], build)
1514 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=libdir)
1515 if p.returncode != 0:
1516 raise BuildException("Error running prepare command for srclib %s"
1522 return (name, number, libdir)
1525 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1528 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1529 """ Prepare the source code for a particular build
1531 :param vcs: the appropriate vcs object for the application
1532 :param app: the application details from the metadata
1533 :param build: the build details from the metadata
1534 :param build_dir: the path to the build directory, usually 'build/app.id'
1535 :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
1536 :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
1538 Returns the (root, srclibpaths) where:
1539 :param root: is the root directory, which may be the same as 'build_dir' or may
1540 be a subdirectory of it.
1541 :param srclibpaths: is information on the srclibs being used
1544 # Optionally, the actual app source can be in a subdirectory
1546 root_dir = os.path.join(build_dir, build.subdir)
1548 root_dir = build_dir
1550 # Get a working copy of the right revision
1551 logging.info("Getting source for revision " + build.commit)
1552 vcs.gotorevision(build.commit, refresh)
1554 # Initialise submodules if required
1555 if build.submodules:
1556 logging.info(_("Initialising submodules"))
1557 vcs.initsubmodules()
1559 # Check that a subdir (if we're using one) exists. This has to happen
1560 # after the checkout, since it might not exist elsewhere
1561 if not os.path.exists(root_dir):
1562 raise BuildException('Missing subdir ' + root_dir)
1564 # Run an init command if one is required
1566 cmd = replace_config_vars(build.init, build)
1567 logging.info("Running 'init' commands in %s" % root_dir)
1569 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir)
1570 if p.returncode != 0:
1571 raise BuildException("Error running init command for %s:%s" %
1572 (app.id, build.versionName), p.output)
1574 # Apply patches if any
1576 logging.info("Applying patches")
1577 for patch in build.patch:
1578 patch = patch.strip()
1579 logging.info("Applying " + patch)
1580 patch_path = os.path.join('metadata', app.id, patch)
1581 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1582 if p.returncode != 0:
1583 raise BuildException("Failed to apply patch %s" % patch_path)
1585 # Get required source libraries
1588 logging.info("Collecting source libraries")
1589 for lib in build.srclibs:
1590 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1591 refresh=refresh, build=build))
1593 for name, number, libpath in srclibpaths:
1594 place_srclib(root_dir, int(number) if number else None, libpath)
1596 basesrclib = vcs.getsrclib()
1597 # If one was used for the main source, add that too.
1599 srclibpaths.append(basesrclib)
1601 # Update the local.properties file
1602 localprops = [os.path.join(build_dir, 'local.properties')]
1604 parts = build.subdir.split(os.sep)
1607 cur = os.path.join(cur, d)
1608 localprops += [os.path.join(cur, 'local.properties')]
1609 for path in localprops:
1611 if os.path.isfile(path):
1612 logging.info("Updating local.properties file at %s" % path)
1613 with open(path, 'r', encoding='iso-8859-1') as f:
1617 logging.info("Creating local.properties file at %s" % path)
1618 # Fix old-fashioned 'sdk-location' by copying
1619 # from sdk.dir, if necessary
1621 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1622 re.S | re.M).group(1)
1623 props += "sdk-location=%s\n" % sdkloc
1625 props += "sdk.dir=%s\n" % config['sdk_path']
1626 props += "sdk-location=%s\n" % config['sdk_path']
1627 ndk_path = build.ndk_path()
1628 # if for any reason the path isn't valid or the directory
1629 # doesn't exist, some versions of Gradle will error with a
1630 # cryptic message (even if the NDK is not even necessary).
1631 # https://gitlab.com/fdroid/fdroidserver/issues/171
1632 if ndk_path and os.path.exists(ndk_path):
1634 props += "ndk.dir=%s\n" % ndk_path
1635 props += "ndk-location=%s\n" % ndk_path
1636 # Add java.encoding if necessary
1638 props += "java.encoding=%s\n" % build.encoding
1639 with open(path, 'w', encoding='iso-8859-1') as f:
1643 if build.build_method() == 'gradle':
1644 flavours = build.gradle
1647 n = build.target.split('-')[1]
1648 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1649 r'compileSdkVersion %s' % n,
1650 os.path.join(root_dir, 'build.gradle'))
1652 # Remove forced debuggable flags
1653 remove_debuggable_flags(root_dir)
1655 # Insert version code and number into the manifest if necessary
1656 if build.forceversion:
1657 logging.info("Changing the version name")
1658 for path in manifest_paths(root_dir, flavours):
1659 if not os.path.isfile(path):
1661 if has_extension(path, 'xml'):
1662 regsub_file(r'android:versionName="[^"]*"',
1663 r'android:versionName="%s"' % build.versionName,
1665 elif has_extension(path, 'gradle'):
1666 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1667 r"""\1versionName '%s'""" % build.versionName,
1670 if build.forcevercode:
1671 logging.info("Changing the version code")
1672 for path in manifest_paths(root_dir, flavours):
1673 if not os.path.isfile(path):
1675 if has_extension(path, 'xml'):
1676 regsub_file(r'android:versionCode="[^"]*"',
1677 r'android:versionCode="%s"' % build.versionCode,
1679 elif has_extension(path, 'gradle'):
1680 regsub_file(r'versionCode[ =]+[0-9]+',
1681 r'versionCode %s' % build.versionCode,
1684 # Delete unwanted files
1686 logging.info(_("Removing specified files"))
1687 for part in getpaths(build_dir, build.rm):
1688 dest = os.path.join(build_dir, part)
1689 logging.info("Removing {0}".format(part))
1690 if os.path.lexists(dest):
1691 # rmtree can only handle directories that are not symlinks, so catch anything else
1692 if not os.path.isdir(dest) or os.path.islink(dest):
1697 logging.info("...but it didn't exist")
1699 remove_signing_keys(build_dir)
1701 # Add required external libraries
1703 logging.info("Collecting prebuilt libraries")
1704 libsdir = os.path.join(root_dir, 'libs')
1705 if not os.path.exists(libsdir):
1707 for lib in build.extlibs:
1709 logging.info("...installing extlib {0}".format(lib))
1710 libf = os.path.basename(lib)
1711 libsrc = os.path.join(extlib_dir, lib)
1712 if not os.path.exists(libsrc):
1713 raise BuildException("Missing extlib file {0}".format(libsrc))
1714 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1716 # Run a pre-build command if one is required
1718 logging.info("Running 'prebuild' commands in %s" % root_dir)
1720 cmd = replace_config_vars(build.prebuild, build)
1722 # Substitute source library paths into prebuild commands
1723 for name, number, libpath in srclibpaths:
1724 libpath = os.path.relpath(libpath, root_dir)
1725 cmd = cmd.replace('$$' + name + '$$', libpath)
1727 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir)
1728 if p.returncode != 0:
1729 raise BuildException("Error running prebuild command for %s:%s" %
1730 (app.id, build.versionName), p.output)
1732 # Generate (or update) the ant build file, build.xml...
1733 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1734 parms = ['android', 'update', 'lib-project']
1735 lparms = ['android', 'update', 'project']
1738 parms += ['-t', build.target]
1739 lparms += ['-t', build.target]
1740 if build.androidupdate:
1741 update_dirs = build.androidupdate
1743 update_dirs = ant_subprojects(root_dir) + ['.']
1745 for d in update_dirs:
1746 subdir = os.path.join(root_dir, d)
1748 logging.debug("Updating main project")
1749 cmd = parms + ['-p', d]
1751 logging.debug("Updating subproject %s" % d)
1752 cmd = lparms + ['-p', d]
1753 p = SdkToolsPopen(cmd, cwd=root_dir)
1754 # Check to see whether an error was returned without a proper exit
1755 # code (this is the case for the 'no target set or target invalid'
1757 if p.returncode != 0 or p.output.startswith("Error: "):
1758 raise BuildException("Failed to update project at %s" % d, p.output)
1759 # Clean update dirs via ant
1761 logging.info("Cleaning subproject %s" % d)
1762 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1764 return (root_dir, srclibpaths)
1767 def getpaths_map(build_dir, globpaths):
1768 """Extend via globbing the paths from a field and return them as a map from original path to resulting paths"""
1772 full_path = os.path.join(build_dir, p)
1773 full_path = os.path.normpath(full_path)
1774 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1776 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1780 def getpaths(build_dir, globpaths):
1781 """Extend via globbing the paths from a field and return them as a set"""
1782 paths_map = getpaths_map(build_dir, globpaths)
1784 for k, v in paths_map.items():
1791 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1794 def check_system_clock(dt_obj, path):
1795 """Check if system clock is updated based on provided date
1797 If an APK has files newer than the system time, suggest updating
1798 the system clock. This is useful for offline systems, used for
1799 signing, which do not have another source of clock sync info. It
1800 has to be more than 24 hours newer because ZIP/APK files do not
1804 checkdt = dt_obj - timedelta(1)
1805 if datetime.today() < checkdt:
1806 logging.warning(_('System clock is older than date in {path}!').format(path=path)
1807 + '\n' + _('Set clock to that time using:') + '\n'
1808 + 'sudo date -s "' + str(dt_obj) + '"')
1812 """permanent store of existing APKs with the date they were added
1814 This is currently the only way to permanently store the "updated"
1819 '''Load filename/date info about previously seen APKs
1821 Since the appid and date strings both will never have spaces,
1822 this is parsed as a list from the end to allow the filename to
1823 have any combo of spaces.
1826 self.path = os.path.join('stats', 'known_apks.txt')
1828 if os.path.isfile(self.path):
1829 with open(self.path, 'r', encoding='utf8') as f:
1831 t = line.rstrip().split(' ')
1833 self.apks[t[0]] = (t[1], None)
1836 date = datetime.strptime(t[-1], '%Y-%m-%d')
1837 filename = line[0:line.rfind(appid) - 1]
1838 self.apks[filename] = (appid, date)
1839 check_system_clock(date, self.path)
1840 self.changed = False
1842 def writeifchanged(self):
1843 if not self.changed:
1846 if not os.path.exists('stats'):
1850 for apk, app in self.apks.items():
1852 line = apk + ' ' + appid
1854 line += ' ' + added.strftime('%Y-%m-%d')
1857 with open(self.path, 'w', encoding='utf8') as f:
1858 for line in sorted(lst, key=natural_key):
1859 f.write(line + '\n')
1861 def recordapk(self, apkName, app, default_date=None):
1863 Record an apk (if it's new, otherwise does nothing)
1864 Returns the date it was added as a datetime instance
1866 if apkName not in self.apks:
1867 if default_date is None:
1868 default_date = datetime.utcnow()
1869 self.apks[apkName] = (app, default_date)
1871 _ignored, added = self.apks[apkName]
1874 def getapp(self, apkname):
1875 """Look up information - given the 'apkname', returns (app id, date added/None).
1877 Or returns None for an unknown apk.
1879 if apkname in self.apks:
1880 return self.apks[apkname]
1883 def getlatest(self, num):
1884 """Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first"""
1886 for apk, app in self.apks.items():
1890 if apps[appid] > added:
1894 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1895 lst = [app for app, _ignored in sortedapps]
1900 def get_file_extension(filename):
1901 """get the normalized file extension, can be blank string but never None"""
1902 if isinstance(filename, bytes):
1903 filename = filename.decode('utf-8')
1904 return os.path.splitext(filename)[1].lower()[1:]
1907 def get_apk_debuggable_aapt(apkfile):
1908 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1910 if p.returncode != 0:
1911 raise FDroidException(_("Failed to get APK manifest information"))
1912 for line in p.output.splitlines():
1913 if 'android:debuggable' in line and not line.endswith('0x0'):
1918 def get_apk_debuggable_androguard(apkfile):
1920 from androguard.core.bytecodes.apk import APK
1922 raise FDroidException("androguard library is not installed and aapt not present")
1924 apkobject = APK(apkfile)
1925 if apkobject.is_valid_APK():
1926 debuggable = apkobject.get_element("application", "debuggable")
1927 if debuggable is not None:
1928 return bool(strtobool(debuggable))
1932 def isApkAndDebuggable(apkfile):
1933 """Returns True if the given file is an APK and is debuggable
1935 :param apkfile: full path to the apk to check"""
1937 if get_file_extension(apkfile) != 'apk':
1940 if SdkToolsPopen(['aapt', 'version'], output=False):
1941 return get_apk_debuggable_aapt(apkfile)
1943 return get_apk_debuggable_androguard(apkfile)
1946 def get_apk_id_aapt(apkfile):
1947 """Extrat identification information from APK using aapt.
1949 :param apkfile: path to an APK file.
1950 :returns: triplet (appid, version code, version name)
1952 r = re.compile("package: name='(?P<appid>.*)' versionCode='(?P<vercode>.*)' versionName='(?P<vername>.*)' platformBuildVersionName='.*'")
1953 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1954 for line in p.output.splitlines():
1957 return m.group('appid'), m.group('vercode'), m.group('vername')
1958 raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'")
1959 .format(apkfilename=apkfile))
1962 def get_minSdkVersion_aapt(apkfile):
1963 """Extract the minimum supported Android SDK from an APK using aapt
1965 :param apkfile: path to an APK file.
1966 :returns: the integer representing the SDK version
1968 r = re.compile(r"^sdkVersion:'([0-9]+)'")
1969 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1970 for line in p.output.splitlines():
1973 return int(m.group(1))
1974 raise FDroidException(_('Reading minSdkVersion failed: "{apkfilename}"')
1975 .format(apkfilename=apkfile))
1980 self.returncode = None
1984 def SdkToolsPopen(commands, cwd=None, output=True):
1986 if cmd not in config:
1987 config[cmd] = find_sdk_tools_cmd(commands[0])
1988 abscmd = config[cmd]
1990 raise FDroidException(_("Could not find '{command}' on your system").format(command=cmd))
1992 test_aapt_version(config['aapt'])
1993 return FDroidPopen([abscmd] + commands[1:],
1994 cwd=cwd, output=output)
1997 def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
1999 Run a command and capture the possibly huge output as bytes.
2001 :param commands: command and argument list like in subprocess.Popen
2002 :param cwd: optionally specifies a working directory
2003 :param envs: a optional dictionary of environment variables and their values
2004 :returns: A PopenResult.
2009 set_FDroidPopen_env()
2011 process_env = env.copy()
2012 if envs is not None and len(envs) > 0:
2013 process_env.update(envs)
2016 cwd = os.path.normpath(cwd)
2017 logging.debug("Directory: %s" % cwd)
2018 logging.debug("> %s" % ' '.join(commands))
2020 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
2021 result = PopenResult()
2024 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
2025 stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
2026 stderr=stderr_param)
2027 except OSError as e:
2028 raise BuildException("OSError while trying to execute " +
2029 ' '.join(commands) + ': ' + str(e))
2031 # TODO are these AsynchronousFileReader threads always exiting?
2032 if not stderr_to_stdout and options.verbose:
2033 stderr_queue = Queue()
2034 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
2036 while not stderr_reader.eof():
2037 while not stderr_queue.empty():
2038 line = stderr_queue.get()
2039 sys.stderr.buffer.write(line)
2044 stdout_queue = Queue()
2045 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
2048 # Check the queue for output (until there is no more to get)
2049 while not stdout_reader.eof():
2050 while not stdout_queue.empty():
2051 line = stdout_queue.get()
2052 if output and options.verbose:
2053 # Output directly to console
2054 sys.stderr.buffer.write(line)
2060 result.returncode = p.wait()
2061 result.output = buf.getvalue()
2063 # make sure all filestreams of the subprocess are closed
2064 for streamvar in ['stdin', 'stdout', 'stderr']:
2065 if hasattr(p, streamvar):
2066 stream = getattr(p, streamvar)
2072 def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2074 Run a command and capture the possibly huge output as a str.
2076 :param commands: command and argument list like in subprocess.Popen
2077 :param cwd: optionally specifies a working directory
2078 :param envs: a optional dictionary of environment variables and their values
2079 :returns: A PopenResult.
2081 result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
2082 result.output = result.output.decode('utf-8', 'ignore')
2086 gradle_comment = re.compile(r'[ ]*//')
2087 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
2088 gradle_line_matches = [
2089 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
2090 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
2091 re.compile(r'.*\.readLine\(.*'),
2095 def remove_signing_keys(build_dir):
2096 for root, dirs, files in os.walk(build_dir):
2097 if 'build.gradle' in files:
2098 path = os.path.join(root, 'build.gradle')
2100 with open(path, "r", encoding='utf8') as o:
2101 lines = o.readlines()
2107 with open(path, "w", encoding='utf8') as o:
2108 while i < len(lines):
2111 while line.endswith('\\\n'):
2112 line = line.rstrip('\\\n') + lines[i]
2115 if gradle_comment.match(line):
2120 opened += line.count('{')
2121 opened -= line.count('}')
2124 if gradle_signing_configs.match(line):
2129 if any(s.match(line) for s in gradle_line_matches):
2137 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
2140 'project.properties',
2142 'default.properties',
2143 'ant.properties', ]:
2144 if propfile in files:
2145 path = os.path.join(root, propfile)
2147 with open(path, "r", encoding='iso-8859-1') as o:
2148 lines = o.readlines()
2152 with open(path, "w", encoding='iso-8859-1') as o:
2154 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
2161 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
2164 def set_FDroidPopen_env(build=None):
2166 set up the environment variables for the build environment
2168 There is only a weak standard, the variables used by gradle, so also set
2169 up the most commonly used environment variables for SDK and NDK. Also, if
2170 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
2172 global env, orig_path
2176 orig_path = env['PATH']
2177 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
2178 env[n] = config['sdk_path']
2179 for k, v in config['java_paths'].items():
2180 env['JAVA%s_HOME' % k] = v
2182 missinglocale = True
2183 for k, v in env.items():
2184 if k == 'LANG' and v != 'C':
2185 missinglocale = False
2187 missinglocale = False
2189 env['LANG'] = 'en_US.UTF-8'
2191 if build is not None:
2192 path = build.ndk_path()
2193 paths = orig_path.split(os.pathsep)
2194 if path not in paths:
2195 paths = [path] + paths
2196 env['PATH'] = os.pathsep.join(paths)
2197 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
2198 env[n] = build.ndk_path()
2201 def replace_build_vars(cmd, build):
2202 cmd = cmd.replace('$$COMMIT$$', build.commit)
2203 cmd = cmd.replace('$$VERSION$$', build.versionName)
2204 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
2208 def replace_config_vars(cmd, build):
2209 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
2210 cmd = cmd.replace('$$NDK$$', build.ndk_path())
2211 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
2212 if build is not None:
2213 cmd = replace_build_vars(cmd, build)
2217 def place_srclib(root_dir, number, libpath):
2220 relpath = os.path.relpath(libpath, root_dir)
2221 proppath = os.path.join(root_dir, 'project.properties')
2224 if os.path.isfile(proppath):
2225 with open(proppath, "r", encoding='iso-8859-1') as o:
2226 lines = o.readlines()
2228 with open(proppath, "w", encoding='iso-8859-1') as o:
2231 if line.startswith('android.library.reference.%d=' % number):
2232 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2237 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2240 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
2243 def signer_fingerprint_short(sig):
2244 """Obtain shortened sha256 signing-key fingerprint for pkcs7 signature.
2246 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2247 for a given pkcs7 signature.
2249 :param sig: Contents of an APK signing certificate.
2250 :returns: shortened signing-key fingerprint.
2252 return signer_fingerprint(sig)[:7]
2255 def signer_fingerprint(sig):
2256 """Obtain sha256 signing-key fingerprint for pkcs7 signature.
2258 Extracts hexadecimal sha256 signing-key fingerprint string
2259 for a given pkcs7 signature.
2261 :param: Contents of an APK signature.
2262 :returns: shortened signature fingerprint.
2264 cert_encoded = get_certificate(sig)
2265 return hashlib.sha256(cert_encoded).hexdigest()
2268 def apk_signer_fingerprint(apk_path):
2269 """Obtain sha256 signing-key fingerprint for APK.
2271 Extracts hexadecimal sha256 signing-key fingerprint string
2274 :param apkpath: path to APK
2275 :returns: signature fingerprint
2278 with zipfile.ZipFile(apk_path, 'r') as apk:
2279 certs = [n for n in apk.namelist() if CERT_PATH_REGEX.match(n)]
2282 logging.error("Found no signing certificates on %s" % apk_path)
2285 logging.error("Found multiple signing certificates on %s" % apk_path)
2288 cert = apk.read(certs[0])
2289 return signer_fingerprint(cert)
2292 def apk_signer_fingerprint_short(apk_path):
2293 """Obtain shortened sha256 signing-key fingerprint for APK.
2295 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2296 for a given pkcs7 APK.
2298 :param apk_path: path to APK
2299 :returns: shortened signing-key fingerprint
2301 return apk_signer_fingerprint(apk_path)[:7]
2304 def metadata_get_sigdir(appid, vercode=None):
2305 """Get signature directory for app"""
2307 return os.path.join('metadata', appid, 'signatures', vercode)
2309 return os.path.join('metadata', appid, 'signatures')
2312 def metadata_find_developer_signature(appid, vercode=None):
2313 """Tires to find the developer signature for given appid.
2315 This picks the first signature file found in metadata an returns its
2318 :returns: sha256 signing key fingerprint of the developer signing key.
2319 None in case no signature can not be found."""
2321 # fetch list of dirs for all versions of signatures
2324 appversigdirs.append(metadata_get_sigdir(appid, vercode))
2326 appsigdir = metadata_get_sigdir(appid)
2327 if os.path.isdir(appsigdir):
2328 numre = re.compile('[0-9]+')
2329 for ver in os.listdir(appsigdir):
2330 if numre.match(ver):
2331 appversigdir = os.path.join(appsigdir, ver)
2332 appversigdirs.append(appversigdir)
2334 for sigdir in appversigdirs:
2335 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2336 glob.glob(os.path.join(sigdir, '*.EC')) + \
2337 glob.glob(os.path.join(sigdir, '*.RSA'))
2339 raise FDroidException('ambiguous signatures, please make sure there is only one signature in \'{}\'. (The signature has to be the App maintainers signature for version of the APK.)'.format(sigdir))
2341 with open(sig, 'rb') as f:
2342 return signer_fingerprint(f.read())
2346 def metadata_find_signing_files(appid, vercode):
2347 """Gets a list of singed manifests and signatures.
2349 :param appid: app id string
2350 :param vercode: app version code
2351 :returns: a list of triplets for each signing key with following paths:
2352 (signature_file, singed_file, manifest_file)
2355 sigdir = metadata_get_sigdir(appid, vercode)
2356 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2357 glob.glob(os.path.join(sigdir, '*.EC')) + \
2358 glob.glob(os.path.join(sigdir, '*.RSA'))
2359 extre = re.compile('(\.DSA|\.EC|\.RSA)$')
2361 sf = extre.sub('.SF', sig)
2362 if os.path.isfile(sf):
2363 mf = os.path.join(sigdir, 'MANIFEST.MF')
2364 if os.path.isfile(mf):
2365 ret.append((sig, sf, mf))
2369 def metadata_find_developer_signing_files(appid, vercode):
2370 """Get developer signature files for specified app from metadata.
2372 :returns: A triplet of paths for signing files from metadata:
2373 (signature_file, singed_file, manifest_file)
2375 allsigningfiles = metadata_find_signing_files(appid, vercode)
2376 if allsigningfiles and len(allsigningfiles) == 1:
2377 return allsigningfiles[0]
2382 def apk_strip_signatures(signed_apk, strip_manifest=False):
2383 """Removes signatures from APK.
2385 :param signed_apk: path to apk file.
2386 :param strip_manifest: when set to True also the manifest file will
2387 be removed from the APK.
2389 with tempfile.TemporaryDirectory() as tmpdir:
2390 tmp_apk = os.path.join(tmpdir, 'tmp.apk')
2391 shutil.move(signed_apk, tmp_apk)
2392 with ZipFile(tmp_apk, 'r') as in_apk:
2393 with ZipFile(signed_apk, 'w') as out_apk:
2394 for info in in_apk.infolist():
2395 if not apk_sigfile.match(info.filename):
2397 if info.filename != 'META-INF/MANIFEST.MF':
2398 buf = in_apk.read(info.filename)
2399 out_apk.writestr(info, buf)
2401 buf = in_apk.read(info.filename)
2402 out_apk.writestr(info, buf)
2405 def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest):
2406 """Implats a signature from metadata into an APK.
2408 Note: this changes there supplied APK in place. So copy it if you
2409 need the original to be preserved.
2411 :param apkpath: location of the apk
2413 # get list of available signature files in metadata
2414 with tempfile.TemporaryDirectory() as tmpdir:
2415 apkwithnewsig = os.path.join(tmpdir, 'newsig.apk')
2416 with ZipFile(apkpath, 'r') as in_apk:
2417 with ZipFile(apkwithnewsig, 'w') as out_apk:
2418 for sig_file in [signaturefile, signedfile, manifest]:
2419 with open(sig_file, 'rb') as fp:
2421 info = zipfile.ZipInfo('META-INF/' + os.path.basename(sig_file))
2422 info.compress_type = zipfile.ZIP_DEFLATED
2423 info.create_system = 0 # "Windows" aka "FAT", what Android SDK uses
2424 out_apk.writestr(info, buf)
2425 for info in in_apk.infolist():
2426 if not apk_sigfile.match(info.filename):
2427 if info.filename != 'META-INF/MANIFEST.MF':
2428 buf = in_apk.read(info.filename)
2429 out_apk.writestr(info, buf)
2431 p = SdkToolsPopen(['zipalign', '-v', '4', apkwithnewsig, apkpath])
2432 if p.returncode != 0:
2433 raise BuildException("Failed to align application")
2436 def apk_extract_signatures(apkpath, outdir, manifest=True):
2437 """Extracts a signature files from APK and puts them into target directory.
2439 :param apkpath: location of the apk
2440 :param outdir: folder where the extracted signature files will be stored
2441 :param manifest: (optionally) disable extracting manifest file
2443 with ZipFile(apkpath, 'r') as in_apk:
2444 for f in in_apk.infolist():
2445 if apk_sigfile.match(f.filename) or \
2446 (manifest and f.filename == 'META-INF/MANIFEST.MF'):
2447 newpath = os.path.join(outdir, os.path.basename(f.filename))
2448 with open(newpath, 'wb') as out_file:
2449 out_file.write(in_apk.read(f.filename))
2452 def sign_apk(unsigned_path, signed_path, keyalias):
2453 """Sign and zipalign an unsigned APK, then save to a new file, deleting the unsigned
2455 android-18 (4.3) finally added support for reasonable hash
2456 algorithms, like SHA-256, before then, the only options were MD5
2457 and SHA1 :-/ This aims to use SHA-256 when the APK does not target
2458 older Android versions, and is therefore safe to do so.
2460 https://issuetracker.google.com/issues/36956587
2461 https://android-review.googlesource.com/c/platform/libcore/+/44491
2465 if get_minSdkVersion_aapt(unsigned_path) < 18:
2466 signature_algorithm = ['-sigalg', 'SHA1withRSA', '-digestalg', 'SHA1']
2468 signature_algorithm = ['-sigalg', 'SHA256withRSA', '-digestalg', 'SHA-256']
2470 p = FDroidPopen([config['jarsigner'], '-keystore', config['keystore'],
2471 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2472 '-keypass:env', 'FDROID_KEY_PASS']
2473 + signature_algorithm + [unsigned_path, keyalias],
2475 'FDROID_KEY_STORE_PASS': config['keystorepass'],
2476 'FDROID_KEY_PASS': config['keypass'], })
2477 if p.returncode != 0:
2478 raise BuildException(_("Failed to sign application"), p.output)
2480 p = SdkToolsPopen(['zipalign', '-v', '4', unsigned_path, signed_path])
2481 if p.returncode != 0:
2482 raise BuildException(_("Failed to zipalign application"))
2483 os.remove(unsigned_path)
2486 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
2487 """Verify that two apks are the same
2489 One of the inputs is signed, the other is unsigned. The signature metadata
2490 is transferred from the signed to the unsigned apk, and then jarsigner is
2491 used to verify that the signature from the signed apk is also varlid for
2492 the unsigned one. If the APK given as unsigned actually does have a
2493 signature, it will be stripped out and ignored.
2495 There are two SHA1 git commit IDs that fdroidserver includes in the builds
2496 it makes: fdroidserverid and buildserverid. Originally, these were inserted
2497 into AndroidManifest.xml, but that makes the build not reproducible. So
2498 instead they are included as separate files in the APK's META-INF/ folder.
2499 If those files exist in the signed APK, they will be part of the signature
2500 and need to also be included in the unsigned APK for it to validate.
2502 :param signed_apk: Path to a signed apk file
2503 :param unsigned_apk: Path to an unsigned apk file expected to match it
2504 :param tmp_dir: Path to directory for temporary files
2505 :returns: None if the verification is successful, otherwise a string
2506 describing what went wrong.
2509 if not os.path.isfile(signed_apk):
2510 return 'can not verify: file does not exists: {}'.format(signed_apk)
2512 if not os.path.isfile(unsigned_apk):
2513 return 'can not verify: file does not exists: {}'.format(unsigned_apk)
2515 with ZipFile(signed_apk, 'r') as signed:
2516 meta_inf_files = ['META-INF/MANIFEST.MF']
2517 for f in signed.namelist():
2518 if apk_sigfile.match(f) \
2519 or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
2520 meta_inf_files.append(f)
2521 if len(meta_inf_files) < 3:
2522 return "Signature files missing from {0}".format(signed_apk)
2524 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2525 with ZipFile(unsigned_apk, 'r') as unsigned:
2526 # only read the signature from the signed APK, everything else from unsigned
2527 with ZipFile(tmp_apk, 'w') as tmp:
2528 for filename in meta_inf_files:
2529 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2530 for info in unsigned.infolist():
2531 if info.filename in meta_inf_files:
2532 logging.warning('Ignoring %s from %s',
2533 info.filename, unsigned_apk)
2535 if info.filename in tmp.namelist():
2536 return "duplicate filename found: " + info.filename
2537 tmp.writestr(info, unsigned.read(info.filename))
2539 verified = verify_apk_signature(tmp_apk)
2542 logging.info("...NOT verified - {0}".format(tmp_apk))
2543 return compare_apks(signed_apk, tmp_apk, tmp_dir,
2544 os.path.dirname(unsigned_apk))
2546 logging.info("...successfully verified")
2550 def verify_jar_signature(jar):
2551 """Verifies the signature of a given JAR file.
2553 jarsigner is very shitty: unsigned JARs pass as "verified"! So
2554 this has to turn on -strict then check for result 4, since this
2555 does not expect the signature to be from a CA-signed certificate.
2557 :raises: VerificationException() if the JAR's signature could not be verified
2561 error = _('JAR signature failed to verify: {path}').format(path=jar)
2563 output = subprocess.check_output([config['jarsigner'], '-strict', '-verify', jar],
2564 stderr=subprocess.STDOUT)
2565 raise VerificationException(error + '\n' + output.decode('utf-8'))
2566 except subprocess.CalledProcessError as e:
2567 if e.returncode == 4:
2568 logging.debug(_('JAR signature verified: {path}').format(path=jar))
2570 raise VerificationException(error + '\n' + e.output.decode('utf-8'))
2573 def verify_apk_signature(apk, min_sdk_version=None):
2574 """verify the signature on an APK
2576 Try to use apksigner whenever possible since jarsigner is very
2577 shitty: unsigned APKs pass as "verified"! Warning, this does
2578 not work on JARs with apksigner >= 0.7 (build-tools 26.0.1)
2580 :returns: boolean whether the APK was verified
2582 if set_command_in_config('apksigner'):
2583 args = [config['apksigner'], 'verify']
2585 args += ['--min-sdk-version=' + min_sdk_version]
2587 args += ['--verbose']
2589 output = subprocess.check_output(args + [apk])
2591 logging.debug(apk + ': ' + output.decode('utf-8'))
2593 except subprocess.CalledProcessError as e:
2594 logging.error('\n' + apk + ': ' + e.output.decode('utf-8'))
2596 if not config.get('jarsigner_warning_displayed'):
2597 config['jarsigner_warning_displayed'] = True
2598 logging.warning(_("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner"))
2600 verify_jar_signature(apk)
2602 except Exception as e:
2607 def verify_old_apk_signature(apk):
2608 """verify the signature on an archived APK, supporting deprecated algorithms
2610 F-Droid aims to keep every single binary that it ever published. Therefore,
2611 it needs to be able to verify APK signatures that include deprecated/removed
2612 algorithms. For example, jarsigner treats an MD5 signature as unsigned.
2614 jarsigner passes unsigned APKs as "verified"! So this has to turn
2615 on -strict then check for result 4.
2617 :returns: boolean whether the APK was verified
2620 _java_security = os.path.join(os.getcwd(), '.java.security')
2621 with open(_java_security, 'w') as fp:
2622 fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
2626 config['jarsigner'],
2627 '-J-Djava.security.properties=' + _java_security,
2628 '-strict', '-verify', apk
2630 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
2631 except subprocess.CalledProcessError as e:
2632 if e.returncode != 4:
2635 logging.debug(_('JAR signature verified: {path}').format(path=apk))
2638 logging.error(_('Old APK signature failed to verify: {path}').format(path=apk)
2639 + '\n' + output.decode('utf-8'))
2643 apk_badchars = re.compile('''[/ :;'"]''')
2646 def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
2649 Returns None if the apk content is the same (apart from the signing key),
2650 otherwise a string describing what's different, or what went wrong when
2651 trying to do the comparison.
2657 absapk1 = os.path.abspath(apk1)
2658 absapk2 = os.path.abspath(apk2)
2660 if set_command_in_config('diffoscope'):
2661 logfilename = os.path.join(log_dir, os.path.basename(absapk1))
2662 htmlfile = logfilename + '.diffoscope.html'
2663 textfile = logfilename + '.diffoscope.txt'
2664 if subprocess.call([config['diffoscope'],
2665 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2666 '--html', htmlfile, '--text', textfile,
2667 absapk1, absapk2]) != 0:
2668 return("Failed to unpack " + apk1)
2670 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2671 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2672 for d in [apk1dir, apk2dir]:
2673 if os.path.exists(d):
2676 os.mkdir(os.path.join(d, 'jar-xf'))
2678 if subprocess.call(['jar', 'xf',
2679 os.path.abspath(apk1)],
2680 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2681 return("Failed to unpack " + apk1)
2682 if subprocess.call(['jar', 'xf',
2683 os.path.abspath(apk2)],
2684 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2685 return("Failed to unpack " + apk2)
2687 if set_command_in_config('apktool'):
2688 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2690 return("Failed to unpack " + apk1)
2691 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2693 return("Failed to unpack " + apk2)
2695 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2696 lines = p.output.splitlines()
2697 if len(lines) != 1 or 'META-INF' not in lines[0]:
2698 if set_command_in_config('meld'):
2699 p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
2700 return("Unexpected diff output - " + p.output)
2702 # since everything verifies, delete the comparison to keep cruft down
2703 shutil.rmtree(apk1dir)
2704 shutil.rmtree(apk2dir)
2706 # If we get here, it seems like they're the same!
2710 def set_command_in_config(command):
2711 '''Try to find specified command in the path, if it hasn't been
2712 manually set in config.py. If found, it is added to the config
2713 dict. The return value says whether the command is available.
2716 if command in config:
2719 tmp = find_command(command)
2721 config[command] = tmp
2726 def find_command(command):
2727 '''find the full path of a command, or None if it can't be found in the PATH'''
2730 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2732 fpath, fname = os.path.split(command)
2737 for path in os.environ["PATH"].split(os.pathsep):
2738 path = path.strip('"')
2739 exe_file = os.path.join(path, command)
2740 if is_exe(exe_file):
2747 '''generate a random password for when generating keys'''
2748 h = hashlib.sha256()
2749 h.update(os.urandom(16)) # salt
2750 h.update(socket.getfqdn().encode('utf-8'))
2751 passwd = base64.b64encode(h.digest()).strip()
2752 return passwd.decode('utf-8')
2755 def genkeystore(localconfig):
2757 Generate a new key with password provided in :param localconfig and add it to new keystore
2758 :return: hexed public key, public key fingerprint
2760 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2761 keystoredir = os.path.dirname(localconfig['keystore'])
2762 if keystoredir is None or keystoredir == '':
2763 keystoredir = os.path.join(os.getcwd(), keystoredir)
2764 if not os.path.exists(keystoredir):
2765 os.makedirs(keystoredir, mode=0o700)
2768 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
2769 'FDROID_KEY_PASS': localconfig['keypass'],
2771 p = FDroidPopen([config['keytool'], '-genkey',
2772 '-keystore', localconfig['keystore'],
2773 '-alias', localconfig['repo_keyalias'],
2774 '-keyalg', 'RSA', '-keysize', '4096',
2775 '-sigalg', 'SHA256withRSA',
2776 '-validity', '10000',
2777 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2778 '-keypass:env', 'FDROID_KEY_PASS',
2779 '-dname', localconfig['keydname']], envs=env_vars)
2780 if p.returncode != 0:
2781 raise BuildException("Failed to generate key", p.output)
2782 os.chmod(localconfig['keystore'], 0o0600)
2783 if not options.quiet:
2784 # now show the lovely key that was just generated
2785 p = FDroidPopen([config['keytool'], '-list', '-v',
2786 '-keystore', localconfig['keystore'],
2787 '-alias', localconfig['repo_keyalias'],
2788 '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
2789 logging.info(p.output.strip() + '\n\n')
2790 # get the public key
2791 p = FDroidPopenBytes([config['keytool'], '-exportcert',
2792 '-keystore', localconfig['keystore'],
2793 '-alias', localconfig['repo_keyalias'],
2794 '-storepass:env', 'FDROID_KEY_STORE_PASS']
2795 + config['smartcardoptions'],
2796 envs=env_vars, output=False, stderr_to_stdout=False)
2797 if p.returncode != 0 or len(p.output) < 20:
2798 raise BuildException("Failed to get public key", p.output)
2800 fingerprint = get_cert_fingerprint(pubkey)
2801 return hexlify(pubkey), fingerprint
2804 def get_cert_fingerprint(pubkey):
2806 Generate a certificate fingerprint the same way keytool does it
2807 (but with slightly different formatting)
2809 digest = hashlib.sha256(pubkey).digest()
2810 ret = [' '.join("%02X" % b for b in bytearray(digest))]
2811 return " ".join(ret)
2814 def get_certificate(certificate_file):
2816 Extracts a certificate from the given file.
2817 :param certificate_file: file bytes (as string) representing the certificate
2818 :return: A binary representation of the certificate's public key, or None in case of error
2820 content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
2821 if content.getComponentByName('contentType') != rfc2315.signedData:
2823 content = decoder.decode(content.getComponentByName('content'),
2824 asn1Spec=rfc2315.SignedData())[0]
2826 certificates = content.getComponentByName('certificates')
2827 cert = certificates[0].getComponentByName('certificate')
2829 logging.error("Certificates not found.")
2831 return encoder.encode(cert)
2834 def load_stats_fdroid_signing_key_fingerprints():
2835 """Load list of signing-key fingerprints stored by fdroid publish from file.
2837 :returns: list of dictionanryies containing the singing-key fingerprints.
2839 jar_file = os.path.join('stats', 'publishsigkeys.jar')
2840 if not os.path.isfile(jar_file):
2842 cmd = [config['jarsigner'], '-strict', '-verify', jar_file]
2843 p = FDroidPopen(cmd, output=False)
2844 if p.returncode != 4:
2845 raise FDroidException("Signature validation of '{}' failed! "
2846 "Please run publish again to rebuild this file.".format(jar_file))
2848 jar_sigkey = apk_signer_fingerprint(jar_file)
2849 repo_key_sig = config.get('repo_key_sha256')
2851 if jar_sigkey != repo_key_sig:
2852 raise FDroidException("Signature key fingerprint of file '{}' does not match repo_key_sha256 in config.py (found fingerprint: '{}')".format(jar_file, jar_sigkey))
2854 logging.warning("repo_key_sha256 not in config.py, setting it to the signature key fingerprint of '{}'".format(jar_file))
2855 config['repo_key_sha256'] = jar_sigkey
2856 write_to_config(config, 'repo_key_sha256')
2858 with zipfile.ZipFile(jar_file, 'r') as f:
2859 return json.loads(str(f.read('publishsigkeys.json'), 'utf-8'))
2862 def write_to_config(thisconfig, key, value=None, config_file=None):
2863 '''write a key/value to the local config.py
2865 NOTE: only supports writing string variables.
2867 :param thisconfig: config dictionary
2868 :param key: variable name in config.py to be overwritten/added
2869 :param value: optional value to be written, instead of fetched
2870 from 'thisconfig' dictionary.
2873 origkey = key + '_orig'
2874 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2875 cfg = config_file if config_file else 'config.py'
2877 # load config file, create one if it doesn't exist
2878 if not os.path.exists(cfg):
2879 open(cfg, 'a').close()
2880 logging.info("Creating empty " + cfg)
2881 with open(cfg, 'r', encoding="utf-8") as f:
2882 lines = f.readlines()
2884 # make sure the file ends with a carraige return
2886 if not lines[-1].endswith('\n'):
2889 # regex for finding and replacing python string variable
2890 # definitions/initializations
2891 pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
2892 repl = key + ' = "' + value + '"'
2893 pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
2894 repl2 = key + " = '" + value + "'"
2896 # If we replaced this line once, we make sure won't be a
2897 # second instance of this line for this key in the document.
2900 with open(cfg, 'w', encoding="utf-8") as f:
2902 if pattern.match(line) or pattern2.match(line):
2904 line = pattern.sub(repl, line)
2905 line = pattern2.sub(repl2, line)
2916 def parse_xml(path):
2917 return XMLElementTree.parse(path).getroot()
2920 def string_is_integer(string):
2928 def local_rsync(options, fromdir, todir):
2929 '''Rsync method for local to local copying of things
2931 This is an rsync wrapper with all the settings for safe use within
2932 the various fdroidserver use cases. This uses stricter rsync
2933 checking on all files since people using offline mode are already
2934 prioritizing security above ease and speed.
2937 rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
2938 '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
2939 if not options.no_checksum:
2940 rsyncargs.append('--checksum')
2942 rsyncargs += ['--verbose']
2944 rsyncargs += ['--quiet']
2945 logging.debug(' '.join(rsyncargs + [fromdir, todir]))
2946 if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
2947 raise FDroidException()
2950 def get_per_app_repos():
2951 '''per-app repos are dirs named with the packageName of a single app'''
2953 # Android packageNames are Java packages, they may contain uppercase or
2954 # lowercase letters ('A' through 'Z'), numbers, and underscores
2955 # ('_'). However, individual package name parts may only start with
2956 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2957 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2960 for root, dirs, files in os.walk(os.getcwd()):
2962 print('checking', root, 'for', d)
2963 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2964 # standard parts of an fdroid repo, so never packageNames
2967 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
2973 def is_repo_file(filename):
2974 '''Whether the file in a repo is a build product to be delivered to users'''
2975 if isinstance(filename, str):
2976 filename = filename.encode('utf-8', errors="surrogateescape")
2977 return os.path.isfile(filename) \
2978 and not filename.endswith(b'.asc') \
2979 and not filename.endswith(b'.sig') \
2980 and os.path.basename(filename) not in [
2982 b'index_unsigned.jar',
2991 def get_examples_dir():
2992 '''Return the dir where the fdroidserver example files are available'''
2994 tmp = os.path.dirname(sys.argv[0])
2995 if os.path.basename(tmp) == 'bin':
2996 egg_links = glob.glob(os.path.join(tmp, '..',
2997 'local/lib/python3.*/site-packages/fdroidserver.egg-link'))
2999 # installed from local git repo
3000 examplesdir = os.path.join(open(egg_links[0]).readline().rstrip(), 'examples')
3003 examplesdir = os.path.dirname(os.path.dirname(__file__)) + '/share/doc/fdroidserver/examples'
3004 if not os.path.exists(examplesdir): # use UNIX layout
3005 examplesdir = os.path.dirname(tmp) + '/share/doc/fdroidserver/examples'
3007 # we're running straight out of the git repo
3008 prefix = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
3009 examplesdir = prefix + '/examples'
3014 def get_wiki_timestamp(timestamp=None):
3015 """Return current time in the standard format for posting to the wiki"""
3017 if timestamp is None:
3018 timestamp = time.gmtime()
3019 return time.strftime("%Y-%m-%d %H:%M:%SZ", timestamp)
3022 def get_android_tools_versions(ndk_path=None):
3023 '''get a list of the versions of all installed Android SDK/NDK components'''
3026 sdk_path = config['sdk_path']
3027 if sdk_path[-1] != '/':
3031 ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT')
3032 if os.path.isfile(ndk_release_txt):
3033 with open(ndk_release_txt, 'r') as fp:
3034 components.append((os.path.basename(ndk_path), fp.read()[:-1]))
3036 pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE)
3037 for root, dirs, files in os.walk(sdk_path):
3038 if 'source.properties' in files:
3039 source_properties = os.path.join(root, 'source.properties')
3040 with open(source_properties, 'r') as fp:
3041 m = pattern.search(fp.read())
3043 components.append((root[len(sdk_path):], m.group(1)))
3048 def get_android_tools_version_log(ndk_path=None):
3049 '''get a list of the versions of all installed Android SDK/NDK components'''
3050 log = '== Installed Android Tools ==\n\n'
3051 components = get_android_tools_versions(ndk_path)
3052 for name, version in sorted(components):
3053 log += '* ' + name + ' (' + version + ')\n'