3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
40 import xml.etree.ElementTree as XMLElementTree
42 from binascii import hexlify
43 from datetime import datetime, timedelta
44 from distutils.version import LooseVersion
45 from queue import Queue
46 from zipfile import ZipFile
48 from pyasn1.codec.der import decoder, encoder
49 from pyasn1_modules import rfc2315
50 from pyasn1.error import PyAsn1Error
52 from distutils.util import strtobool
54 import fdroidserver.metadata
55 from fdroidserver import _
56 from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException,\
57 BuildException, VerificationException
58 from .asynchronousfilereader import AsynchronousFileReader
60 # this is the build-tools version, aapt has a separate version that
61 # has to be manually set in test_aapt_version()
62 MINIMUM_AAPT_VERSION = '26.0.0'
64 # A signature block file with a .DSA, .RSA, or .EC extension
65 CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
66 APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
67 STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
69 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
78 'sdk_path': "$ANDROID_HOME",
82 'r12b': "$ANDROID_NDK",
88 'build_tools': MINIMUM_AAPT_VERSION,
89 'force_build_tools': False,
94 'accepted_formats': ['txt', 'yml'],
95 'sync_from_local_copy_dir': False,
96 'allow_disabled_algorithms': False,
97 'per_app_repos': False,
98 'make_current_version_link': True,
99 'current_version_name_source': 'Name',
100 'update_stats': False,
102 'stats_server': None,
104 'stats_to_carbon': False,
106 'build_server_always': False,
107 'keystore': 'keystore.jks',
108 'smartcardoptions': [],
118 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
119 'repo_name': "My First FDroid Repo Demo",
120 'repo_icon': "fdroid-icon.png",
121 'repo_description': '''
122 This is a repository of apps to be used with FDroid. Applications in this
123 repository are either official binaries built by the original application
124 developers, or are binaries built from source by the admin of f-droid.org
125 using the tools on https://gitlab.com/u/fdroid.
131 def setup_global_opts(parser):
132 try: # the buildserver VM might not have PIL installed
133 from PIL import PngImagePlugin
134 logger = logging.getLogger(PngImagePlugin.__name__)
135 logger.setLevel(logging.INFO) # tame the "STREAM" debug messages
139 parser.add_argument("-v", "--verbose", action="store_true", default=False,
140 help=_("Spew out even more information than normal"))
141 parser.add_argument("-q", "--quiet", action="store_true", default=False,
142 help=_("Restrict output to warnings and errors"))
145 def _add_java_paths_to_config(pathlist, thisconfig):
146 def path_version_key(s):
148 for u in re.split('[^0-9]+', s):
150 versionlist.append(int(u))
155 for d in sorted(pathlist, key=path_version_key):
156 if os.path.islink(d):
158 j = os.path.basename(d)
159 # the last one found will be the canonical one, so order appropriately
161 r'^1\.([6-9])\.0\.jdk$', # OSX
162 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
163 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
164 r'^jdk([6-9])-openjdk$', # Arch
165 r'^java-([6-9])-openjdk$', # Arch
166 r'^java-([6-9])-jdk$', # Arch (oracle)
167 r'^java-1\.([6-9])\.0-.*$', # RedHat
168 r'^java-([6-9])-oracle$', # Debian WebUpd8
169 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
170 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
172 m = re.match(regex, j)
175 for p in [d, os.path.join(d, 'Contents', 'Home')]:
176 if os.path.exists(os.path.join(p, 'bin', 'javac')):
177 thisconfig['java_paths'][m.group(1)] = p
180 def fill_config_defaults(thisconfig):
181 for k, v in default_config.items():
182 if k not in thisconfig:
185 # Expand paths (~users and $vars)
186 def expand_path(path):
190 path = os.path.expanduser(path)
191 path = os.path.expandvars(path)
196 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
201 thisconfig[k + '_orig'] = v
203 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
204 if thisconfig['java_paths'] is None:
205 thisconfig['java_paths'] = dict()
207 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
208 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
209 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
210 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
211 if os.getenv('JAVA_HOME') is not None:
212 pathlist.append(os.getenv('JAVA_HOME'))
213 if os.getenv('PROGRAMFILES') is not None:
214 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
215 _add_java_paths_to_config(pathlist, thisconfig)
217 for java_version in ('7', '8', '9'):
218 if java_version not in thisconfig['java_paths']:
220 java_home = thisconfig['java_paths'][java_version]
221 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
222 if os.path.exists(jarsigner):
223 thisconfig['jarsigner'] = jarsigner
224 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
225 break # Java7 is preferred, so quit if found
227 for k in ['ndk_paths', 'java_paths']:
233 thisconfig[k][k2] = exp
234 thisconfig[k][k2 + '_orig'] = v
237 def regsub_file(pattern, repl, path):
238 with open(path, 'rb') as f:
240 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
241 with open(path, 'wb') as f:
245 def read_config(opts, config_file='config.py'):
246 """Read the repository config
248 The config is read from config_file, which is in the current
249 directory when any of the repo management commands are used. If
250 there is a local metadata file in the git repo, then config.py is
251 not required, just use defaults.
254 global config, options
256 if config is not None:
263 if os.path.isfile(config_file):
264 logging.debug(_("Reading '{config_file}'").format(config_file=config_file))
265 with io.open(config_file, "rb") as f:
266 code = compile(f.read(), config_file, 'exec')
267 exec(code, None, config)
269 logging.warning(_("No 'config.py' found, using defaults."))
271 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
273 if not type(config[k]) in (str, list, tuple):
275 _("'{field}' will be in random order! Use () or [] brackets if order is important!")
278 # smartcardoptions must be a list since its command line args for Popen
279 if 'smartcardoptions' in config:
280 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
281 elif 'keystore' in config and config['keystore'] == 'NONE':
282 # keystore='NONE' means use smartcard, these are required defaults
283 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
284 'SunPKCS11-OpenSC', '-providerClass',
285 'sun.security.pkcs11.SunPKCS11',
286 '-providerArg', 'opensc-fdroid.cfg']
288 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
289 st = os.stat(config_file)
290 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
291 logging.warning(_("unsafe permissions on '{config_file}' (should be 0600)!")
292 .format(config_file=config_file))
294 fill_config_defaults(config)
296 for k in ["repo_description", "archive_description"]:
298 config[k] = clean_description(config[k])
300 if 'serverwebroot' in config:
301 if isinstance(config['serverwebroot'], str):
302 roots = [config['serverwebroot']]
303 elif all(isinstance(item, str) for item in config['serverwebroot']):
304 roots = config['serverwebroot']
306 raise TypeError(_('only accepts strings, lists, and tuples'))
308 for rootstr in roots:
309 # since this is used with rsync, where trailing slashes have
310 # meaning, ensure there is always a trailing slash
311 if rootstr[-1] != '/':
313 rootlist.append(rootstr.replace('//', '/'))
314 config['serverwebroot'] = rootlist
316 if 'servergitmirrors' in config:
317 if isinstance(config['servergitmirrors'], str):
318 roots = [config['servergitmirrors']]
319 elif all(isinstance(item, str) for item in config['servergitmirrors']):
320 roots = config['servergitmirrors']
322 raise TypeError(_('only accepts strings, lists, and tuples'))
323 config['servergitmirrors'] = roots
328 def assert_config_keystore(config):
329 """Check weather keystore is configured correctly and raise exception if not."""
332 if 'repo_keyalias' not in config:
334 logging.critical(_("'repo_keyalias' not found in config.py!"))
335 if 'keystore' not in config:
337 logging.critical(_("'keystore' not found in config.py!"))
338 elif not os.path.exists(config['keystore']):
340 logging.critical("'" + config['keystore'] + "' does not exist!")
341 if 'keystorepass' not in config:
343 logging.critical(_("'keystorepass' not found in config.py!"))
344 if 'keypass' not in config:
346 logging.critical(_("'keypass' not found in config.py!"))
348 raise FDroidException("This command requires a signing key, " +
349 "you can create one using: fdroid update --create-key")
352 def find_sdk_tools_cmd(cmd):
353 '''find a working path to a tool from the Android SDK'''
356 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
357 # try to find a working path to this command, in all the recent possible paths
358 if 'build_tools' in config:
359 build_tools = os.path.join(config['sdk_path'], 'build-tools')
360 # if 'build_tools' was manually set and exists, check only that one
361 configed_build_tools = os.path.join(build_tools, config['build_tools'])
362 if os.path.exists(configed_build_tools):
363 tooldirs.append(configed_build_tools)
365 # no configed version, so hunt known paths for it
366 for f in sorted(os.listdir(build_tools), reverse=True):
367 if os.path.isdir(os.path.join(build_tools, f)):
368 tooldirs.append(os.path.join(build_tools, f))
369 tooldirs.append(build_tools)
370 sdk_tools = os.path.join(config['sdk_path'], 'tools')
371 if os.path.exists(sdk_tools):
372 tooldirs.append(sdk_tools)
373 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
374 if os.path.exists(sdk_platform_tools):
375 tooldirs.append(sdk_platform_tools)
376 tooldirs.append('/usr/bin')
378 path = os.path.join(d, cmd)
379 if os.path.isfile(path):
381 test_aapt_version(path)
383 # did not find the command, exit with error message
384 ensure_build_tools_exists(config)
387 def test_aapt_version(aapt):
388 '''Check whether the version of aapt is new enough'''
389 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
390 if output is None or output == '':
391 logging.error(_("'{path}' failed to execute!").format(path=aapt))
393 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
398 # the Debian package has the version string like "v0.2-23.0.2"
401 if LooseVersion(bugfix) < LooseVersion(MINIMUM_AAPT_VERSION):
403 elif LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.4062713'):
406 logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-{version} or newer!")
407 .format(aapt=aapt, version=MINIMUM_AAPT_VERSION))
409 logging.warning(_('Unknown version of aapt, might cause problems: ') + output)
412 def test_sdk_exists(thisconfig):
413 if 'sdk_path' not in thisconfig:
414 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
415 test_aapt_version(thisconfig['aapt'])
418 logging.error(_("'sdk_path' not set in 'config.py'!"))
420 if thisconfig['sdk_path'] == default_config['sdk_path']:
421 logging.error(_('No Android SDK found!'))
422 logging.error(_('You can use ANDROID_HOME to set the path to your SDK, i.e.:'))
423 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
425 if not os.path.exists(thisconfig['sdk_path']):
426 logging.critical(_("Android SDK path '{path}' does not exist!")
427 .format(path=thisconfig['sdk_path']))
429 if not os.path.isdir(thisconfig['sdk_path']):
430 logging.critical(_("Android SDK path '{path}' is not a directory!")
431 .format(path=thisconfig['sdk_path']))
433 for d in ['build-tools', 'platform-tools', 'tools']:
434 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
435 logging.critical(_("Android SDK '{path}' does not have '{dirname}' installed!")
436 .format(path=thisconfig['sdk_path'], dirname=d))
441 def ensure_build_tools_exists(thisconfig):
442 if not test_sdk_exists(thisconfig):
443 raise FDroidException(_("Android SDK not found!"))
444 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
445 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
446 if not os.path.isdir(versioned_build_tools):
447 raise FDroidException(
448 _("Android build-tools path '{path}' does not exist!")
449 .format(path=versioned_build_tools))
452 def get_local_metadata_files():
453 '''get any metadata files local to an app's source repo
455 This tries to ignore anything that does not count as app metdata,
456 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
459 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
462 def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False):
464 :param appids: arguments in the form of multiple appid:[vc] strings
465 :returns: a dictionary with the set of vercodes specified for each package
468 if not appid_versionCode_pairs:
471 for p in appid_versionCode_pairs:
472 if allow_vercodes and ':' in p:
473 package, vercode = p.split(':')
475 package, vercode = p, None
476 if package not in vercodes:
477 vercodes[package] = [vercode] if vercode else []
479 elif vercode and vercode not in vercodes[package]:
480 vercodes[package] += [vercode] if vercode else []
485 def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
486 """Build a list of App instances for processing
488 On top of what read_pkg_args does, this returns the whole app
489 metadata, but limiting the builds list to the builds matching the
490 appid_versionCode_pairs and vercodes specified. If no appid_versionCode_pairs are specified, then
491 all App and Build instances are returned.
495 vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes)
501 for appid, app in allapps.items():
502 if appid in vercodes:
505 if len(apps) != len(vercodes):
508 logging.critical(_("No such package: %s") % p)
509 raise FDroidException(_("Found invalid appids in arguments"))
511 raise FDroidException(_("No packages specified"))
514 for appid, app in apps.items():
518 app.builds = [b for b in app.builds if b.versionCode in vc]
519 if len(app.builds) != len(vercodes[appid]):
521 allvcs = [b.versionCode for b in app.builds]
522 for v in vercodes[appid]:
524 logging.critical(_("No such versionCode {versionCode} for app {appid}")
525 .format(versionCode=v, appid=appid))
528 raise FDroidException(_("Found invalid versionCodes for some apps"))
533 def get_extension(filename):
534 base, ext = os.path.splitext(filename)
537 return base, ext.lower()[1:]
540 def has_extension(filename, ext):
541 _ignored, f_ext = get_extension(filename)
545 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
548 def clean_description(description):
549 'Remove unneeded newlines and spaces from a block of description text'
551 # this is split up by paragraph to make removing the newlines easier
552 for paragraph in re.split(r'\n\n', description):
553 paragraph = re.sub('\r', '', paragraph)
554 paragraph = re.sub('\n', ' ', paragraph)
555 paragraph = re.sub(' {2,}', ' ', paragraph)
556 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
557 returnstring += paragraph + '\n\n'
558 return returnstring.rstrip('\n')
561 def publishednameinfo(filename):
562 filename = os.path.basename(filename)
563 m = publish_name_regex.match(filename)
565 result = (m.group(1), m.group(2))
566 except AttributeError:
567 raise FDroidException(_("Invalid name for published file: %s") % filename)
571 apk_release_filename = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)\.apk')
572 apk_release_filename_with_sigfp = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)_(?P<sigfp>[0-9a-f]{7})\.apk')
575 def apk_parse_release_filename(apkname):
576 """Parses the name of an APK file according the F-Droids APK naming
577 scheme and returns the tokens.
579 WARNING: Returned values don't necessarily represent the APKs actual
580 properties, the are just paresed from the file name.
582 :returns: A triplet containing (appid, versionCode, signer), where appid
583 should be the package name, versionCode should be the integer
584 represion of the APKs version and signer should be the first 7 hex
585 digists of the sha256 signing key fingerprint which was used to sign
588 m = apk_release_filename_with_sigfp.match(apkname)
590 return m.group('appid'), m.group('vercode'), m.group('sigfp')
591 m = apk_release_filename.match(apkname)
593 return m.group('appid'), m.group('vercode'), None
594 return None, None, None
597 def get_release_filename(app, build):
599 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
601 return "%s_%s.apk" % (app.id, build.versionCode)
604 def get_toolsversion_logname(app, build):
605 return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
608 def getsrcname(app, build):
609 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
621 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
624 def get_build_dir(app):
625 '''get the dir that this app will be built in'''
627 if app.RepoType == 'srclib':
628 return os.path.join('build', 'srclib', app.Repo)
630 return os.path.join('build', app.id)
634 '''checkout code from VCS and return instance of vcs and the build dir'''
635 build_dir = get_build_dir(app)
637 # Set up vcs interface and make sure we have the latest code...
638 logging.debug("Getting {0} vcs interface for {1}"
639 .format(app.RepoType, app.Repo))
640 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
644 vcs = getvcs(app.RepoType, remote, build_dir)
646 return vcs, build_dir
649 def getvcs(vcstype, remote, local):
651 return vcs_git(remote, local)
652 if vcstype == 'git-svn':
653 return vcs_gitsvn(remote, local)
655 return vcs_hg(remote, local)
657 return vcs_bzr(remote, local)
658 if vcstype == 'srclib':
659 if local != os.path.join('build', 'srclib', remote):
660 raise VCSException("Error: srclib paths are hard-coded!")
661 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
663 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
664 raise VCSException("Invalid vcs type " + vcstype)
667 def getsrclibvcs(name):
668 if name not in fdroidserver.metadata.srclibs:
669 raise VCSException("Missing srclib " + name)
670 return fdroidserver.metadata.srclibs[name]['Repo Type']
675 def __init__(self, remote, local):
677 # svn, git-svn and bzr may require auth
679 if self.repotype() in ('git-svn', 'bzr'):
681 if self.repotype == 'git-svn':
682 raise VCSException("Authentication is not supported for git-svn")
683 self.username, remote = remote.split('@')
684 if ':' not in self.username:
685 raise VCSException(_("Password required with username"))
686 self.username, self.password = self.username.split(':')
690 self.clone_failed = False
691 self.refreshed = False
697 def clientversion(self):
698 versionstr = FDroidPopen(self.clientversioncmd()).output
699 return versionstr[0:versionstr.find('\n')]
701 def clientversioncmd(self):
704 def gotorevision(self, rev, refresh=True):
705 """Take the local repository to a clean version of the given
706 revision, which is specificed in the VCS's native
707 format. Beforehand, the repository can be dirty, or even
708 non-existent. If the repository does already exist locally, it
709 will be updated from the origin, but only once in the lifetime
710 of the vcs object. None is acceptable for 'rev' if you know
711 you are cloning a clean copy of the repo - otherwise it must
712 specify a valid revision.
715 if self.clone_failed:
716 raise VCSException(_("Downloading the repository already failed once, not trying again."))
718 # The .fdroidvcs-id file for a repo tells us what VCS type
719 # and remote that directory was created from, allowing us to drop it
720 # automatically if either of those things changes.
721 fdpath = os.path.join(self.local, '..',
722 '.fdroidvcs-' + os.path.basename(self.local))
723 fdpath = os.path.normpath(fdpath)
724 cdata = self.repotype() + ' ' + self.remote
727 if os.path.exists(self.local):
728 if os.path.exists(fdpath):
729 with open(fdpath, 'r') as f:
730 fsdata = f.read().strip()
735 logging.info("Repository details for %s changed - deleting" % (
739 logging.info("Repository details for %s missing - deleting" % (
742 shutil.rmtree(self.local)
746 self.refreshed = True
749 self.gotorevisionx(rev)
750 except FDroidException as e:
753 # If necessary, write the .fdroidvcs file.
754 if writeback and not self.clone_failed:
755 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
756 with open(fdpath, 'w+') as f:
762 def gotorevisionx(self, rev): # pylint: disable=unused-argument
763 """Derived classes need to implement this.
765 It's called once basic checking has been performed.
767 raise VCSException("This VCS type doesn't define gotorevisionx")
769 # Initialise and update submodules
770 def initsubmodules(self):
771 raise VCSException('Submodules not supported for this vcs type')
773 # Get a list of all known tags
775 if not self._gettags:
776 raise VCSException('gettags not supported for this vcs type')
778 for tag in self._gettags():
779 if re.match('[-A-Za-z0-9_. /]+$', tag):
783 def latesttags(self):
784 """Get a list of all the known tags, sorted from newest to oldest"""
785 raise VCSException('latesttags not supported for this vcs type')
788 """Get current commit reference (hash, revision, etc)"""
789 raise VCSException('getref not supported for this vcs type')
792 """Returns the srclib (name, path) used in setting up the current revision, or None."""
801 def clientversioncmd(self):
802 return ['git', '--version']
804 def git(self, args, envs=dict(), cwd=None, output=True):
805 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
807 While fetch/pull/clone respect the command line option flags,
808 it seems that submodule commands do not. They do seem to
809 follow whatever is in env vars, if the version of git is new
810 enough. So we just throw the kitchen sink at it to see what
813 Also, because of CVE-2017-1000117, block all SSH URLs.
816 # supported in git >= 2.3
818 '-c', 'core.askpass=/bin/true',
819 '-c', 'core.sshCommand=/bin/false',
820 '-c', 'url.https://.insteadOf=ssh://',
822 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
823 git_config.append('-c')
824 git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':')
825 git_config.append('-c')
826 git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain)
827 git_config.append('-c')
828 git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain)
830 'GIT_TERMINAL_PROMPT': '0',
831 'GIT_ASKPASS': '/bin/true',
832 'SSH_ASKPASS': '/bin/true',
833 'GIT_SSH': '/bin/false', # for git < 2.3
835 return FDroidPopen(['git', ] + git_config + args,
836 envs=envs, cwd=cwd, output=output)
839 """If the local directory exists, but is somehow not a git repository,
840 git will traverse up the directory tree until it finds one
841 that is (i.e. fdroidserver) and then we'll proceed to destroy
842 it! This is called as a safety check.
846 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
847 result = p.output.rstrip()
848 if not result.endswith(self.local):
849 raise VCSException('Repository mismatch')
851 def gotorevisionx(self, rev):
852 if not os.path.exists(self.local):
854 p = self.git(['clone', '--', self.remote, self.local])
855 if p.returncode != 0:
856 self.clone_failed = True
857 raise VCSException("Git clone failed", p.output)
861 # Discard any working tree changes
862 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
863 'git', 'reset', '--hard'], cwd=self.local, output=False)
864 if p.returncode != 0:
865 raise VCSException(_("Git reset failed"), p.output)
866 # Remove untracked files now, in case they're tracked in the target
867 # revision (it happens!)
868 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
869 'git', 'clean', '-dffx'], cwd=self.local, output=False)
870 if p.returncode != 0:
871 raise VCSException(_("Git clean failed"), p.output)
872 if not self.refreshed:
873 # Get latest commits and tags from remote
874 p = self.git(['fetch', 'origin'], cwd=self.local)
875 if p.returncode != 0:
876 raise VCSException(_("Git fetch failed"), p.output)
877 p = self.git(['fetch', '--prune', '--tags', 'origin'], output=False, cwd=self.local)
878 if p.returncode != 0:
879 raise VCSException(_("Git fetch failed"), p.output)
880 # Recreate origin/HEAD as git clone would do it, in case it disappeared
881 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
882 if p.returncode != 0:
883 lines = p.output.splitlines()
884 if 'Multiple remote HEAD branches' not in lines[0]:
885 raise VCSException(_("Git remote set-head failed"), p.output)
886 branch = lines[1].split(' ')[-1]
887 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--', branch],
888 cwd=self.local, output=False)
889 if p2.returncode != 0:
890 raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output)
891 self.refreshed = True
892 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
893 # a github repo. Most of the time this is the same as origin/master.
894 rev = rev or 'origin/HEAD'
895 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
896 if p.returncode != 0:
897 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
898 # Get rid of any uncontrolled files left behind
899 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
900 if p.returncode != 0:
901 raise VCSException(_("Git clean failed"), p.output)
903 def initsubmodules(self):
905 submfile = os.path.join(self.local, '.gitmodules')
906 if not os.path.isfile(submfile):
907 raise NoSubmodulesException(_("No git submodules available"))
909 # fix submodules not accessible without an account and public key auth
910 with open(submfile, 'r') as f:
911 lines = f.readlines()
912 with open(submfile, 'w') as f:
914 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
915 line = re.sub('git@' + domain + ':', 'https://u:p@' + domain + '/', line)
918 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
919 if p.returncode != 0:
920 raise VCSException(_("Git submodule sync failed"), p.output)
921 p = self.git(['submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
922 if p.returncode != 0:
923 raise VCSException(_("Git submodule update failed"), p.output)
927 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
928 return p.output.splitlines()
930 tag_format = re.compile(r'tag: ([^),]*)')
932 def latesttags(self):
934 p = FDroidPopen(['git', 'log', '--tags',
935 '--simplify-by-decoration', '--pretty=format:%d'],
936 cwd=self.local, output=False)
938 for line in p.output.splitlines():
939 for tag in self.tag_format.findall(line):
944 class vcs_gitsvn(vcs):
949 def clientversioncmd(self):
950 return ['git', 'svn', '--version']
953 """If the local directory exists, but is somehow not a git repository,
954 git will traverse up the directory tree until it finds one that
955 is (i.e. fdroidserver) and then we'll proceed to destory it!
956 This is called as a safety check.
959 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
960 result = p.output.rstrip()
961 if not result.endswith(self.local):
962 raise VCSException('Repository mismatch')
964 def git(self, args, envs=dict(), cwd=None, output=True):
965 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
967 AskPass is set to /bin/true to let the process try to connect
968 without a username/password.
970 The SSH command is set to /bin/false to block all SSH URLs
971 (supported in git >= 2.3). This protects against
976 '-c', 'core.askpass=/bin/true',
977 '-c', 'core.sshCommand=/bin/false',
980 'GIT_TERMINAL_PROMPT': '0',
981 'GIT_ASKPASS': '/bin/true',
982 'SSH_ASKPASS': '/bin/true',
983 'GIT_SSH': '/bin/false', # for git < 2.3
984 'SVN_SSH': '/bin/false',
986 return FDroidPopen(['git', ] + git_config + args,
987 envs=envs, cwd=cwd, output=output)
989 def gotorevisionx(self, rev):
990 if not os.path.exists(self.local):
992 gitsvn_args = ['svn', 'clone']
994 if ';' in self.remote:
995 remote_split = self.remote.split(';')
996 for i in remote_split[1:]:
997 if i.startswith('trunk='):
998 gitsvn_args.extend(['-T', i[6:]])
999 elif i.startswith('tags='):
1000 gitsvn_args.extend(['-t', i[5:]])
1001 elif i.startswith('branches='):
1002 gitsvn_args.extend(['-b', i[9:]])
1003 remote = remote_split[0]
1005 remote = self.remote
1007 gitsvn_args.extend(['--', remote, self.local])
1008 p = self.git(gitsvn_args)
1009 if p.returncode != 0:
1010 self.clone_failed = True
1011 raise VCSException(_('git svn clone failed'), p.output)
1015 # Discard any working tree changes
1016 p = self.git(['reset', '--hard'], cwd=self.local, output=False)
1017 if p.returncode != 0:
1018 raise VCSException("Git reset failed", p.output)
1019 # Remove untracked files now, in case they're tracked in the target
1020 # revision (it happens!)
1021 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1022 if p.returncode != 0:
1023 raise VCSException("Git clean failed", p.output)
1024 if not self.refreshed:
1025 # Get new commits, branches and tags from repo
1026 p = self.git(['svn', 'fetch'], cwd=self.local, output=False)
1027 if p.returncode != 0:
1028 raise VCSException("Git svn fetch failed")
1029 p = self.git(['svn', 'rebase'], cwd=self.local, output=False)
1030 if p.returncode != 0:
1031 raise VCSException("Git svn rebase failed", p.output)
1032 self.refreshed = True
1034 rev = rev or 'master'
1036 nospaces_rev = rev.replace(' ', '%20')
1037 # Try finding a svn tag
1038 for treeish in ['origin/', '']:
1039 p = self.git(['checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
1040 if p.returncode == 0:
1042 if p.returncode != 0:
1043 # No tag found, normal svn rev translation
1044 # Translate svn rev into git format
1045 rev_split = rev.split('/')
1048 for treeish in ['origin/', '']:
1049 if len(rev_split) > 1:
1050 treeish += rev_split[0]
1051 svn_rev = rev_split[1]
1054 # if no branch is specified, then assume trunk (i.e. 'master' branch):
1058 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
1060 p = self.git(['svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
1061 git_rev = p.output.rstrip()
1063 if p.returncode == 0 and git_rev:
1066 if p.returncode != 0 or not git_rev:
1067 # Try a plain git checkout as a last resort
1068 p = self.git(['checkout', rev], cwd=self.local, output=False)
1069 if p.returncode != 0:
1070 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
1072 # Check out the git rev equivalent to the svn rev
1073 p = self.git(['checkout', git_rev], cwd=self.local, output=False)
1074 if p.returncode != 0:
1075 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
1077 # Get rid of any uncontrolled files left behind
1078 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1079 if p.returncode != 0:
1080 raise VCSException(_("Git clean failed"), p.output)
1084 for treeish in ['origin/', '']:
1085 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
1086 if os.path.isdir(d):
1087 return os.listdir(d)
1091 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
1092 if p.returncode != 0:
1094 return p.output.strip()
1102 def clientversioncmd(self):
1103 return ['hg', '--version']
1105 def gotorevisionx(self, rev):
1106 if not os.path.exists(self.local):
1107 p = FDroidPopen(['hg', 'clone', '--ssh', 'false', '--', self.remote, self.local],
1109 if p.returncode != 0:
1110 self.clone_failed = True
1111 raise VCSException("Hg clone failed", p.output)
1113 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
1114 if p.returncode != 0:
1115 raise VCSException("Hg status failed", p.output)
1116 for line in p.output.splitlines():
1117 if not line.startswith('? '):
1118 raise VCSException("Unexpected output from hg status -uS: " + line)
1119 FDroidPopen(['rm', '-rf', '--', line[2:]], cwd=self.local, output=False)
1120 if not self.refreshed:
1121 p = FDroidPopen(['hg', 'pull', '--ssh', 'false'], cwd=self.local, output=False)
1122 if p.returncode != 0:
1123 raise VCSException("Hg pull failed", p.output)
1124 self.refreshed = True
1126 rev = rev or 'default'
1129 p = FDroidPopen(['hg', 'update', '-C', '--', rev], cwd=self.local, output=False)
1130 if p.returncode != 0:
1131 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
1132 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1133 # Also delete untracked files, we have to enable purge extension for that:
1134 if "'purge' is provided by the following extension" in p.output:
1135 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
1136 myfile.write("\n[extensions]\nhgext.purge=\n")
1137 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1138 if p.returncode != 0:
1139 raise VCSException("HG purge failed", p.output)
1140 elif p.returncode != 0:
1141 raise VCSException("HG purge failed", p.output)
1144 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
1145 return p.output.splitlines()[1:]
1153 def clientversioncmd(self):
1154 return ['bzr', '--version']
1156 def bzr(self, args, envs=dict(), cwd=None, output=True):
1157 '''Prevent bzr from ever using SSH to avoid security vulns'''
1161 return FDroidPopen(['bzr', ] + args, envs=envs, cwd=cwd, output=output)
1163 def gotorevisionx(self, rev):
1164 if not os.path.exists(self.local):
1165 p = self.bzr(['branch', self.remote, self.local], output=False)
1166 if p.returncode != 0:
1167 self.clone_failed = True
1168 raise VCSException("Bzr branch failed", p.output)
1170 p = self.bzr(['clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
1171 if p.returncode != 0:
1172 raise VCSException("Bzr revert failed", p.output)
1173 if not self.refreshed:
1174 p = self.bzr(['pull'], cwd=self.local, output=False)
1175 if p.returncode != 0:
1176 raise VCSException("Bzr update failed", p.output)
1177 self.refreshed = True
1179 revargs = list(['-r', rev] if rev else [])
1180 p = self.bzr(['revert'] + revargs, cwd=self.local, output=False)
1181 if p.returncode != 0:
1182 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1185 p = self.bzr(['tags'], cwd=self.local, output=False)
1186 return [tag.split(' ')[0].strip() for tag in
1187 p.output.splitlines()]
1190 def unescape_string(string):
1193 if string[0] == '"' and string[-1] == '"':
1196 return string.replace("\\'", "'")
1199 def retrieve_string(app_dir, string, xmlfiles=None):
1201 if not string.startswith('@string/'):
1202 return unescape_string(string)
1204 if xmlfiles is None:
1207 os.path.join(app_dir, 'res'),
1208 os.path.join(app_dir, 'src', 'main', 'res'),
1210 for root, dirs, files in os.walk(res_dir):
1211 if os.path.basename(root) == 'values':
1212 xmlfiles += [os.path.join(root, x) for x in files if x.endswith('.xml')]
1214 name = string[len('@string/'):]
1216 def element_content(element):
1217 if element.text is None:
1219 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1220 return s.decode('utf-8').strip()
1222 for path in xmlfiles:
1223 if not os.path.isfile(path):
1225 xml = parse_xml(path)
1226 element = xml.find('string[@name="' + name + '"]')
1227 if element is not None:
1228 content = element_content(element)
1229 return retrieve_string(app_dir, content, xmlfiles)
1234 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1235 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1238 def manifest_paths(app_dir, flavours):
1239 '''Return list of existing files that will be used to find the highest vercode'''
1241 possible_manifests = \
1242 [os.path.join(app_dir, 'AndroidManifest.xml'),
1243 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1244 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1245 os.path.join(app_dir, 'build.gradle')]
1247 for flavour in flavours:
1248 if flavour == 'yes':
1250 possible_manifests.append(
1251 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1253 return [path for path in possible_manifests if os.path.isfile(path)]
1256 def fetch_real_name(app_dir, flavours):
1257 '''Retrieve the package name. Returns the name, or None if not found.'''
1258 for path in manifest_paths(app_dir, flavours):
1259 if not has_extension(path, 'xml') or not os.path.isfile(path):
1261 logging.debug("fetch_real_name: Checking manifest at " + path)
1262 xml = parse_xml(path)
1263 app = xml.find('application')
1266 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1268 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1269 result = retrieve_string_singleline(app_dir, label)
1271 result = result.strip()
1276 def get_library_references(root_dir):
1278 proppath = os.path.join(root_dir, 'project.properties')
1279 if not os.path.isfile(proppath):
1281 with open(proppath, 'r', encoding='iso-8859-1') as f:
1283 if not line.startswith('android.library.reference.'):
1285 path = line.split('=')[1].strip()
1286 relpath = os.path.join(root_dir, path)
1287 if not os.path.isdir(relpath):
1289 logging.debug("Found subproject at %s" % path)
1290 libraries.append(path)
1294 def ant_subprojects(root_dir):
1295 subprojects = get_library_references(root_dir)
1296 for subpath in subprojects:
1297 subrelpath = os.path.join(root_dir, subpath)
1298 for p in get_library_references(subrelpath):
1299 relp = os.path.normpath(os.path.join(subpath, p))
1300 if relp not in subprojects:
1301 subprojects.insert(0, relp)
1305 def remove_debuggable_flags(root_dir):
1306 # Remove forced debuggable flags
1307 logging.debug("Removing debuggable flags from %s" % root_dir)
1308 for root, dirs, files in os.walk(root_dir):
1309 if 'AndroidManifest.xml' in files and os.path.isfile(os.path.join(root, 'AndroidManifest.xml')):
1310 regsub_file(r'android:debuggable="[^"]*"',
1312 os.path.join(root, 'AndroidManifest.xml'))
1315 vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search
1316 vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1317 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1320 def app_matches_packagename(app, package):
1323 appid = app.UpdateCheckName or app.id
1324 if appid is None or appid == "Ignore":
1326 return appid == package
1329 def parse_androidmanifests(paths, app):
1331 Extract some information from the AndroidManifest.xml at the given path.
1332 Returns (version, vercode, package), any or all of which might be None.
1333 All values returned are strings.
1336 ignoreversions = app.UpdateCheckIgnore
1337 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1340 return (None, None, None)
1348 if not os.path.isfile(path):
1351 logging.debug(_("Parsing manifest at '{path}'").format(path=path))
1357 if app.builds and 'gradle' in app.builds[-1] and app.builds[-1].gradle:
1358 flavour = app.builds[-1].gradle[-1]
1360 if has_extension(path, 'gradle'):
1361 with open(path, 'r') as f:
1362 inside_flavour_group = 0
1363 inside_required_flavour = 0
1365 if gradle_comment.match(line):
1368 if inside_flavour_group > 0:
1369 if inside_required_flavour > 0:
1370 matches = psearch_g(line)
1372 s = matches.group(2)
1373 if app_matches_packagename(app, s):
1376 matches = vnsearch_g(line)
1378 version = matches.group(2)
1380 matches = vcsearch_g(line)
1382 vercode = matches.group(1)
1385 inside_required_flavour += 1
1387 inside_required_flavour -= 1
1389 if flavour and (flavour in line):
1390 inside_required_flavour = 1
1393 inside_flavour_group += 1
1395 inside_flavour_group -= 1
1397 if "productFlavors" in line:
1398 inside_flavour_group = 1
1400 matches = psearch_g(line)
1402 s = matches.group(2)
1403 if app_matches_packagename(app, s):
1406 matches = vnsearch_g(line)
1408 version = matches.group(2)
1410 matches = vcsearch_g(line)
1412 vercode = matches.group(1)
1415 xml = parse_xml(path)
1416 if "package" in xml.attrib:
1417 s = xml.attrib["package"]
1418 if app_matches_packagename(app, s):
1420 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1421 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1422 base_dir = os.path.dirname(path)
1423 version = retrieve_string_singleline(base_dir, version)
1424 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1425 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1426 if string_is_integer(a):
1429 logging.warning(_("Problem with xml at '{path}'").format(path=path))
1431 # Remember package name, may be defined separately from version+vercode
1433 package = max_package
1435 logging.debug("..got package={0}, version={1}, vercode={2}"
1436 .format(package, version, vercode))
1438 # Always grab the package name and version name in case they are not
1439 # together with the highest version code
1440 if max_package is None and package is not None:
1441 max_package = package
1442 if max_version is None and version is not None:
1443 max_version = version
1445 if vercode is not None \
1446 and (max_vercode is None or vercode > max_vercode):
1447 if not ignoresearch or not ignoresearch(version):
1448 if version is not None:
1449 max_version = version
1450 if vercode is not None:
1451 max_vercode = vercode
1452 if package is not None:
1453 max_package = package
1455 max_version = "Ignore"
1457 if max_version is None:
1458 max_version = "Unknown"
1460 if max_package and not is_valid_package_name(max_package):
1461 raise FDroidException(_("Invalid package name {0}").format(max_package))
1463 return (max_version, max_vercode, max_package)
1466 def is_valid_package_name(name):
1467 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1470 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1471 raw=False, prepare=True, preponly=False, refresh=True,
1473 """Get the specified source library.
1475 Returns the path to it. Normally this is the path to be used when
1476 referencing it, which may be a subdirectory of the actual project. If
1477 you want the base directory of the project, pass 'basepath=True'.
1486 name, ref = spec.split('@')
1488 number, name = name.split(':', 1)
1490 name, subdir = name.split('/', 1)
1492 if name not in fdroidserver.metadata.srclibs:
1493 raise VCSException('srclib ' + name + ' not found.')
1495 srclib = fdroidserver.metadata.srclibs[name]
1497 sdir = os.path.join(srclib_dir, name)
1500 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1501 vcs.srclib = (name, number, sdir)
1503 vcs.gotorevision(ref, refresh)
1510 libdir = os.path.join(sdir, subdir)
1511 elif srclib["Subdir"]:
1512 for subdir in srclib["Subdir"]:
1513 libdir_candidate = os.path.join(sdir, subdir)
1514 if os.path.exists(libdir_candidate):
1515 libdir = libdir_candidate
1521 remove_signing_keys(sdir)
1522 remove_debuggable_flags(sdir)
1526 if srclib["Prepare"]:
1527 cmd = replace_config_vars(srclib["Prepare"], build)
1529 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=libdir)
1530 if p.returncode != 0:
1531 raise BuildException("Error running prepare command for srclib %s"
1537 return (name, number, libdir)
1540 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1543 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1544 """ Prepare the source code for a particular build
1546 :param vcs: the appropriate vcs object for the application
1547 :param app: the application details from the metadata
1548 :param build: the build details from the metadata
1549 :param build_dir: the path to the build directory, usually 'build/app.id'
1550 :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
1551 :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
1553 Returns the (root, srclibpaths) where:
1554 :param root: is the root directory, which may be the same as 'build_dir' or may
1555 be a subdirectory of it.
1556 :param srclibpaths: is information on the srclibs being used
1559 # Optionally, the actual app source can be in a subdirectory
1561 root_dir = os.path.join(build_dir, build.subdir)
1563 root_dir = build_dir
1565 # Get a working copy of the right revision
1566 logging.info("Getting source for revision " + build.commit)
1567 vcs.gotorevision(build.commit, refresh)
1569 # Initialise submodules if required
1570 if build.submodules:
1571 logging.info(_("Initialising submodules"))
1572 vcs.initsubmodules()
1574 # Check that a subdir (if we're using one) exists. This has to happen
1575 # after the checkout, since it might not exist elsewhere
1576 if not os.path.exists(root_dir):
1577 raise BuildException('Missing subdir ' + root_dir)
1579 # Run an init command if one is required
1581 cmd = replace_config_vars(build.init, build)
1582 logging.info("Running 'init' commands in %s" % root_dir)
1584 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir)
1585 if p.returncode != 0:
1586 raise BuildException("Error running init command for %s:%s" %
1587 (app.id, build.versionName), p.output)
1589 # Apply patches if any
1591 logging.info("Applying patches")
1592 for patch in build.patch:
1593 patch = patch.strip()
1594 logging.info("Applying " + patch)
1595 patch_path = os.path.join('metadata', app.id, patch)
1596 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1597 if p.returncode != 0:
1598 raise BuildException("Failed to apply patch %s" % patch_path)
1600 # Get required source libraries
1603 logging.info("Collecting source libraries")
1604 for lib in build.srclibs:
1605 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1606 refresh=refresh, build=build))
1608 for name, number, libpath in srclibpaths:
1609 place_srclib(root_dir, int(number) if number else None, libpath)
1611 basesrclib = vcs.getsrclib()
1612 # If one was used for the main source, add that too.
1614 srclibpaths.append(basesrclib)
1616 # Update the local.properties file
1617 localprops = [os.path.join(build_dir, 'local.properties')]
1619 parts = build.subdir.split(os.sep)
1622 cur = os.path.join(cur, d)
1623 localprops += [os.path.join(cur, 'local.properties')]
1624 for path in localprops:
1626 if os.path.isfile(path):
1627 logging.info("Updating local.properties file at %s" % path)
1628 with open(path, 'r', encoding='iso-8859-1') as f:
1632 logging.info("Creating local.properties file at %s" % path)
1633 # Fix old-fashioned 'sdk-location' by copying
1634 # from sdk.dir, if necessary
1636 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1637 re.S | re.M).group(1)
1638 props += "sdk-location=%s\n" % sdkloc
1640 props += "sdk.dir=%s\n" % config['sdk_path']
1641 props += "sdk-location=%s\n" % config['sdk_path']
1642 ndk_path = build.ndk_path()
1643 # if for any reason the path isn't valid or the directory
1644 # doesn't exist, some versions of Gradle will error with a
1645 # cryptic message (even if the NDK is not even necessary).
1646 # https://gitlab.com/fdroid/fdroidserver/issues/171
1647 if ndk_path and os.path.exists(ndk_path):
1649 props += "ndk.dir=%s\n" % ndk_path
1650 props += "ndk-location=%s\n" % ndk_path
1651 # Add java.encoding if necessary
1653 props += "java.encoding=%s\n" % build.encoding
1654 with open(path, 'w', encoding='iso-8859-1') as f:
1658 if build.build_method() == 'gradle':
1659 flavours = build.gradle
1662 n = build.target.split('-')[1]
1663 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1664 r'compileSdkVersion %s' % n,
1665 os.path.join(root_dir, 'build.gradle'))
1667 # Remove forced debuggable flags
1668 remove_debuggable_flags(root_dir)
1670 # Insert version code and number into the manifest if necessary
1671 if build.forceversion:
1672 logging.info("Changing the version name")
1673 for path in manifest_paths(root_dir, flavours):
1674 if not os.path.isfile(path):
1676 if has_extension(path, 'xml'):
1677 regsub_file(r'android:versionName="[^"]*"',
1678 r'android:versionName="%s"' % build.versionName,
1680 elif has_extension(path, 'gradle'):
1681 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1682 r"""\1versionName '%s'""" % build.versionName,
1685 if build.forcevercode:
1686 logging.info("Changing the version code")
1687 for path in manifest_paths(root_dir, flavours):
1688 if not os.path.isfile(path):
1690 if has_extension(path, 'xml'):
1691 regsub_file(r'android:versionCode="[^"]*"',
1692 r'android:versionCode="%s"' % build.versionCode,
1694 elif has_extension(path, 'gradle'):
1695 regsub_file(r'versionCode[ =]+[0-9]+',
1696 r'versionCode %s' % build.versionCode,
1699 # Delete unwanted files
1701 logging.info(_("Removing specified files"))
1702 for part in getpaths(build_dir, build.rm):
1703 dest = os.path.join(build_dir, part)
1704 logging.info("Removing {0}".format(part))
1705 if os.path.lexists(dest):
1706 # rmtree can only handle directories that are not symlinks, so catch anything else
1707 if not os.path.isdir(dest) or os.path.islink(dest):
1712 logging.info("...but it didn't exist")
1714 remove_signing_keys(build_dir)
1716 # Add required external libraries
1718 logging.info("Collecting prebuilt libraries")
1719 libsdir = os.path.join(root_dir, 'libs')
1720 if not os.path.exists(libsdir):
1722 for lib in build.extlibs:
1724 logging.info("...installing extlib {0}".format(lib))
1725 libf = os.path.basename(lib)
1726 libsrc = os.path.join(extlib_dir, lib)
1727 if not os.path.exists(libsrc):
1728 raise BuildException("Missing extlib file {0}".format(libsrc))
1729 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1731 # Run a pre-build command if one is required
1733 logging.info("Running 'prebuild' commands in %s" % root_dir)
1735 cmd = replace_config_vars(build.prebuild, build)
1737 # Substitute source library paths into prebuild commands
1738 for name, number, libpath in srclibpaths:
1739 libpath = os.path.relpath(libpath, root_dir)
1740 cmd = cmd.replace('$$' + name + '$$', libpath)
1742 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir)
1743 if p.returncode != 0:
1744 raise BuildException("Error running prebuild command for %s:%s" %
1745 (app.id, build.versionName), p.output)
1747 # Generate (or update) the ant build file, build.xml...
1748 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1749 parms = ['android', 'update', 'lib-project']
1750 lparms = ['android', 'update', 'project']
1753 parms += ['-t', build.target]
1754 lparms += ['-t', build.target]
1755 if build.androidupdate:
1756 update_dirs = build.androidupdate
1758 update_dirs = ant_subprojects(root_dir) + ['.']
1760 for d in update_dirs:
1761 subdir = os.path.join(root_dir, d)
1763 logging.debug("Updating main project")
1764 cmd = parms + ['-p', d]
1766 logging.debug("Updating subproject %s" % d)
1767 cmd = lparms + ['-p', d]
1768 p = SdkToolsPopen(cmd, cwd=root_dir)
1769 # Check to see whether an error was returned without a proper exit
1770 # code (this is the case for the 'no target set or target invalid'
1772 if p.returncode != 0 or p.output.startswith("Error: "):
1773 raise BuildException("Failed to update project at %s" % d, p.output)
1774 # Clean update dirs via ant
1776 logging.info("Cleaning subproject %s" % d)
1777 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1779 return (root_dir, srclibpaths)
1782 def getpaths_map(build_dir, globpaths):
1783 """Extend via globbing the paths from a field and return them as a map from original path to resulting paths"""
1787 full_path = os.path.join(build_dir, p)
1788 full_path = os.path.normpath(full_path)
1789 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1791 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1795 def getpaths(build_dir, globpaths):
1796 """Extend via globbing the paths from a field and return them as a set"""
1797 paths_map = getpaths_map(build_dir, globpaths)
1799 for k, v in paths_map.items():
1806 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1809 def check_system_clock(dt_obj, path):
1810 """Check if system clock is updated based on provided date
1812 If an APK has files newer than the system time, suggest updating
1813 the system clock. This is useful for offline systems, used for
1814 signing, which do not have another source of clock sync info. It
1815 has to be more than 24 hours newer because ZIP/APK files do not
1819 checkdt = dt_obj - timedelta(1)
1820 if datetime.today() < checkdt:
1821 logging.warning(_('System clock is older than date in {path}!').format(path=path)
1822 + '\n' + _('Set clock to that time using:') + '\n'
1823 + 'sudo date -s "' + str(dt_obj) + '"')
1827 """permanent store of existing APKs with the date they were added
1829 This is currently the only way to permanently store the "updated"
1834 '''Load filename/date info about previously seen APKs
1836 Since the appid and date strings both will never have spaces,
1837 this is parsed as a list from the end to allow the filename to
1838 have any combo of spaces.
1841 self.path = os.path.join('stats', 'known_apks.txt')
1843 if os.path.isfile(self.path):
1844 with open(self.path, 'r', encoding='utf8') as f:
1846 t = line.rstrip().split(' ')
1848 self.apks[t[0]] = (t[1], None)
1851 date = datetime.strptime(t[-1], '%Y-%m-%d')
1852 filename = line[0:line.rfind(appid) - 1]
1853 self.apks[filename] = (appid, date)
1854 check_system_clock(date, self.path)
1855 self.changed = False
1857 def writeifchanged(self):
1858 if not self.changed:
1861 if not os.path.exists('stats'):
1865 for apk, app in self.apks.items():
1867 line = apk + ' ' + appid
1869 line += ' ' + added.strftime('%Y-%m-%d')
1872 with open(self.path, 'w', encoding='utf8') as f:
1873 for line in sorted(lst, key=natural_key):
1874 f.write(line + '\n')
1876 def recordapk(self, apkName, app, default_date=None):
1878 Record an apk (if it's new, otherwise does nothing)
1879 Returns the date it was added as a datetime instance
1881 if apkName not in self.apks:
1882 if default_date is None:
1883 default_date = datetime.utcnow()
1884 self.apks[apkName] = (app, default_date)
1886 _ignored, added = self.apks[apkName]
1889 def getapp(self, apkname):
1890 """Look up information - given the 'apkname', returns (app id, date added/None).
1892 Or returns None for an unknown apk.
1894 if apkname in self.apks:
1895 return self.apks[apkname]
1898 def getlatest(self, num):
1899 """Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first"""
1901 for apk, app in self.apks.items():
1905 if apps[appid] > added:
1909 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1910 lst = [app for app, _ignored in sortedapps]
1915 def get_file_extension(filename):
1916 """get the normalized file extension, can be blank string but never None"""
1917 if isinstance(filename, bytes):
1918 filename = filename.decode('utf-8')
1919 return os.path.splitext(filename)[1].lower()[1:]
1922 def get_apk_debuggable_aapt(apkfile):
1923 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1925 if p.returncode != 0:
1926 raise FDroidException(_("Failed to get APK manifest information"))
1927 for line in p.output.splitlines():
1928 if 'android:debuggable' in line and not line.endswith('0x0'):
1933 def get_apk_debuggable_androguard(apkfile):
1935 from androguard.core.bytecodes.apk import APK
1937 raise FDroidException("androguard library is not installed and aapt not present")
1939 apkobject = APK(apkfile)
1940 if apkobject.is_valid_APK():
1941 debuggable = apkobject.get_element("application", "debuggable")
1942 if debuggable is not None:
1943 return bool(strtobool(debuggable))
1947 def isApkAndDebuggable(apkfile):
1948 """Returns True if the given file is an APK and is debuggable
1950 :param apkfile: full path to the apk to check"""
1952 if get_file_extension(apkfile) != 'apk':
1955 if SdkToolsPopen(['aapt', 'version'], output=False):
1956 return get_apk_debuggable_aapt(apkfile)
1958 return get_apk_debuggable_androguard(apkfile)
1961 def get_apk_id_aapt(apkfile):
1962 """Extrat identification information from APK using aapt.
1964 :param apkfile: path to an APK file.
1965 :returns: triplet (appid, version code, version name)
1967 r = re.compile("package: name='(?P<appid>.*)' versionCode='(?P<vercode>.*)' versionName='(?P<vername>.*)' platformBuildVersionName='.*'")
1968 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1969 for line in p.output.splitlines():
1972 return m.group('appid'), m.group('vercode'), m.group('vername')
1973 raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'")
1974 .format(apkfilename=apkfile))
1977 def get_minSdkVersion_aapt(apkfile):
1978 """Extract the minimum supported Android SDK from an APK using aapt
1980 :param apkfile: path to an APK file.
1981 :returns: the integer representing the SDK version
1983 r = re.compile(r"^sdkVersion:'([0-9]+)'")
1984 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1985 for line in p.output.splitlines():
1988 return int(m.group(1))
1989 raise FDroidException(_('Reading minSdkVersion failed: "{apkfilename}"')
1990 .format(apkfilename=apkfile))
1995 self.returncode = None
1999 def SdkToolsPopen(commands, cwd=None, output=True):
2001 if cmd not in config:
2002 config[cmd] = find_sdk_tools_cmd(commands[0])
2003 abscmd = config[cmd]
2005 raise FDroidException(_("Could not find '{command}' on your system").format(command=cmd))
2007 test_aapt_version(config['aapt'])
2008 return FDroidPopen([abscmd] + commands[1:],
2009 cwd=cwd, output=output)
2012 def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2014 Run a command and capture the possibly huge output as bytes.
2016 :param commands: command and argument list like in subprocess.Popen
2017 :param cwd: optionally specifies a working directory
2018 :param envs: a optional dictionary of environment variables and their values
2019 :returns: A PopenResult.
2024 set_FDroidPopen_env()
2026 process_env = env.copy()
2027 if envs is not None and len(envs) > 0:
2028 process_env.update(envs)
2031 cwd = os.path.normpath(cwd)
2032 logging.debug("Directory: %s" % cwd)
2033 logging.debug("> %s" % ' '.join(commands))
2035 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
2036 result = PopenResult()
2039 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
2040 stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
2041 stderr=stderr_param)
2042 except OSError as e:
2043 raise BuildException("OSError while trying to execute " +
2044 ' '.join(commands) + ': ' + str(e))
2046 # TODO are these AsynchronousFileReader threads always exiting?
2047 if not stderr_to_stdout and options.verbose:
2048 stderr_queue = Queue()
2049 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
2051 while not stderr_reader.eof():
2052 while not stderr_queue.empty():
2053 line = stderr_queue.get()
2054 sys.stderr.buffer.write(line)
2059 stdout_queue = Queue()
2060 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
2063 # Check the queue for output (until there is no more to get)
2064 while not stdout_reader.eof():
2065 while not stdout_queue.empty():
2066 line = stdout_queue.get()
2067 if output and options.verbose:
2068 # Output directly to console
2069 sys.stderr.buffer.write(line)
2075 result.returncode = p.wait()
2076 result.output = buf.getvalue()
2078 # make sure all filestreams of the subprocess are closed
2079 for streamvar in ['stdin', 'stdout', 'stderr']:
2080 if hasattr(p, streamvar):
2081 stream = getattr(p, streamvar)
2087 def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2089 Run a command and capture the possibly huge output as a str.
2091 :param commands: command and argument list like in subprocess.Popen
2092 :param cwd: optionally specifies a working directory
2093 :param envs: a optional dictionary of environment variables and their values
2094 :returns: A PopenResult.
2096 result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
2097 result.output = result.output.decode('utf-8', 'ignore')
2101 gradle_comment = re.compile(r'[ ]*//')
2102 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
2103 gradle_line_matches = [
2104 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
2105 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
2106 re.compile(r'.*\.readLine\(.*'),
2110 def remove_signing_keys(build_dir):
2111 for root, dirs, files in os.walk(build_dir):
2112 if 'build.gradle' in files:
2113 path = os.path.join(root, 'build.gradle')
2115 with open(path, "r", encoding='utf8') as o:
2116 lines = o.readlines()
2122 with open(path, "w", encoding='utf8') as o:
2123 while i < len(lines):
2126 while line.endswith('\\\n'):
2127 line = line.rstrip('\\\n') + lines[i]
2130 if gradle_comment.match(line):
2135 opened += line.count('{')
2136 opened -= line.count('}')
2139 if gradle_signing_configs.match(line):
2144 if any(s.match(line) for s in gradle_line_matches):
2152 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
2155 'project.properties',
2157 'default.properties',
2158 'ant.properties', ]:
2159 if propfile in files:
2160 path = os.path.join(root, propfile)
2162 with open(path, "r", encoding='iso-8859-1') as o:
2163 lines = o.readlines()
2167 with open(path, "w", encoding='iso-8859-1') as o:
2169 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
2176 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
2179 def set_FDroidPopen_env(build=None):
2181 set up the environment variables for the build environment
2183 There is only a weak standard, the variables used by gradle, so also set
2184 up the most commonly used environment variables for SDK and NDK. Also, if
2185 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
2187 global env, orig_path
2191 orig_path = env['PATH']
2192 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
2193 env[n] = config['sdk_path']
2194 for k, v in config['java_paths'].items():
2195 env['JAVA%s_HOME' % k] = v
2197 missinglocale = True
2198 for k, v in env.items():
2199 if k == 'LANG' and v != 'C':
2200 missinglocale = False
2202 missinglocale = False
2204 env['LANG'] = 'en_US.UTF-8'
2206 if build is not None:
2207 path = build.ndk_path()
2208 paths = orig_path.split(os.pathsep)
2209 if path not in paths:
2210 paths = [path] + paths
2211 env['PATH'] = os.pathsep.join(paths)
2212 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
2213 env[n] = build.ndk_path()
2216 def replace_build_vars(cmd, build):
2217 cmd = cmd.replace('$$COMMIT$$', build.commit)
2218 cmd = cmd.replace('$$VERSION$$', build.versionName)
2219 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
2223 def replace_config_vars(cmd, build):
2224 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
2225 cmd = cmd.replace('$$NDK$$', build.ndk_path())
2226 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
2227 if build is not None:
2228 cmd = replace_build_vars(cmd, build)
2232 def place_srclib(root_dir, number, libpath):
2235 relpath = os.path.relpath(libpath, root_dir)
2236 proppath = os.path.join(root_dir, 'project.properties')
2239 if os.path.isfile(proppath):
2240 with open(proppath, "r", encoding='iso-8859-1') as o:
2241 lines = o.readlines()
2243 with open(proppath, "w", encoding='iso-8859-1') as o:
2246 if line.startswith('android.library.reference.%d=' % number):
2247 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2252 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2255 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
2258 def signer_fingerprint_short(sig):
2259 """Obtain shortened sha256 signing-key fingerprint for pkcs7 signature.
2261 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2262 for a given pkcs7 signature.
2264 :param sig: Contents of an APK signing certificate.
2265 :returns: shortened signing-key fingerprint.
2267 return signer_fingerprint(sig)[:7]
2270 def signer_fingerprint(sig):
2271 """Obtain sha256 signing-key fingerprint for pkcs7 signature.
2273 Extracts hexadecimal sha256 signing-key fingerprint string
2274 for a given pkcs7 signature.
2276 :param: Contents of an APK signature.
2277 :returns: shortened signature fingerprint.
2279 cert_encoded = get_certificate(sig)
2280 return hashlib.sha256(cert_encoded).hexdigest()
2283 def apk_signer_fingerprint(apk_path):
2284 """Obtain sha256 signing-key fingerprint for APK.
2286 Extracts hexadecimal sha256 signing-key fingerprint string
2289 :param apkpath: path to APK
2290 :returns: signature fingerprint
2293 with zipfile.ZipFile(apk_path, 'r') as apk:
2294 certs = [n for n in apk.namelist() if CERT_PATH_REGEX.match(n)]
2297 logging.error("Found no signing certificates on %s" % apk_path)
2300 logging.error("Found multiple signing certificates on %s" % apk_path)
2303 cert = apk.read(certs[0])
2304 return signer_fingerprint(cert)
2307 def apk_signer_fingerprint_short(apk_path):
2308 """Obtain shortened sha256 signing-key fingerprint for APK.
2310 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2311 for a given pkcs7 APK.
2313 :param apk_path: path to APK
2314 :returns: shortened signing-key fingerprint
2316 return apk_signer_fingerprint(apk_path)[:7]
2319 def metadata_get_sigdir(appid, vercode=None):
2320 """Get signature directory for app"""
2322 return os.path.join('metadata', appid, 'signatures', vercode)
2324 return os.path.join('metadata', appid, 'signatures')
2327 def metadata_find_developer_signature(appid, vercode=None):
2328 """Tires to find the developer signature for given appid.
2330 This picks the first signature file found in metadata an returns its
2333 :returns: sha256 signing key fingerprint of the developer signing key.
2334 None in case no signature can not be found."""
2336 # fetch list of dirs for all versions of signatures
2339 appversigdirs.append(metadata_get_sigdir(appid, vercode))
2341 appsigdir = metadata_get_sigdir(appid)
2342 if os.path.isdir(appsigdir):
2343 numre = re.compile('[0-9]+')
2344 for ver in os.listdir(appsigdir):
2345 if numre.match(ver):
2346 appversigdir = os.path.join(appsigdir, ver)
2347 appversigdirs.append(appversigdir)
2349 for sigdir in appversigdirs:
2350 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2351 glob.glob(os.path.join(sigdir, '*.EC')) + \
2352 glob.glob(os.path.join(sigdir, '*.RSA'))
2354 raise FDroidException('ambiguous signatures, please make sure there is only one signature in \'{}\'. (The signature has to be the App maintainers signature for version of the APK.)'.format(sigdir))
2356 with open(sig, 'rb') as f:
2357 return signer_fingerprint(f.read())
2361 def metadata_find_signing_files(appid, vercode):
2362 """Gets a list of singed manifests and signatures.
2364 :param appid: app id string
2365 :param vercode: app version code
2366 :returns: a list of triplets for each signing key with following paths:
2367 (signature_file, singed_file, manifest_file)
2370 sigdir = metadata_get_sigdir(appid, vercode)
2371 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2372 glob.glob(os.path.join(sigdir, '*.EC')) + \
2373 glob.glob(os.path.join(sigdir, '*.RSA'))
2374 extre = re.compile('(\.DSA|\.EC|\.RSA)$')
2376 sf = extre.sub('.SF', sig)
2377 if os.path.isfile(sf):
2378 mf = os.path.join(sigdir, 'MANIFEST.MF')
2379 if os.path.isfile(mf):
2380 ret.append((sig, sf, mf))
2384 def metadata_find_developer_signing_files(appid, vercode):
2385 """Get developer signature files for specified app from metadata.
2387 :returns: A triplet of paths for signing files from metadata:
2388 (signature_file, singed_file, manifest_file)
2390 allsigningfiles = metadata_find_signing_files(appid, vercode)
2391 if allsigningfiles and len(allsigningfiles) == 1:
2392 return allsigningfiles[0]
2397 def apk_strip_signatures(signed_apk, strip_manifest=False):
2398 """Removes signatures from APK.
2400 :param signed_apk: path to apk file.
2401 :param strip_manifest: when set to True also the manifest file will
2402 be removed from the APK.
2404 with tempfile.TemporaryDirectory() as tmpdir:
2405 tmp_apk = os.path.join(tmpdir, 'tmp.apk')
2406 shutil.move(signed_apk, tmp_apk)
2407 with ZipFile(tmp_apk, 'r') as in_apk:
2408 with ZipFile(signed_apk, 'w') as out_apk:
2409 for info in in_apk.infolist():
2410 if not apk_sigfile.match(info.filename):
2412 if info.filename != 'META-INF/MANIFEST.MF':
2413 buf = in_apk.read(info.filename)
2414 out_apk.writestr(info, buf)
2416 buf = in_apk.read(info.filename)
2417 out_apk.writestr(info, buf)
2420 def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest):
2421 """Implats a signature from metadata into an APK.
2423 Note: this changes there supplied APK in place. So copy it if you
2424 need the original to be preserved.
2426 :param apkpath: location of the apk
2428 # get list of available signature files in metadata
2429 with tempfile.TemporaryDirectory() as tmpdir:
2430 apkwithnewsig = os.path.join(tmpdir, 'newsig.apk')
2431 with ZipFile(apkpath, 'r') as in_apk:
2432 with ZipFile(apkwithnewsig, 'w') as out_apk:
2433 for sig_file in [signaturefile, signedfile, manifest]:
2434 with open(sig_file, 'rb') as fp:
2436 info = zipfile.ZipInfo('META-INF/' + os.path.basename(sig_file))
2437 info.compress_type = zipfile.ZIP_DEFLATED
2438 info.create_system = 0 # "Windows" aka "FAT", what Android SDK uses
2439 out_apk.writestr(info, buf)
2440 for info in in_apk.infolist():
2441 if not apk_sigfile.match(info.filename):
2442 if info.filename != 'META-INF/MANIFEST.MF':
2443 buf = in_apk.read(info.filename)
2444 out_apk.writestr(info, buf)
2446 p = SdkToolsPopen(['zipalign', '-v', '4', apkwithnewsig, apkpath])
2447 if p.returncode != 0:
2448 raise BuildException("Failed to align application")
2451 def apk_extract_signatures(apkpath, outdir, manifest=True):
2452 """Extracts a signature files from APK and puts them into target directory.
2454 :param apkpath: location of the apk
2455 :param outdir: folder where the extracted signature files will be stored
2456 :param manifest: (optionally) disable extracting manifest file
2458 with ZipFile(apkpath, 'r') as in_apk:
2459 for f in in_apk.infolist():
2460 if apk_sigfile.match(f.filename) or \
2461 (manifest and f.filename == 'META-INF/MANIFEST.MF'):
2462 newpath = os.path.join(outdir, os.path.basename(f.filename))
2463 with open(newpath, 'wb') as out_file:
2464 out_file.write(in_apk.read(f.filename))
2467 def sign_apk(unsigned_path, signed_path, keyalias):
2468 """Sign and zipalign an unsigned APK, then save to a new file, deleting the unsigned
2470 android-18 (4.3) finally added support for reasonable hash
2471 algorithms, like SHA-256, before then, the only options were MD5
2472 and SHA1 :-/ This aims to use SHA-256 when the APK does not target
2473 older Android versions, and is therefore safe to do so.
2475 https://issuetracker.google.com/issues/36956587
2476 https://android-review.googlesource.com/c/platform/libcore/+/44491
2480 if get_minSdkVersion_aapt(unsigned_path) < 18:
2481 signature_algorithm = ['-sigalg', 'SHA1withRSA', '-digestalg', 'SHA1']
2483 signature_algorithm = ['-sigalg', 'SHA256withRSA', '-digestalg', 'SHA-256']
2485 p = FDroidPopen([config['jarsigner'], '-keystore', config['keystore'],
2486 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2487 '-keypass:env', 'FDROID_KEY_PASS']
2488 + signature_algorithm + [unsigned_path, keyalias],
2490 'FDROID_KEY_STORE_PASS': config['keystorepass'],
2491 'FDROID_KEY_PASS': config['keypass'], })
2492 if p.returncode != 0:
2493 raise BuildException(_("Failed to sign application"), p.output)
2495 p = SdkToolsPopen(['zipalign', '-v', '4', unsigned_path, signed_path])
2496 if p.returncode != 0:
2497 raise BuildException(_("Failed to zipalign application"))
2498 os.remove(unsigned_path)
2501 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
2502 """Verify that two apks are the same
2504 One of the inputs is signed, the other is unsigned. The signature metadata
2505 is transferred from the signed to the unsigned apk, and then jarsigner is
2506 used to verify that the signature from the signed apk is also varlid for
2507 the unsigned one. If the APK given as unsigned actually does have a
2508 signature, it will be stripped out and ignored.
2510 There are two SHA1 git commit IDs that fdroidserver includes in the builds
2511 it makes: fdroidserverid and buildserverid. Originally, these were inserted
2512 into AndroidManifest.xml, but that makes the build not reproducible. So
2513 instead they are included as separate files in the APK's META-INF/ folder.
2514 If those files exist in the signed APK, they will be part of the signature
2515 and need to also be included in the unsigned APK for it to validate.
2517 :param signed_apk: Path to a signed apk file
2518 :param unsigned_apk: Path to an unsigned apk file expected to match it
2519 :param tmp_dir: Path to directory for temporary files
2520 :returns: None if the verification is successful, otherwise a string
2521 describing what went wrong.
2524 if not os.path.isfile(signed_apk):
2525 return 'can not verify: file does not exists: {}'.format(signed_apk)
2527 if not os.path.isfile(unsigned_apk):
2528 return 'can not verify: file does not exists: {}'.format(unsigned_apk)
2530 with ZipFile(signed_apk, 'r') as signed:
2531 meta_inf_files = ['META-INF/MANIFEST.MF']
2532 for f in signed.namelist():
2533 if apk_sigfile.match(f) \
2534 or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
2535 meta_inf_files.append(f)
2536 if len(meta_inf_files) < 3:
2537 return "Signature files missing from {0}".format(signed_apk)
2539 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2540 with ZipFile(unsigned_apk, 'r') as unsigned:
2541 # only read the signature from the signed APK, everything else from unsigned
2542 with ZipFile(tmp_apk, 'w') as tmp:
2543 for filename in meta_inf_files:
2544 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2545 for info in unsigned.infolist():
2546 if info.filename in meta_inf_files:
2547 logging.warning('Ignoring %s from %s',
2548 info.filename, unsigned_apk)
2550 if info.filename in tmp.namelist():
2551 return "duplicate filename found: " + info.filename
2552 tmp.writestr(info, unsigned.read(info.filename))
2554 verified = verify_apk_signature(tmp_apk)
2557 logging.info("...NOT verified - {0}".format(tmp_apk))
2558 return compare_apks(signed_apk, tmp_apk, tmp_dir,
2559 os.path.dirname(unsigned_apk))
2561 logging.info("...successfully verified")
2565 def verify_jar_signature(jar):
2566 """Verifies the signature of a given JAR file.
2568 jarsigner is very shitty: unsigned JARs pass as "verified"! So
2569 this has to turn on -strict then check for result 4, since this
2570 does not expect the signature to be from a CA-signed certificate.
2572 :raises: VerificationException() if the JAR's signature could not be verified
2576 error = _('JAR signature failed to verify: {path}').format(path=jar)
2578 output = subprocess.check_output([config['jarsigner'], '-strict', '-verify', jar],
2579 stderr=subprocess.STDOUT)
2580 raise VerificationException(error + '\n' + output.decode('utf-8'))
2581 except subprocess.CalledProcessError as e:
2582 if e.returncode == 4:
2583 logging.debug(_('JAR signature verified: {path}').format(path=jar))
2585 raise VerificationException(error + '\n' + e.output.decode('utf-8'))
2588 def verify_apk_signature(apk, min_sdk_version=None):
2589 """verify the signature on an APK
2591 Try to use apksigner whenever possible since jarsigner is very
2592 shitty: unsigned APKs pass as "verified"! Warning, this does
2593 not work on JARs with apksigner >= 0.7 (build-tools 26.0.1)
2595 :returns: boolean whether the APK was verified
2597 if set_command_in_config('apksigner'):
2598 args = [config['apksigner'], 'verify']
2600 args += ['--min-sdk-version=' + min_sdk_version]
2602 args += ['--verbose']
2604 output = subprocess.check_output(args + [apk])
2606 logging.debug(apk + ': ' + output.decode('utf-8'))
2608 except subprocess.CalledProcessError as e:
2609 logging.error('\n' + apk + ': ' + e.output.decode('utf-8'))
2611 if not config.get('jarsigner_warning_displayed'):
2612 config['jarsigner_warning_displayed'] = True
2613 logging.warning(_("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner"))
2615 verify_jar_signature(apk)
2617 except Exception as e:
2622 def verify_old_apk_signature(apk):
2623 """verify the signature on an archived APK, supporting deprecated algorithms
2625 F-Droid aims to keep every single binary that it ever published. Therefore,
2626 it needs to be able to verify APK signatures that include deprecated/removed
2627 algorithms. For example, jarsigner treats an MD5 signature as unsigned.
2629 jarsigner passes unsigned APKs as "verified"! So this has to turn
2630 on -strict then check for result 4.
2632 :returns: boolean whether the APK was verified
2635 _java_security = os.path.join(os.getcwd(), '.java.security')
2636 with open(_java_security, 'w') as fp:
2637 fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
2641 config['jarsigner'],
2642 '-J-Djava.security.properties=' + _java_security,
2643 '-strict', '-verify', apk
2645 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
2646 except subprocess.CalledProcessError as e:
2647 if e.returncode != 4:
2650 logging.debug(_('JAR signature verified: {path}').format(path=apk))
2653 logging.error(_('Old APK signature failed to verify: {path}').format(path=apk)
2654 + '\n' + output.decode('utf-8'))
2658 apk_badchars = re.compile('''[/ :;'"]''')
2661 def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
2664 Returns None if the apk content is the same (apart from the signing key),
2665 otherwise a string describing what's different, or what went wrong when
2666 trying to do the comparison.
2672 absapk1 = os.path.abspath(apk1)
2673 absapk2 = os.path.abspath(apk2)
2675 if set_command_in_config('diffoscope'):
2676 logfilename = os.path.join(log_dir, os.path.basename(absapk1))
2677 htmlfile = logfilename + '.diffoscope.html'
2678 textfile = logfilename + '.diffoscope.txt'
2679 if subprocess.call([config['diffoscope'],
2680 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2681 '--html', htmlfile, '--text', textfile,
2682 absapk1, absapk2]) != 0:
2683 return("Failed to unpack " + apk1)
2685 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2686 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2687 for d in [apk1dir, apk2dir]:
2688 if os.path.exists(d):
2691 os.mkdir(os.path.join(d, 'jar-xf'))
2693 if subprocess.call(['jar', 'xf',
2694 os.path.abspath(apk1)],
2695 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2696 return("Failed to unpack " + apk1)
2697 if subprocess.call(['jar', 'xf',
2698 os.path.abspath(apk2)],
2699 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2700 return("Failed to unpack " + apk2)
2702 if set_command_in_config('apktool'):
2703 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2705 return("Failed to unpack " + apk1)
2706 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2708 return("Failed to unpack " + apk2)
2710 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2711 lines = p.output.splitlines()
2712 if len(lines) != 1 or 'META-INF' not in lines[0]:
2713 if set_command_in_config('meld'):
2714 p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
2715 return("Unexpected diff output - " + p.output)
2717 # since everything verifies, delete the comparison to keep cruft down
2718 shutil.rmtree(apk1dir)
2719 shutil.rmtree(apk2dir)
2721 # If we get here, it seems like they're the same!
2725 def set_command_in_config(command):
2726 '''Try to find specified command in the path, if it hasn't been
2727 manually set in config.py. If found, it is added to the config
2728 dict. The return value says whether the command is available.
2731 if command in config:
2734 tmp = find_command(command)
2736 config[command] = tmp
2741 def find_command(command):
2742 '''find the full path of a command, or None if it can't be found in the PATH'''
2745 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2747 fpath, fname = os.path.split(command)
2752 for path in os.environ["PATH"].split(os.pathsep):
2753 path = path.strip('"')
2754 exe_file = os.path.join(path, command)
2755 if is_exe(exe_file):
2762 '''generate a random password for when generating keys'''
2763 h = hashlib.sha256()
2764 h.update(os.urandom(16)) # salt
2765 h.update(socket.getfqdn().encode('utf-8'))
2766 passwd = base64.b64encode(h.digest()).strip()
2767 return passwd.decode('utf-8')
2770 def genkeystore(localconfig):
2772 Generate a new key with password provided in :param localconfig and add it to new keystore
2773 :return: hexed public key, public key fingerprint
2775 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2776 keystoredir = os.path.dirname(localconfig['keystore'])
2777 if keystoredir is None or keystoredir == '':
2778 keystoredir = os.path.join(os.getcwd(), keystoredir)
2779 if not os.path.exists(keystoredir):
2780 os.makedirs(keystoredir, mode=0o700)
2783 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
2784 'FDROID_KEY_PASS': localconfig['keypass'],
2786 p = FDroidPopen([config['keytool'], '-genkey',
2787 '-keystore', localconfig['keystore'],
2788 '-alias', localconfig['repo_keyalias'],
2789 '-keyalg', 'RSA', '-keysize', '4096',
2790 '-sigalg', 'SHA256withRSA',
2791 '-validity', '10000',
2792 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2793 '-keypass:env', 'FDROID_KEY_PASS',
2794 '-dname', localconfig['keydname']], envs=env_vars)
2795 if p.returncode != 0:
2796 raise BuildException("Failed to generate key", p.output)
2797 os.chmod(localconfig['keystore'], 0o0600)
2798 if not options.quiet:
2799 # now show the lovely key that was just generated
2800 p = FDroidPopen([config['keytool'], '-list', '-v',
2801 '-keystore', localconfig['keystore'],
2802 '-alias', localconfig['repo_keyalias'],
2803 '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
2804 logging.info(p.output.strip() + '\n\n')
2805 # get the public key
2806 p = FDroidPopenBytes([config['keytool'], '-exportcert',
2807 '-keystore', localconfig['keystore'],
2808 '-alias', localconfig['repo_keyalias'],
2809 '-storepass:env', 'FDROID_KEY_STORE_PASS']
2810 + config['smartcardoptions'],
2811 envs=env_vars, output=False, stderr_to_stdout=False)
2812 if p.returncode != 0 or len(p.output) < 20:
2813 raise BuildException("Failed to get public key", p.output)
2815 fingerprint = get_cert_fingerprint(pubkey)
2816 return hexlify(pubkey), fingerprint
2819 def get_cert_fingerprint(pubkey):
2821 Generate a certificate fingerprint the same way keytool does it
2822 (but with slightly different formatting)
2824 digest = hashlib.sha256(pubkey).digest()
2825 ret = [' '.join("%02X" % b for b in bytearray(digest))]
2826 return " ".join(ret)
2829 def get_certificate(certificate_file):
2831 Extracts a certificate from the given file.
2832 :param certificate_file: file bytes (as string) representing the certificate
2833 :return: A binary representation of the certificate's public key, or None in case of error
2835 content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
2836 if content.getComponentByName('contentType') != rfc2315.signedData:
2838 content = decoder.decode(content.getComponentByName('content'),
2839 asn1Spec=rfc2315.SignedData())[0]
2841 certificates = content.getComponentByName('certificates')
2842 cert = certificates[0].getComponentByName('certificate')
2844 logging.error("Certificates not found.")
2846 return encoder.encode(cert)
2849 def load_stats_fdroid_signing_key_fingerprints():
2850 """Load list of signing-key fingerprints stored by fdroid publish from file.
2852 :returns: list of dictionanryies containing the singing-key fingerprints.
2854 jar_file = os.path.join('stats', 'publishsigkeys.jar')
2855 if not os.path.isfile(jar_file):
2857 cmd = [config['jarsigner'], '-strict', '-verify', jar_file]
2858 p = FDroidPopen(cmd, output=False)
2859 if p.returncode != 4:
2860 raise FDroidException("Signature validation of '{}' failed! "
2861 "Please run publish again to rebuild this file.".format(jar_file))
2863 jar_sigkey = apk_signer_fingerprint(jar_file)
2864 repo_key_sig = config.get('repo_key_sha256')
2866 if jar_sigkey != repo_key_sig:
2867 raise FDroidException("Signature key fingerprint of file '{}' does not match repo_key_sha256 in config.py (found fingerprint: '{}')".format(jar_file, jar_sigkey))
2869 logging.warning("repo_key_sha256 not in config.py, setting it to the signature key fingerprint of '{}'".format(jar_file))
2870 config['repo_key_sha256'] = jar_sigkey
2871 write_to_config(config, 'repo_key_sha256')
2873 with zipfile.ZipFile(jar_file, 'r') as f:
2874 return json.loads(str(f.read('publishsigkeys.json'), 'utf-8'))
2877 def write_to_config(thisconfig, key, value=None, config_file=None):
2878 '''write a key/value to the local config.py
2880 NOTE: only supports writing string variables.
2882 :param thisconfig: config dictionary
2883 :param key: variable name in config.py to be overwritten/added
2884 :param value: optional value to be written, instead of fetched
2885 from 'thisconfig' dictionary.
2888 origkey = key + '_orig'
2889 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2890 cfg = config_file if config_file else 'config.py'
2892 # load config file, create one if it doesn't exist
2893 if not os.path.exists(cfg):
2894 open(cfg, 'a').close()
2895 logging.info("Creating empty " + cfg)
2896 with open(cfg, 'r', encoding="utf-8") as f:
2897 lines = f.readlines()
2899 # make sure the file ends with a carraige return
2901 if not lines[-1].endswith('\n'):
2904 # regex for finding and replacing python string variable
2905 # definitions/initializations
2906 pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
2907 repl = key + ' = "' + value + '"'
2908 pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
2909 repl2 = key + " = '" + value + "'"
2911 # If we replaced this line once, we make sure won't be a
2912 # second instance of this line for this key in the document.
2915 with open(cfg, 'w', encoding="utf-8") as f:
2917 if pattern.match(line) or pattern2.match(line):
2919 line = pattern.sub(repl, line)
2920 line = pattern2.sub(repl2, line)
2931 def parse_xml(path):
2932 return XMLElementTree.parse(path).getroot()
2935 def string_is_integer(string):
2943 def local_rsync(options, fromdir, todir):
2944 '''Rsync method for local to local copying of things
2946 This is an rsync wrapper with all the settings for safe use within
2947 the various fdroidserver use cases. This uses stricter rsync
2948 checking on all files since people using offline mode are already
2949 prioritizing security above ease and speed.
2952 rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
2953 '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
2954 if not options.no_checksum:
2955 rsyncargs.append('--checksum')
2957 rsyncargs += ['--verbose']
2959 rsyncargs += ['--quiet']
2960 logging.debug(' '.join(rsyncargs + [fromdir, todir]))
2961 if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
2962 raise FDroidException()
2965 def get_per_app_repos():
2966 '''per-app repos are dirs named with the packageName of a single app'''
2968 # Android packageNames are Java packages, they may contain uppercase or
2969 # lowercase letters ('A' through 'Z'), numbers, and underscores
2970 # ('_'). However, individual package name parts may only start with
2971 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2972 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2975 for root, dirs, files in os.walk(os.getcwd()):
2977 print('checking', root, 'for', d)
2978 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2979 # standard parts of an fdroid repo, so never packageNames
2982 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
2988 def is_repo_file(filename):
2989 '''Whether the file in a repo is a build product to be delivered to users'''
2990 if isinstance(filename, str):
2991 filename = filename.encode('utf-8', errors="surrogateescape")
2992 return os.path.isfile(filename) \
2993 and not filename.endswith(b'.asc') \
2994 and not filename.endswith(b'.sig') \
2995 and os.path.basename(filename) not in [
2997 b'index_unsigned.jar',
3006 def get_examples_dir():
3007 '''Return the dir where the fdroidserver example files are available'''
3009 tmp = os.path.dirname(sys.argv[0])
3010 if os.path.basename(tmp) == 'bin':
3011 egg_links = glob.glob(os.path.join(tmp, '..',
3012 'local/lib/python3.*/site-packages/fdroidserver.egg-link'))
3014 # installed from local git repo
3015 examplesdir = os.path.join(open(egg_links[0]).readline().rstrip(), 'examples')
3018 examplesdir = os.path.dirname(os.path.dirname(__file__)) + '/share/doc/fdroidserver/examples'
3019 if not os.path.exists(examplesdir): # use UNIX layout
3020 examplesdir = os.path.dirname(tmp) + '/share/doc/fdroidserver/examples'
3022 # we're running straight out of the git repo
3023 prefix = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
3024 examplesdir = prefix + '/examples'
3029 def get_wiki_timestamp(timestamp=None):
3030 """Return current time in the standard format for posting to the wiki"""
3032 if timestamp is None:
3033 timestamp = time.gmtime()
3034 return time.strftime("%Y-%m-%d %H:%M:%SZ", timestamp)
3037 def get_android_tools_versions(ndk_path=None):
3038 '''get a list of the versions of all installed Android SDK/NDK components'''
3041 sdk_path = config['sdk_path']
3042 if sdk_path[-1] != '/':
3046 ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT')
3047 if os.path.isfile(ndk_release_txt):
3048 with open(ndk_release_txt, 'r') as fp:
3049 components.append((os.path.basename(ndk_path), fp.read()[:-1]))
3051 pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE)
3052 for root, dirs, files in os.walk(sdk_path):
3053 if 'source.properties' in files:
3054 source_properties = os.path.join(root, 'source.properties')
3055 with open(source_properties, 'r') as fp:
3056 m = pattern.search(fp.read())
3058 components.append((root[len(sdk_path):], m.group(1)))
3063 def get_android_tools_version_log(ndk_path=None):
3064 '''get a list of the versions of all installed Android SDK/NDK components'''
3065 log = '== Installed Android Tools ==\n\n'
3066 components = get_android_tools_versions(ndk_path)
3067 for name, version in sorted(components):
3068 log += '* ' + name + ' (' + version + ')\n'