3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
40 import xml.etree.ElementTree as XMLElementTree
42 from binascii import hexlify
43 from datetime import datetime, timedelta
44 from distutils.version import LooseVersion
45 from queue import Queue
46 from zipfile import ZipFile
48 from pyasn1.codec.der import decoder, encoder
49 from pyasn1_modules import rfc2315
50 from pyasn1.error import PyAsn1Error
52 from distutils.util import strtobool
54 import fdroidserver.metadata
55 from fdroidserver import _
56 from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException,\
57 BuildException, VerificationException
58 from .asynchronousfilereader import AsynchronousFileReader
60 # this is the build-tools version, aapt has a separate version that
61 # has to be manually set in test_aapt_version()
62 MINIMUM_AAPT_VERSION = '26.0.0'
64 # A signature block file with a .DSA, .RSA, or .EC extension
65 CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
66 APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
67 STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
69 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
78 'sdk_path': "$ANDROID_HOME",
82 'r12b': "$ANDROID_NDK",
88 'build_tools': MINIMUM_AAPT_VERSION,
89 'force_build_tools': False,
94 'accepted_formats': ['txt', 'yml'],
95 'sync_from_local_copy_dir': False,
96 'allow_disabled_algorithms': False,
97 'per_app_repos': False,
98 'make_current_version_link': True,
99 'current_version_name_source': 'Name',
100 'update_stats': False,
102 'stats_server': None,
104 'stats_to_carbon': False,
106 'build_server_always': False,
107 'keystore': 'keystore.jks',
108 'smartcardoptions': [],
118 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
119 'repo_name': "My First FDroid Repo Demo",
120 'repo_icon': "fdroid-icon.png",
121 'repo_description': '''
122 This is a repository of apps to be used with FDroid. Applications in this
123 repository are either official binaries built by the original application
124 developers, or are binaries built from source by the admin of f-droid.org
125 using the tools on https://gitlab.com/u/fdroid.
131 def setup_global_opts(parser):
132 try: # the buildserver VM might not have PIL installed
133 from PIL import PngImagePlugin
134 logger = logging.getLogger(PngImagePlugin.__name__)
135 logger.setLevel(logging.INFO) # tame the "STREAM" debug messages
139 parser.add_argument("-v", "--verbose", action="store_true", default=False,
140 help=_("Spew out even more information than normal"))
141 parser.add_argument("-q", "--quiet", action="store_true", default=False,
142 help=_("Restrict output to warnings and errors"))
145 def _add_java_paths_to_config(pathlist, thisconfig):
146 def path_version_key(s):
148 for u in re.split('[^0-9]+', s):
150 versionlist.append(int(u))
155 for d in sorted(pathlist, key=path_version_key):
156 if os.path.islink(d):
158 j = os.path.basename(d)
159 # the last one found will be the canonical one, so order appropriately
161 r'^1\.([6-9])\.0\.jdk$', # OSX
162 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
163 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
164 r'^jdk([6-9])-openjdk$', # Arch
165 r'^java-([6-9])-openjdk$', # Arch
166 r'^java-([6-9])-jdk$', # Arch (oracle)
167 r'^java-1\.([6-9])\.0-.*$', # RedHat
168 r'^java-([6-9])-oracle$', # Debian WebUpd8
169 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
170 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
172 m = re.match(regex, j)
175 for p in [d, os.path.join(d, 'Contents', 'Home')]:
176 if os.path.exists(os.path.join(p, 'bin', 'javac')):
177 thisconfig['java_paths'][m.group(1)] = p
180 def fill_config_defaults(thisconfig):
181 for k, v in default_config.items():
182 if k not in thisconfig:
185 # Expand paths (~users and $vars)
186 def expand_path(path):
190 path = os.path.expanduser(path)
191 path = os.path.expandvars(path)
196 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
201 thisconfig[k + '_orig'] = v
203 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
204 if thisconfig['java_paths'] is None:
205 thisconfig['java_paths'] = dict()
207 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
208 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
209 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
210 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
211 if os.getenv('JAVA_HOME') is not None:
212 pathlist.append(os.getenv('JAVA_HOME'))
213 if os.getenv('PROGRAMFILES') is not None:
214 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
215 _add_java_paths_to_config(pathlist, thisconfig)
217 for java_version in ('7', '8', '9'):
218 if java_version not in thisconfig['java_paths']:
220 java_home = thisconfig['java_paths'][java_version]
221 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
222 if os.path.exists(jarsigner):
223 thisconfig['jarsigner'] = jarsigner
224 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
225 break # Java7 is preferred, so quit if found
227 for k in ['ndk_paths', 'java_paths']:
233 thisconfig[k][k2] = exp
234 thisconfig[k][k2 + '_orig'] = v
237 def regsub_file(pattern, repl, path):
238 with open(path, 'rb') as f:
240 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
241 with open(path, 'wb') as f:
245 def read_config(opts, config_file='config.py'):
246 """Read the repository config
248 The config is read from config_file, which is in the current
249 directory when any of the repo management commands are used. If
250 there is a local metadata file in the git repo, then config.py is
251 not required, just use defaults.
254 global config, options
256 if config is not None:
263 if os.path.isfile(config_file):
264 logging.debug(_("Reading '{config_file}'").format(config_file=config_file))
265 with io.open(config_file, "rb") as f:
266 code = compile(f.read(), config_file, 'exec')
267 exec(code, None, config)
269 logging.warning(_("No 'config.py' found, using defaults."))
271 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
273 if not type(config[k]) in (str, list, tuple):
275 _("'{field}' will be in random order! Use () or [] brackets if order is important!")
278 # smartcardoptions must be a list since its command line args for Popen
279 if 'smartcardoptions' in config:
280 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
281 elif 'keystore' in config and config['keystore'] == 'NONE':
282 # keystore='NONE' means use smartcard, these are required defaults
283 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
284 'SunPKCS11-OpenSC', '-providerClass',
285 'sun.security.pkcs11.SunPKCS11',
286 '-providerArg', 'opensc-fdroid.cfg']
288 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
289 st = os.stat(config_file)
290 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
291 logging.warning(_("unsafe permissions on '{config_file}' (should be 0600)!")
292 .format(config_file=config_file))
294 fill_config_defaults(config)
296 for k in ["repo_description", "archive_description"]:
298 config[k] = clean_description(config[k])
300 if 'serverwebroot' in config:
301 if isinstance(config['serverwebroot'], str):
302 roots = [config['serverwebroot']]
303 elif all(isinstance(item, str) for item in config['serverwebroot']):
304 roots = config['serverwebroot']
306 raise TypeError(_('only accepts strings, lists, and tuples'))
308 for rootstr in roots:
309 # since this is used with rsync, where trailing slashes have
310 # meaning, ensure there is always a trailing slash
311 if rootstr[-1] != '/':
313 rootlist.append(rootstr.replace('//', '/'))
314 config['serverwebroot'] = rootlist
316 if 'servergitmirrors' in config:
317 if isinstance(config['servergitmirrors'], str):
318 roots = [config['servergitmirrors']]
319 elif all(isinstance(item, str) for item in config['servergitmirrors']):
320 roots = config['servergitmirrors']
322 raise TypeError(_('only accepts strings, lists, and tuples'))
323 config['servergitmirrors'] = roots
328 def assert_config_keystore(config):
329 """Check weather keystore is configured correctly and raise exception if not."""
332 if 'repo_keyalias' not in config:
334 logging.critical(_("'repo_keyalias' not found in config.py!"))
335 if 'keystore' not in config:
337 logging.critical(_("'keystore' not found in config.py!"))
338 elif not os.path.exists(config['keystore']):
340 logging.critical("'" + config['keystore'] + "' does not exist!")
341 if 'keystorepass' not in config:
343 logging.critical(_("'keystorepass' not found in config.py!"))
344 if 'keypass' not in config:
346 logging.critical(_("'keypass' not found in config.py!"))
348 raise FDroidException("This command requires a signing key, " +
349 "you can create one using: fdroid update --create-key")
352 def find_sdk_tools_cmd(cmd):
353 '''find a working path to a tool from the Android SDK'''
356 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
357 # try to find a working path to this command, in all the recent possible paths
358 if 'build_tools' in config:
359 build_tools = os.path.join(config['sdk_path'], 'build-tools')
360 # if 'build_tools' was manually set and exists, check only that one
361 configed_build_tools = os.path.join(build_tools, config['build_tools'])
362 if os.path.exists(configed_build_tools):
363 tooldirs.append(configed_build_tools)
365 # no configed version, so hunt known paths for it
366 for f in sorted(os.listdir(build_tools), reverse=True):
367 if os.path.isdir(os.path.join(build_tools, f)):
368 tooldirs.append(os.path.join(build_tools, f))
369 tooldirs.append(build_tools)
370 sdk_tools = os.path.join(config['sdk_path'], 'tools')
371 if os.path.exists(sdk_tools):
372 tooldirs.append(sdk_tools)
373 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
374 if os.path.exists(sdk_platform_tools):
375 tooldirs.append(sdk_platform_tools)
376 tooldirs.append('/usr/bin')
378 path = os.path.join(d, cmd)
379 if os.path.isfile(path):
381 test_aapt_version(path)
383 # did not find the command, exit with error message
384 ensure_build_tools_exists(config)
387 def test_aapt_version(aapt):
388 '''Check whether the version of aapt is new enough'''
389 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
390 if output is None or output == '':
391 logging.error(_("'{path}' failed to execute!").format(path=aapt))
393 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
398 # the Debian package has the version string like "v0.2-23.0.2"
401 if LooseVersion(bugfix) < LooseVersion(MINIMUM_AAPT_VERSION):
403 elif LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.4062713'):
406 logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-{version} or newer!")
407 .format(aapt=aapt, version=MINIMUM_AAPT_VERSION))
409 logging.warning(_('Unknown version of aapt, might cause problems: ') + output)
412 def test_sdk_exists(thisconfig):
413 if 'sdk_path' not in thisconfig:
414 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
415 test_aapt_version(thisconfig['aapt'])
418 logging.error(_("'sdk_path' not set in 'config.py'!"))
420 if thisconfig['sdk_path'] == default_config['sdk_path']:
421 logging.error(_('No Android SDK found!'))
422 logging.error(_('You can use ANDROID_HOME to set the path to your SDK, i.e.:'))
423 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
425 if not os.path.exists(thisconfig['sdk_path']):
426 logging.critical(_("Android SDK path '{path}' does not exist!")
427 .format(path=thisconfig['sdk_path']))
429 if not os.path.isdir(thisconfig['sdk_path']):
430 logging.critical(_("Android SDK path '{path}' is not a directory!")
431 .format(path=thisconfig['sdk_path']))
433 for d in ['build-tools', 'platform-tools', 'tools']:
434 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
435 logging.critical(_("Android SDK '{path}' does not have '{dirname}' installed!")
436 .format(path=thisconfig['sdk_path'], dirname=d))
441 def ensure_build_tools_exists(thisconfig):
442 if not test_sdk_exists(thisconfig):
443 raise FDroidException(_("Android SDK not found!"))
444 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
445 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
446 if not os.path.isdir(versioned_build_tools):
447 raise FDroidException(
448 _("Android build-tools path '{path}' does not exist!")
449 .format(path=versioned_build_tools))
452 def get_local_metadata_files():
453 '''get any metadata files local to an app's source repo
455 This tries to ignore anything that does not count as app metdata,
456 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
459 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
462 def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False):
464 :param appids: arguments in the form of multiple appid:[vc] strings
465 :returns: a dictionary with the set of vercodes specified for each package
468 if not appid_versionCode_pairs:
471 for p in appid_versionCode_pairs:
472 if allow_vercodes and ':' in p:
473 package, vercode = p.split(':')
475 package, vercode = p, None
476 if package not in vercodes:
477 vercodes[package] = [vercode] if vercode else []
479 elif vercode and vercode not in vercodes[package]:
480 vercodes[package] += [vercode] if vercode else []
485 def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
486 """Build a list of App instances for processing
488 On top of what read_pkg_args does, this returns the whole app
489 metadata, but limiting the builds list to the builds matching the
490 appid_versionCode_pairs and vercodes specified. If no appid_versionCode_pairs are specified, then
491 all App and Build instances are returned.
495 vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes)
501 for appid, app in allapps.items():
502 if appid in vercodes:
505 if len(apps) != len(vercodes):
508 logging.critical(_("No such package: %s") % p)
509 raise FDroidException(_("Found invalid appids in arguments"))
511 raise FDroidException(_("No packages specified"))
514 for appid, app in apps.items():
518 app.builds = [b for b in app.builds if b.versionCode in vc]
519 if len(app.builds) != len(vercodes[appid]):
521 allvcs = [b.versionCode for b in app.builds]
522 for v in vercodes[appid]:
524 logging.critical(_("No such versionCode {versionCode} for app {appid}")
525 .format(versionCode=v, appid=appid))
528 raise FDroidException(_("Found invalid versionCodes for some apps"))
533 def get_extension(filename):
534 base, ext = os.path.splitext(filename)
537 return base, ext.lower()[1:]
540 def has_extension(filename, ext):
541 _ignored, f_ext = get_extension(filename)
545 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
548 def clean_description(description):
549 'Remove unneeded newlines and spaces from a block of description text'
551 # this is split up by paragraph to make removing the newlines easier
552 for paragraph in re.split(r'\n\n', description):
553 paragraph = re.sub('\r', '', paragraph)
554 paragraph = re.sub('\n', ' ', paragraph)
555 paragraph = re.sub(' {2,}', ' ', paragraph)
556 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
557 returnstring += paragraph + '\n\n'
558 return returnstring.rstrip('\n')
561 def publishednameinfo(filename):
562 filename = os.path.basename(filename)
563 m = publish_name_regex.match(filename)
565 result = (m.group(1), m.group(2))
566 except AttributeError:
567 raise FDroidException(_("Invalid name for published file: %s") % filename)
571 apk_release_filename = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)\.apk')
572 apk_release_filename_with_sigfp = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)_(?P<sigfp>[0-9a-f]{7})\.apk')
575 def apk_parse_release_filename(apkname):
576 """Parses the name of an APK file according the F-Droids APK naming
577 scheme and returns the tokens.
579 WARNING: Returned values don't necessarily represent the APKs actual
580 properties, the are just paresed from the file name.
582 :returns: A triplet containing (appid, versionCode, signer), where appid
583 should be the package name, versionCode should be the integer
584 represion of the APKs version and signer should be the first 7 hex
585 digists of the sha256 signing key fingerprint which was used to sign
588 m = apk_release_filename_with_sigfp.match(apkname)
590 return m.group('appid'), m.group('vercode'), m.group('sigfp')
591 m = apk_release_filename.match(apkname)
593 return m.group('appid'), m.group('vercode'), None
594 return None, None, None
597 def get_release_filename(app, build):
599 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
601 return "%s_%s.apk" % (app.id, build.versionCode)
604 def get_toolsversion_logname(app, build):
605 return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
608 def getsrcname(app, build):
609 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
621 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
624 def get_build_dir(app):
625 '''get the dir that this app will be built in'''
627 if app.RepoType == 'srclib':
628 return os.path.join('build', 'srclib', app.Repo)
630 return os.path.join('build', app.id)
634 '''checkout code from VCS and return instance of vcs and the build dir'''
635 build_dir = get_build_dir(app)
637 # Set up vcs interface and make sure we have the latest code...
638 logging.debug("Getting {0} vcs interface for {1}"
639 .format(app.RepoType, app.Repo))
640 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
644 vcs = getvcs(app.RepoType, remote, build_dir)
646 return vcs, build_dir
649 def getvcs(vcstype, remote, local):
651 return vcs_git(remote, local)
652 if vcstype == 'git-svn':
653 return vcs_gitsvn(remote, local)
655 return vcs_hg(remote, local)
657 return vcs_bzr(remote, local)
658 if vcstype == 'srclib':
659 if local != os.path.join('build', 'srclib', remote):
660 raise VCSException("Error: srclib paths are hard-coded!")
661 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
663 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
664 raise VCSException("Invalid vcs type " + vcstype)
667 def getsrclibvcs(name):
668 if name not in fdroidserver.metadata.srclibs:
669 raise VCSException("Missing srclib " + name)
670 return fdroidserver.metadata.srclibs[name]['Repo Type']
675 def __init__(self, remote, local):
677 # svn, git-svn and bzr may require auth
679 if self.repotype() in ('git-svn', 'bzr'):
681 if self.repotype == 'git-svn':
682 raise VCSException("Authentication is not supported for git-svn")
683 self.username, remote = remote.split('@')
684 if ':' not in self.username:
685 raise VCSException(_("Password required with username"))
686 self.username, self.password = self.username.split(':')
690 self.clone_failed = False
691 self.refreshed = False
697 def clientversion(self):
698 versionstr = FDroidPopen(self.clientversioncmd()).output
699 return versionstr[0:versionstr.find('\n')]
701 def clientversioncmd(self):
704 def gotorevision(self, rev, refresh=True):
705 """Take the local repository to a clean version of the given
706 revision, which is specificed in the VCS's native
707 format. Beforehand, the repository can be dirty, or even
708 non-existent. If the repository does already exist locally, it
709 will be updated from the origin, but only once in the lifetime
710 of the vcs object. None is acceptable for 'rev' if you know
711 you are cloning a clean copy of the repo - otherwise it must
712 specify a valid revision.
715 if self.clone_failed:
716 raise VCSException(_("Downloading the repository already failed once, not trying again."))
718 # The .fdroidvcs-id file for a repo tells us what VCS type
719 # and remote that directory was created from, allowing us to drop it
720 # automatically if either of those things changes.
721 fdpath = os.path.join(self.local, '..',
722 '.fdroidvcs-' + os.path.basename(self.local))
723 fdpath = os.path.normpath(fdpath)
724 cdata = self.repotype() + ' ' + self.remote
727 if os.path.exists(self.local):
728 if os.path.exists(fdpath):
729 with open(fdpath, 'r') as f:
730 fsdata = f.read().strip()
735 logging.info("Repository details for %s changed - deleting" % (
739 logging.info("Repository details for %s missing - deleting" % (
742 shutil.rmtree(self.local)
746 self.refreshed = True
749 self.gotorevisionx(rev)
750 except FDroidException as e:
753 # If necessary, write the .fdroidvcs file.
754 if writeback and not self.clone_failed:
755 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
756 with open(fdpath, 'w+') as f:
762 def gotorevisionx(self, rev): # pylint: disable=unused-argument
763 """Derived classes need to implement this.
765 It's called once basic checking has been performed.
767 raise VCSException("This VCS type doesn't define gotorevisionx")
769 # Initialise and update submodules
770 def initsubmodules(self):
771 raise VCSException('Submodules not supported for this vcs type')
773 # Get a list of all known tags
775 if not self._gettags:
776 raise VCSException('gettags not supported for this vcs type')
778 for tag in self._gettags():
779 if re.match('[-A-Za-z0-9_. /]+$', tag):
783 def latesttags(self):
784 """Get a list of all the known tags, sorted from newest to oldest"""
785 raise VCSException('latesttags not supported for this vcs type')
788 """Get current commit reference (hash, revision, etc)"""
789 raise VCSException('getref not supported for this vcs type')
792 """Returns the srclib (name, path) used in setting up the current revision, or None."""
801 def clientversioncmd(self):
802 return ['git', '--version']
804 def git(self, args, envs=dict(), cwd=None, output=True):
805 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
807 While fetch/pull/clone respect the command line option flags,
808 it seems that submodule commands do not. They do seem to
809 follow whatever is in env vars, if the version of git is new
810 enough. So we just throw the kitchen sink at it to see what
813 Also, because of CVE-2017-1000117, block all SSH URLs.
816 # supported in git >= 2.3
818 '-c', 'core.askpass=/bin/true',
819 '-c', 'core.sshCommand=/bin/false',
820 '-c', 'url.https://.insteadOf=ssh://',
822 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
823 git_config.append('-c')
824 git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':')
825 git_config.append('-c')
826 git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain)
827 git_config.append('-c')
828 git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain)
830 'GIT_TERMINAL_PROMPT': '0',
831 'GIT_ASKPASS': '/bin/true',
832 'SSH_ASKPASS': '/bin/true',
833 'GIT_SSH': '/bin/false', # for git < 2.3
835 return FDroidPopen(['git', ] + git_config + args,
836 envs=envs, cwd=cwd, output=output)
839 """If the local directory exists, but is somehow not a git repository,
840 git will traverse up the directory tree until it finds one
841 that is (i.e. fdroidserver) and then we'll proceed to destroy
842 it! This is called as a safety check.
846 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
847 result = p.output.rstrip()
848 if not result.endswith(self.local):
849 raise VCSException('Repository mismatch')
851 def gotorevisionx(self, rev):
852 if not os.path.exists(self.local):
854 p = self.git(['clone', '--', self.remote, self.local])
855 if p.returncode != 0:
856 self.clone_failed = True
857 raise VCSException("Git clone failed", p.output)
861 # Discard any working tree changes
862 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
863 'git', 'reset', '--hard'], cwd=self.local, output=False)
864 if p.returncode != 0:
865 raise VCSException(_("Git reset failed"), p.output)
866 # Remove untracked files now, in case they're tracked in the target
867 # revision (it happens!)
868 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
869 'git', 'clean', '-dffx'], cwd=self.local, output=False)
870 if p.returncode != 0:
871 raise VCSException(_("Git clean failed"), p.output)
872 if not self.refreshed:
873 # Get latest commits and tags from remote
874 p = self.git(['fetch', 'origin'], cwd=self.local)
875 if p.returncode != 0:
876 raise VCSException(_("Git fetch failed"), p.output)
877 p = self.git(['fetch', '--prune', '--tags', 'origin'], output=False, cwd=self.local)
878 if p.returncode != 0:
879 raise VCSException(_("Git fetch failed"), p.output)
880 # Recreate origin/HEAD as git clone would do it, in case it disappeared
881 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
882 if p.returncode != 0:
883 lines = p.output.splitlines()
884 if 'Multiple remote HEAD branches' not in lines[0]:
885 raise VCSException(_("Git remote set-head failed"), p.output)
886 branch = lines[1].split(' ')[-1]
887 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--', branch],
888 cwd=self.local, output=False)
889 if p2.returncode != 0:
890 raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output)
891 self.refreshed = True
892 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
893 # a github repo. Most of the time this is the same as origin/master.
894 rev = rev or 'origin/HEAD'
895 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
896 if p.returncode != 0:
897 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
898 # Get rid of any uncontrolled files left behind
899 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
900 if p.returncode != 0:
901 raise VCSException(_("Git clean failed"), p.output)
903 def initsubmodules(self):
905 submfile = os.path.join(self.local, '.gitmodules')
906 if not os.path.isfile(submfile):
907 raise NoSubmodulesException(_("No git submodules available"))
909 # fix submodules not accessible without an account and public key auth
910 with open(submfile, 'r') as f:
911 lines = f.readlines()
912 with open(submfile, 'w') as f:
914 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
915 line = re.sub('git@' + domain + ':', 'https://u:p@' + domain + '/', line)
918 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
919 if p.returncode != 0:
920 raise VCSException(_("Git submodule sync failed"), p.output)
921 p = self.git(['submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
922 if p.returncode != 0:
923 raise VCSException(_("Git submodule update failed"), p.output)
927 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
928 return p.output.splitlines()
930 tag_format = re.compile(r'tag: ([^),]*)')
932 def latesttags(self):
934 p = FDroidPopen(['git', 'log', '--tags',
935 '--simplify-by-decoration', '--pretty=format:%d'],
936 cwd=self.local, output=False)
938 for line in p.output.splitlines():
939 for tag in self.tag_format.findall(line):
944 class vcs_gitsvn(vcs):
949 def clientversioncmd(self):
950 return ['git', 'svn', '--version']
953 """If the local directory exists, but is somehow not a git repository,
954 git will traverse up the directory tree until it finds one that
955 is (i.e. fdroidserver) and then we'll proceed to destory it!
956 This is called as a safety check.
959 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
960 result = p.output.rstrip()
961 if not result.endswith(self.local):
962 raise VCSException('Repository mismatch')
964 def git(self, args, envs=dict(), cwd=None, output=True):
965 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
967 AskPass is set to /bin/true to let the process try to connect
968 without a username/password.
970 The SSH command is set to /bin/false to block all SSH URLs
971 (supported in git >= 2.3). This protects against
976 '-c', 'core.askpass=/bin/true',
977 '-c', 'core.sshCommand=/bin/false',
980 'GIT_TERMINAL_PROMPT': '0',
981 'GIT_ASKPASS': '/bin/true',
982 'SSH_ASKPASS': '/bin/true',
983 'GIT_SSH': '/bin/false', # for git < 2.3
984 'SVN_SSH': '/bin/false',
986 return FDroidPopen(['git', ] + git_config + args,
987 envs=envs, cwd=cwd, output=output)
989 def gotorevisionx(self, rev):
990 if not os.path.exists(self.local):
992 gitsvn_args = ['svn', 'clone']
994 if ';' in self.remote:
995 remote_split = self.remote.split(';')
996 for i in remote_split[1:]:
997 if i.startswith('trunk='):
998 gitsvn_args.extend(['-T', i[6:]])
999 elif i.startswith('tags='):
1000 gitsvn_args.extend(['-t', i[5:]])
1001 elif i.startswith('branches='):
1002 gitsvn_args.extend(['-b', i[9:]])
1003 remote = remote_split[0]
1005 remote = self.remote
1007 if not remote.startswith('https://'):
1008 raise VCSException(_('HTTPS must be used with Subversion URLs!'))
1010 gitsvn_args.extend(['--', remote, self.local])
1011 p = self.git(gitsvn_args)
1012 if p.returncode != 0:
1013 self.clone_failed = True
1014 raise VCSException(_('git svn clone failed'), p.output)
1018 # Discard any working tree changes
1019 p = self.git(['reset', '--hard'], cwd=self.local, output=False)
1020 if p.returncode != 0:
1021 raise VCSException("Git reset failed", p.output)
1022 # Remove untracked files now, in case they're tracked in the target
1023 # revision (it happens!)
1024 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1025 if p.returncode != 0:
1026 raise VCSException("Git clean failed", p.output)
1027 if not self.refreshed:
1028 # Get new commits, branches and tags from repo
1029 p = self.git(['svn', 'fetch'], cwd=self.local, output=False)
1030 if p.returncode != 0:
1031 raise VCSException("Git svn fetch failed")
1032 p = self.git(['svn', 'rebase'], cwd=self.local, output=False)
1033 if p.returncode != 0:
1034 raise VCSException("Git svn rebase failed", p.output)
1035 self.refreshed = True
1037 rev = rev or 'master'
1039 nospaces_rev = rev.replace(' ', '%20')
1040 # Try finding a svn tag
1041 for treeish in ['origin/', '']:
1042 p = self.git(['checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
1043 if p.returncode == 0:
1045 if p.returncode != 0:
1046 # No tag found, normal svn rev translation
1047 # Translate svn rev into git format
1048 rev_split = rev.split('/')
1051 for treeish in ['origin/', '']:
1052 if len(rev_split) > 1:
1053 treeish += rev_split[0]
1054 svn_rev = rev_split[1]
1057 # if no branch is specified, then assume trunk (i.e. 'master' branch):
1061 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
1063 p = self.git(['svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
1064 git_rev = p.output.rstrip()
1066 if p.returncode == 0 and git_rev:
1069 if p.returncode != 0 or not git_rev:
1070 # Try a plain git checkout as a last resort
1071 p = self.git(['checkout', rev], cwd=self.local, output=False)
1072 if p.returncode != 0:
1073 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
1075 # Check out the git rev equivalent to the svn rev
1076 p = self.git(['checkout', git_rev], cwd=self.local, output=False)
1077 if p.returncode != 0:
1078 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
1080 # Get rid of any uncontrolled files left behind
1081 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1082 if p.returncode != 0:
1083 raise VCSException(_("Git clean failed"), p.output)
1087 for treeish in ['origin/', '']:
1088 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
1089 if os.path.isdir(d):
1090 return os.listdir(d)
1094 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
1095 if p.returncode != 0:
1097 return p.output.strip()
1105 def clientversioncmd(self):
1106 return ['hg', '--version']
1108 def gotorevisionx(self, rev):
1109 if not os.path.exists(self.local):
1110 p = FDroidPopen(['hg', 'clone', '--ssh', 'false', '--', self.remote, self.local],
1112 if p.returncode != 0:
1113 self.clone_failed = True
1114 raise VCSException("Hg clone failed", p.output)
1116 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
1117 if p.returncode != 0:
1118 raise VCSException("Hg status failed", p.output)
1119 for line in p.output.splitlines():
1120 if not line.startswith('? '):
1121 raise VCSException("Unexpected output from hg status -uS: " + line)
1122 FDroidPopen(['rm', '-rf', '--', line[2:]], cwd=self.local, output=False)
1123 if not self.refreshed:
1124 p = FDroidPopen(['hg', 'pull', '--ssh', 'false'], cwd=self.local, output=False)
1125 if p.returncode != 0:
1126 raise VCSException("Hg pull failed", p.output)
1127 self.refreshed = True
1129 rev = rev or 'default'
1132 p = FDroidPopen(['hg', 'update', '-C', '--', rev], cwd=self.local, output=False)
1133 if p.returncode != 0:
1134 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
1135 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1136 # Also delete untracked files, we have to enable purge extension for that:
1137 if "'purge' is provided by the following extension" in p.output:
1138 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
1139 myfile.write("\n[extensions]\nhgext.purge=\n")
1140 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1141 if p.returncode != 0:
1142 raise VCSException("HG purge failed", p.output)
1143 elif p.returncode != 0:
1144 raise VCSException("HG purge failed", p.output)
1147 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
1148 return p.output.splitlines()[1:]
1156 def clientversioncmd(self):
1157 return ['bzr', '--version']
1159 def bzr(self, args, envs=dict(), cwd=None, output=True):
1160 '''Prevent bzr from ever using SSH to avoid security vulns'''
1164 return FDroidPopen(['bzr', ] + args, envs=envs, cwd=cwd, output=output)
1166 def gotorevisionx(self, rev):
1167 if not os.path.exists(self.local):
1168 p = self.bzr(['branch', self.remote, self.local], output=False)
1169 if p.returncode != 0:
1170 self.clone_failed = True
1171 raise VCSException("Bzr branch failed", p.output)
1173 p = self.bzr(['clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
1174 if p.returncode != 0:
1175 raise VCSException("Bzr revert failed", p.output)
1176 if not self.refreshed:
1177 p = self.bzr(['pull'], cwd=self.local, output=False)
1178 if p.returncode != 0:
1179 raise VCSException("Bzr update failed", p.output)
1180 self.refreshed = True
1182 revargs = list(['-r', rev] if rev else [])
1183 p = self.bzr(['revert'] + revargs, cwd=self.local, output=False)
1184 if p.returncode != 0:
1185 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1188 p = self.bzr(['tags'], cwd=self.local, output=False)
1189 return [tag.split(' ')[0].strip() for tag in
1190 p.output.splitlines()]
1193 def unescape_string(string):
1196 if string[0] == '"' and string[-1] == '"':
1199 return string.replace("\\'", "'")
1202 def retrieve_string(app_dir, string, xmlfiles=None):
1204 if not string.startswith('@string/'):
1205 return unescape_string(string)
1207 if xmlfiles is None:
1210 os.path.join(app_dir, 'res'),
1211 os.path.join(app_dir, 'src', 'main', 'res'),
1213 for root, dirs, files in os.walk(res_dir):
1214 if os.path.basename(root) == 'values':
1215 xmlfiles += [os.path.join(root, x) for x in files if x.endswith('.xml')]
1217 name = string[len('@string/'):]
1219 def element_content(element):
1220 if element.text is None:
1222 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1223 return s.decode('utf-8').strip()
1225 for path in xmlfiles:
1226 if not os.path.isfile(path):
1228 xml = parse_xml(path)
1229 element = xml.find('string[@name="' + name + '"]')
1230 if element is not None:
1231 content = element_content(element)
1232 return retrieve_string(app_dir, content, xmlfiles)
1237 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1238 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1241 def manifest_paths(app_dir, flavours):
1242 '''Return list of existing files that will be used to find the highest vercode'''
1244 possible_manifests = \
1245 [os.path.join(app_dir, 'AndroidManifest.xml'),
1246 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1247 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1248 os.path.join(app_dir, 'build.gradle')]
1250 for flavour in flavours:
1251 if flavour == 'yes':
1253 possible_manifests.append(
1254 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1256 return [path for path in possible_manifests if os.path.isfile(path)]
1259 def fetch_real_name(app_dir, flavours):
1260 '''Retrieve the package name. Returns the name, or None if not found.'''
1261 for path in manifest_paths(app_dir, flavours):
1262 if not has_extension(path, 'xml') or not os.path.isfile(path):
1264 logging.debug("fetch_real_name: Checking manifest at " + path)
1265 xml = parse_xml(path)
1266 app = xml.find('application')
1269 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1271 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1272 result = retrieve_string_singleline(app_dir, label)
1274 result = result.strip()
1279 def get_library_references(root_dir):
1281 proppath = os.path.join(root_dir, 'project.properties')
1282 if not os.path.isfile(proppath):
1284 with open(proppath, 'r', encoding='iso-8859-1') as f:
1286 if not line.startswith('android.library.reference.'):
1288 path = line.split('=')[1].strip()
1289 relpath = os.path.join(root_dir, path)
1290 if not os.path.isdir(relpath):
1292 logging.debug("Found subproject at %s" % path)
1293 libraries.append(path)
1297 def ant_subprojects(root_dir):
1298 subprojects = get_library_references(root_dir)
1299 for subpath in subprojects:
1300 subrelpath = os.path.join(root_dir, subpath)
1301 for p in get_library_references(subrelpath):
1302 relp = os.path.normpath(os.path.join(subpath, p))
1303 if relp not in subprojects:
1304 subprojects.insert(0, relp)
1308 def remove_debuggable_flags(root_dir):
1309 # Remove forced debuggable flags
1310 logging.debug("Removing debuggable flags from %s" % root_dir)
1311 for root, dirs, files in os.walk(root_dir):
1312 if 'AndroidManifest.xml' in files and os.path.isfile(os.path.join(root, 'AndroidManifest.xml')):
1313 regsub_file(r'android:debuggable="[^"]*"',
1315 os.path.join(root, 'AndroidManifest.xml'))
1318 vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search
1319 vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1320 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1323 def app_matches_packagename(app, package):
1326 appid = app.UpdateCheckName or app.id
1327 if appid is None or appid == "Ignore":
1329 return appid == package
1332 def parse_androidmanifests(paths, app):
1334 Extract some information from the AndroidManifest.xml at the given path.
1335 Returns (version, vercode, package), any or all of which might be None.
1336 All values returned are strings.
1339 ignoreversions = app.UpdateCheckIgnore
1340 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1343 return (None, None, None)
1351 if not os.path.isfile(path):
1354 logging.debug(_("Parsing manifest at '{path}'").format(path=path))
1360 if app.builds and 'gradle' in app.builds[-1] and app.builds[-1].gradle:
1361 flavour = app.builds[-1].gradle[-1]
1363 if has_extension(path, 'gradle'):
1364 with open(path, 'r') as f:
1365 inside_flavour_group = 0
1366 inside_required_flavour = 0
1368 if gradle_comment.match(line):
1371 if inside_flavour_group > 0:
1372 if inside_required_flavour > 0:
1373 matches = psearch_g(line)
1375 s = matches.group(2)
1376 if app_matches_packagename(app, s):
1379 matches = vnsearch_g(line)
1381 version = matches.group(2)
1383 matches = vcsearch_g(line)
1385 vercode = matches.group(1)
1388 inside_required_flavour += 1
1390 inside_required_flavour -= 1
1392 if flavour and (flavour in line):
1393 inside_required_flavour = 1
1396 inside_flavour_group += 1
1398 inside_flavour_group -= 1
1400 if "productFlavors" in line:
1401 inside_flavour_group = 1
1403 matches = psearch_g(line)
1405 s = matches.group(2)
1406 if app_matches_packagename(app, s):
1409 matches = vnsearch_g(line)
1411 version = matches.group(2)
1413 matches = vcsearch_g(line)
1415 vercode = matches.group(1)
1418 xml = parse_xml(path)
1419 if "package" in xml.attrib:
1420 s = xml.attrib["package"]
1421 if app_matches_packagename(app, s):
1423 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1424 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1425 base_dir = os.path.dirname(path)
1426 version = retrieve_string_singleline(base_dir, version)
1427 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1428 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1429 if string_is_integer(a):
1432 logging.warning(_("Problem with xml at '{path}'").format(path=path))
1434 # Remember package name, may be defined separately from version+vercode
1436 package = max_package
1438 logging.debug("..got package={0}, version={1}, vercode={2}"
1439 .format(package, version, vercode))
1441 # Always grab the package name and version name in case they are not
1442 # together with the highest version code
1443 if max_package is None and package is not None:
1444 max_package = package
1445 if max_version is None and version is not None:
1446 max_version = version
1448 if vercode is not None \
1449 and (max_vercode is None or vercode > max_vercode):
1450 if not ignoresearch or not ignoresearch(version):
1451 if version is not None:
1452 max_version = version
1453 if vercode is not None:
1454 max_vercode = vercode
1455 if package is not None:
1456 max_package = package
1458 max_version = "Ignore"
1460 if max_version is None:
1461 max_version = "Unknown"
1463 if max_package and not is_valid_package_name(max_package):
1464 raise FDroidException(_("Invalid package name {0}").format(max_package))
1466 return (max_version, max_vercode, max_package)
1469 def is_valid_package_name(name):
1470 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1473 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1474 raw=False, prepare=True, preponly=False, refresh=True,
1476 """Get the specified source library.
1478 Returns the path to it. Normally this is the path to be used when
1479 referencing it, which may be a subdirectory of the actual project. If
1480 you want the base directory of the project, pass 'basepath=True'.
1489 name, ref = spec.split('@')
1491 number, name = name.split(':', 1)
1493 name, subdir = name.split('/', 1)
1495 if name not in fdroidserver.metadata.srclibs:
1496 raise VCSException('srclib ' + name + ' not found.')
1498 srclib = fdroidserver.metadata.srclibs[name]
1500 sdir = os.path.join(srclib_dir, name)
1503 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1504 vcs.srclib = (name, number, sdir)
1506 vcs.gotorevision(ref, refresh)
1513 libdir = os.path.join(sdir, subdir)
1514 elif srclib["Subdir"]:
1515 for subdir in srclib["Subdir"]:
1516 libdir_candidate = os.path.join(sdir, subdir)
1517 if os.path.exists(libdir_candidate):
1518 libdir = libdir_candidate
1524 remove_signing_keys(sdir)
1525 remove_debuggable_flags(sdir)
1529 if srclib["Prepare"]:
1530 cmd = replace_config_vars(srclib["Prepare"], build)
1532 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=libdir)
1533 if p.returncode != 0:
1534 raise BuildException("Error running prepare command for srclib %s"
1540 return (name, number, libdir)
1543 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1546 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1547 """ Prepare the source code for a particular build
1549 :param vcs: the appropriate vcs object for the application
1550 :param app: the application details from the metadata
1551 :param build: the build details from the metadata
1552 :param build_dir: the path to the build directory, usually 'build/app.id'
1553 :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
1554 :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
1556 Returns the (root, srclibpaths) where:
1557 :param root: is the root directory, which may be the same as 'build_dir' or may
1558 be a subdirectory of it.
1559 :param srclibpaths: is information on the srclibs being used
1562 # Optionally, the actual app source can be in a subdirectory
1564 root_dir = os.path.join(build_dir, build.subdir)
1566 root_dir = build_dir
1568 # Get a working copy of the right revision
1569 logging.info("Getting source for revision " + build.commit)
1570 vcs.gotorevision(build.commit, refresh)
1572 # Initialise submodules if required
1573 if build.submodules:
1574 logging.info(_("Initialising submodules"))
1575 vcs.initsubmodules()
1577 # Check that a subdir (if we're using one) exists. This has to happen
1578 # after the checkout, since it might not exist elsewhere
1579 if not os.path.exists(root_dir):
1580 raise BuildException('Missing subdir ' + root_dir)
1582 # Run an init command if one is required
1584 cmd = replace_config_vars(build.init, build)
1585 logging.info("Running 'init' commands in %s" % root_dir)
1587 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir)
1588 if p.returncode != 0:
1589 raise BuildException("Error running init command for %s:%s" %
1590 (app.id, build.versionName), p.output)
1592 # Apply patches if any
1594 logging.info("Applying patches")
1595 for patch in build.patch:
1596 patch = patch.strip()
1597 logging.info("Applying " + patch)
1598 patch_path = os.path.join('metadata', app.id, patch)
1599 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1600 if p.returncode != 0:
1601 raise BuildException("Failed to apply patch %s" % patch_path)
1603 # Get required source libraries
1606 logging.info("Collecting source libraries")
1607 for lib in build.srclibs:
1608 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1609 refresh=refresh, build=build))
1611 for name, number, libpath in srclibpaths:
1612 place_srclib(root_dir, int(number) if number else None, libpath)
1614 basesrclib = vcs.getsrclib()
1615 # If one was used for the main source, add that too.
1617 srclibpaths.append(basesrclib)
1619 # Update the local.properties file
1620 localprops = [os.path.join(build_dir, 'local.properties')]
1622 parts = build.subdir.split(os.sep)
1625 cur = os.path.join(cur, d)
1626 localprops += [os.path.join(cur, 'local.properties')]
1627 for path in localprops:
1629 if os.path.isfile(path):
1630 logging.info("Updating local.properties file at %s" % path)
1631 with open(path, 'r', encoding='iso-8859-1') as f:
1635 logging.info("Creating local.properties file at %s" % path)
1636 # Fix old-fashioned 'sdk-location' by copying
1637 # from sdk.dir, if necessary
1639 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1640 re.S | re.M).group(1)
1641 props += "sdk-location=%s\n" % sdkloc
1643 props += "sdk.dir=%s\n" % config['sdk_path']
1644 props += "sdk-location=%s\n" % config['sdk_path']
1645 ndk_path = build.ndk_path()
1646 # if for any reason the path isn't valid or the directory
1647 # doesn't exist, some versions of Gradle will error with a
1648 # cryptic message (even if the NDK is not even necessary).
1649 # https://gitlab.com/fdroid/fdroidserver/issues/171
1650 if ndk_path and os.path.exists(ndk_path):
1652 props += "ndk.dir=%s\n" % ndk_path
1653 props += "ndk-location=%s\n" % ndk_path
1654 # Add java.encoding if necessary
1656 props += "java.encoding=%s\n" % build.encoding
1657 with open(path, 'w', encoding='iso-8859-1') as f:
1661 if build.build_method() == 'gradle':
1662 flavours = build.gradle
1665 n = build.target.split('-')[1]
1666 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1667 r'compileSdkVersion %s' % n,
1668 os.path.join(root_dir, 'build.gradle'))
1670 # Remove forced debuggable flags
1671 remove_debuggable_flags(root_dir)
1673 # Insert version code and number into the manifest if necessary
1674 if build.forceversion:
1675 logging.info("Changing the version name")
1676 for path in manifest_paths(root_dir, flavours):
1677 if not os.path.isfile(path):
1679 if has_extension(path, 'xml'):
1680 regsub_file(r'android:versionName="[^"]*"',
1681 r'android:versionName="%s"' % build.versionName,
1683 elif has_extension(path, 'gradle'):
1684 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1685 r"""\1versionName '%s'""" % build.versionName,
1688 if build.forcevercode:
1689 logging.info("Changing the version code")
1690 for path in manifest_paths(root_dir, flavours):
1691 if not os.path.isfile(path):
1693 if has_extension(path, 'xml'):
1694 regsub_file(r'android:versionCode="[^"]*"',
1695 r'android:versionCode="%s"' % build.versionCode,
1697 elif has_extension(path, 'gradle'):
1698 regsub_file(r'versionCode[ =]+[0-9]+',
1699 r'versionCode %s' % build.versionCode,
1702 # Delete unwanted files
1704 logging.info(_("Removing specified files"))
1705 for part in getpaths(build_dir, build.rm):
1706 dest = os.path.join(build_dir, part)
1707 logging.info("Removing {0}".format(part))
1708 if os.path.lexists(dest):
1709 # rmtree can only handle directories that are not symlinks, so catch anything else
1710 if not os.path.isdir(dest) or os.path.islink(dest):
1715 logging.info("...but it didn't exist")
1717 remove_signing_keys(build_dir)
1719 # Add required external libraries
1721 logging.info("Collecting prebuilt libraries")
1722 libsdir = os.path.join(root_dir, 'libs')
1723 if not os.path.exists(libsdir):
1725 for lib in build.extlibs:
1727 logging.info("...installing extlib {0}".format(lib))
1728 libf = os.path.basename(lib)
1729 libsrc = os.path.join(extlib_dir, lib)
1730 if not os.path.exists(libsrc):
1731 raise BuildException("Missing extlib file {0}".format(libsrc))
1732 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1734 # Run a pre-build command if one is required
1736 logging.info("Running 'prebuild' commands in %s" % root_dir)
1738 cmd = replace_config_vars(build.prebuild, build)
1740 # Substitute source library paths into prebuild commands
1741 for name, number, libpath in srclibpaths:
1742 libpath = os.path.relpath(libpath, root_dir)
1743 cmd = cmd.replace('$$' + name + '$$', libpath)
1745 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir)
1746 if p.returncode != 0:
1747 raise BuildException("Error running prebuild command for %s:%s" %
1748 (app.id, build.versionName), p.output)
1750 # Generate (or update) the ant build file, build.xml...
1751 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1752 parms = ['android', 'update', 'lib-project']
1753 lparms = ['android', 'update', 'project']
1756 parms += ['-t', build.target]
1757 lparms += ['-t', build.target]
1758 if build.androidupdate:
1759 update_dirs = build.androidupdate
1761 update_dirs = ant_subprojects(root_dir) + ['.']
1763 for d in update_dirs:
1764 subdir = os.path.join(root_dir, d)
1766 logging.debug("Updating main project")
1767 cmd = parms + ['-p', d]
1769 logging.debug("Updating subproject %s" % d)
1770 cmd = lparms + ['-p', d]
1771 p = SdkToolsPopen(cmd, cwd=root_dir)
1772 # Check to see whether an error was returned without a proper exit
1773 # code (this is the case for the 'no target set or target invalid'
1775 if p.returncode != 0 or p.output.startswith("Error: "):
1776 raise BuildException("Failed to update project at %s" % d, p.output)
1777 # Clean update dirs via ant
1779 logging.info("Cleaning subproject %s" % d)
1780 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1782 return (root_dir, srclibpaths)
1785 def getpaths_map(build_dir, globpaths):
1786 """Extend via globbing the paths from a field and return them as a map from original path to resulting paths"""
1790 full_path = os.path.join(build_dir, p)
1791 full_path = os.path.normpath(full_path)
1792 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1794 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1798 def getpaths(build_dir, globpaths):
1799 """Extend via globbing the paths from a field and return them as a set"""
1800 paths_map = getpaths_map(build_dir, globpaths)
1802 for k, v in paths_map.items():
1809 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1812 def check_system_clock(dt_obj, path):
1813 """Check if system clock is updated based on provided date
1815 If an APK has files newer than the system time, suggest updating
1816 the system clock. This is useful for offline systems, used for
1817 signing, which do not have another source of clock sync info. It
1818 has to be more than 24 hours newer because ZIP/APK files do not
1822 checkdt = dt_obj - timedelta(1)
1823 if datetime.today() < checkdt:
1824 logging.warning(_('System clock is older than date in {path}!').format(path=path)
1825 + '\n' + _('Set clock to that time using:') + '\n'
1826 + 'sudo date -s "' + str(dt_obj) + '"')
1830 """permanent store of existing APKs with the date they were added
1832 This is currently the only way to permanently store the "updated"
1837 '''Load filename/date info about previously seen APKs
1839 Since the appid and date strings both will never have spaces,
1840 this is parsed as a list from the end to allow the filename to
1841 have any combo of spaces.
1844 self.path = os.path.join('stats', 'known_apks.txt')
1846 if os.path.isfile(self.path):
1847 with open(self.path, 'r', encoding='utf8') as f:
1849 t = line.rstrip().split(' ')
1851 self.apks[t[0]] = (t[1], None)
1854 date = datetime.strptime(t[-1], '%Y-%m-%d')
1855 filename = line[0:line.rfind(appid) - 1]
1856 self.apks[filename] = (appid, date)
1857 check_system_clock(date, self.path)
1858 self.changed = False
1860 def writeifchanged(self):
1861 if not self.changed:
1864 if not os.path.exists('stats'):
1868 for apk, app in self.apks.items():
1870 line = apk + ' ' + appid
1872 line += ' ' + added.strftime('%Y-%m-%d')
1875 with open(self.path, 'w', encoding='utf8') as f:
1876 for line in sorted(lst, key=natural_key):
1877 f.write(line + '\n')
1879 def recordapk(self, apkName, app, default_date=None):
1881 Record an apk (if it's new, otherwise does nothing)
1882 Returns the date it was added as a datetime instance
1884 if apkName not in self.apks:
1885 if default_date is None:
1886 default_date = datetime.utcnow()
1887 self.apks[apkName] = (app, default_date)
1889 _ignored, added = self.apks[apkName]
1892 def getapp(self, apkname):
1893 """Look up information - given the 'apkname', returns (app id, date added/None).
1895 Or returns None for an unknown apk.
1897 if apkname in self.apks:
1898 return self.apks[apkname]
1901 def getlatest(self, num):
1902 """Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first"""
1904 for apk, app in self.apks.items():
1908 if apps[appid] > added:
1912 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1913 lst = [app for app, _ignored in sortedapps]
1918 def get_file_extension(filename):
1919 """get the normalized file extension, can be blank string but never None"""
1920 if isinstance(filename, bytes):
1921 filename = filename.decode('utf-8')
1922 return os.path.splitext(filename)[1].lower()[1:]
1925 def get_apk_debuggable_aapt(apkfile):
1926 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1928 if p.returncode != 0:
1929 raise FDroidException(_("Failed to get APK manifest information"))
1930 for line in p.output.splitlines():
1931 if 'android:debuggable' in line and not line.endswith('0x0'):
1936 def get_apk_debuggable_androguard(apkfile):
1938 from androguard.core.bytecodes.apk import APK
1940 raise FDroidException("androguard library is not installed and aapt not present")
1942 apkobject = APK(apkfile)
1943 if apkobject.is_valid_APK():
1944 debuggable = apkobject.get_element("application", "debuggable")
1945 if debuggable is not None:
1946 return bool(strtobool(debuggable))
1950 def isApkAndDebuggable(apkfile):
1951 """Returns True if the given file is an APK and is debuggable
1953 :param apkfile: full path to the apk to check"""
1955 if get_file_extension(apkfile) != 'apk':
1958 if SdkToolsPopen(['aapt', 'version'], output=False):
1959 return get_apk_debuggable_aapt(apkfile)
1961 return get_apk_debuggable_androguard(apkfile)
1964 def get_apk_id_aapt(apkfile):
1965 """Extrat identification information from APK using aapt.
1967 :param apkfile: path to an APK file.
1968 :returns: triplet (appid, version code, version name)
1970 r = re.compile("package: name='(?P<appid>.*)' versionCode='(?P<vercode>.*)' versionName='(?P<vername>.*)' platformBuildVersionName='.*'")
1971 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1972 for line in p.output.splitlines():
1975 return m.group('appid'), m.group('vercode'), m.group('vername')
1976 raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'")
1977 .format(apkfilename=apkfile))
1980 def get_minSdkVersion_aapt(apkfile):
1981 """Extract the minimum supported Android SDK from an APK using aapt
1983 :param apkfile: path to an APK file.
1984 :returns: the integer representing the SDK version
1986 r = re.compile(r"^sdkVersion:'([0-9]+)'")
1987 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1988 for line in p.output.splitlines():
1991 return int(m.group(1))
1992 raise FDroidException(_('Reading minSdkVersion failed: "{apkfilename}"')
1993 .format(apkfilename=apkfile))
1998 self.returncode = None
2002 def SdkToolsPopen(commands, cwd=None, output=True):
2004 if cmd not in config:
2005 config[cmd] = find_sdk_tools_cmd(commands[0])
2006 abscmd = config[cmd]
2008 raise FDroidException(_("Could not find '{command}' on your system").format(command=cmd))
2010 test_aapt_version(config['aapt'])
2011 return FDroidPopen([abscmd] + commands[1:],
2012 cwd=cwd, output=output)
2015 def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2017 Run a command and capture the possibly huge output as bytes.
2019 :param commands: command and argument list like in subprocess.Popen
2020 :param cwd: optionally specifies a working directory
2021 :param envs: a optional dictionary of environment variables and their values
2022 :returns: A PopenResult.
2027 set_FDroidPopen_env()
2029 process_env = env.copy()
2030 if envs is not None and len(envs) > 0:
2031 process_env.update(envs)
2034 cwd = os.path.normpath(cwd)
2035 logging.debug("Directory: %s" % cwd)
2036 logging.debug("> %s" % ' '.join(commands))
2038 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
2039 result = PopenResult()
2042 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
2043 stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
2044 stderr=stderr_param)
2045 except OSError as e:
2046 raise BuildException("OSError while trying to execute " +
2047 ' '.join(commands) + ': ' + str(e))
2049 # TODO are these AsynchronousFileReader threads always exiting?
2050 if not stderr_to_stdout and options.verbose:
2051 stderr_queue = Queue()
2052 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
2054 while not stderr_reader.eof():
2055 while not stderr_queue.empty():
2056 line = stderr_queue.get()
2057 sys.stderr.buffer.write(line)
2062 stdout_queue = Queue()
2063 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
2066 # Check the queue for output (until there is no more to get)
2067 while not stdout_reader.eof():
2068 while not stdout_queue.empty():
2069 line = stdout_queue.get()
2070 if output and options.verbose:
2071 # Output directly to console
2072 sys.stderr.buffer.write(line)
2078 result.returncode = p.wait()
2079 result.output = buf.getvalue()
2081 # make sure all filestreams of the subprocess are closed
2082 for streamvar in ['stdin', 'stdout', 'stderr']:
2083 if hasattr(p, streamvar):
2084 stream = getattr(p, streamvar)
2090 def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2092 Run a command and capture the possibly huge output as a str.
2094 :param commands: command and argument list like in subprocess.Popen
2095 :param cwd: optionally specifies a working directory
2096 :param envs: a optional dictionary of environment variables and their values
2097 :returns: A PopenResult.
2099 result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
2100 result.output = result.output.decode('utf-8', 'ignore')
2104 gradle_comment = re.compile(r'[ ]*//')
2105 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
2106 gradle_line_matches = [
2107 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
2108 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
2109 re.compile(r'.*\.readLine\(.*'),
2113 def remove_signing_keys(build_dir):
2114 for root, dirs, files in os.walk(build_dir):
2115 if 'build.gradle' in files:
2116 path = os.path.join(root, 'build.gradle')
2118 with open(path, "r", encoding='utf8') as o:
2119 lines = o.readlines()
2125 with open(path, "w", encoding='utf8') as o:
2126 while i < len(lines):
2129 while line.endswith('\\\n'):
2130 line = line.rstrip('\\\n') + lines[i]
2133 if gradle_comment.match(line):
2138 opened += line.count('{')
2139 opened -= line.count('}')
2142 if gradle_signing_configs.match(line):
2147 if any(s.match(line) for s in gradle_line_matches):
2155 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
2158 'project.properties',
2160 'default.properties',
2161 'ant.properties', ]:
2162 if propfile in files:
2163 path = os.path.join(root, propfile)
2165 with open(path, "r", encoding='iso-8859-1') as o:
2166 lines = o.readlines()
2170 with open(path, "w", encoding='iso-8859-1') as o:
2172 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
2179 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
2182 def set_FDroidPopen_env(build=None):
2184 set up the environment variables for the build environment
2186 There is only a weak standard, the variables used by gradle, so also set
2187 up the most commonly used environment variables for SDK and NDK. Also, if
2188 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
2190 global env, orig_path
2194 orig_path = env['PATH']
2195 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
2196 env[n] = config['sdk_path']
2197 for k, v in config['java_paths'].items():
2198 env['JAVA%s_HOME' % k] = v
2200 missinglocale = True
2201 for k, v in env.items():
2202 if k == 'LANG' and v != 'C':
2203 missinglocale = False
2205 missinglocale = False
2207 env['LANG'] = 'en_US.UTF-8'
2209 if build is not None:
2210 path = build.ndk_path()
2211 paths = orig_path.split(os.pathsep)
2212 if path not in paths:
2213 paths = [path] + paths
2214 env['PATH'] = os.pathsep.join(paths)
2215 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
2216 env[n] = build.ndk_path()
2219 def replace_build_vars(cmd, build):
2220 cmd = cmd.replace('$$COMMIT$$', build.commit)
2221 cmd = cmd.replace('$$VERSION$$', build.versionName)
2222 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
2226 def replace_config_vars(cmd, build):
2227 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
2228 cmd = cmd.replace('$$NDK$$', build.ndk_path())
2229 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
2230 if build is not None:
2231 cmd = replace_build_vars(cmd, build)
2235 def place_srclib(root_dir, number, libpath):
2238 relpath = os.path.relpath(libpath, root_dir)
2239 proppath = os.path.join(root_dir, 'project.properties')
2242 if os.path.isfile(proppath):
2243 with open(proppath, "r", encoding='iso-8859-1') as o:
2244 lines = o.readlines()
2246 with open(proppath, "w", encoding='iso-8859-1') as o:
2249 if line.startswith('android.library.reference.%d=' % number):
2250 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2255 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2258 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
2261 def signer_fingerprint_short(sig):
2262 """Obtain shortened sha256 signing-key fingerprint for pkcs7 signature.
2264 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2265 for a given pkcs7 signature.
2267 :param sig: Contents of an APK signing certificate.
2268 :returns: shortened signing-key fingerprint.
2270 return signer_fingerprint(sig)[:7]
2273 def signer_fingerprint(sig):
2274 """Obtain sha256 signing-key fingerprint for pkcs7 signature.
2276 Extracts hexadecimal sha256 signing-key fingerprint string
2277 for a given pkcs7 signature.
2279 :param: Contents of an APK signature.
2280 :returns: shortened signature fingerprint.
2282 cert_encoded = get_certificate(sig)
2283 return hashlib.sha256(cert_encoded).hexdigest()
2286 def apk_signer_fingerprint(apk_path):
2287 """Obtain sha256 signing-key fingerprint for APK.
2289 Extracts hexadecimal sha256 signing-key fingerprint string
2292 :param apkpath: path to APK
2293 :returns: signature fingerprint
2296 with zipfile.ZipFile(apk_path, 'r') as apk:
2297 certs = [n for n in apk.namelist() if CERT_PATH_REGEX.match(n)]
2300 logging.error("Found no signing certificates on %s" % apk_path)
2303 logging.error("Found multiple signing certificates on %s" % apk_path)
2306 cert = apk.read(certs[0])
2307 return signer_fingerprint(cert)
2310 def apk_signer_fingerprint_short(apk_path):
2311 """Obtain shortened sha256 signing-key fingerprint for APK.
2313 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2314 for a given pkcs7 APK.
2316 :param apk_path: path to APK
2317 :returns: shortened signing-key fingerprint
2319 return apk_signer_fingerprint(apk_path)[:7]
2322 def metadata_get_sigdir(appid, vercode=None):
2323 """Get signature directory for app"""
2325 return os.path.join('metadata', appid, 'signatures', vercode)
2327 return os.path.join('metadata', appid, 'signatures')
2330 def metadata_find_developer_signature(appid, vercode=None):
2331 """Tires to find the developer signature for given appid.
2333 This picks the first signature file found in metadata an returns its
2336 :returns: sha256 signing key fingerprint of the developer signing key.
2337 None in case no signature can not be found."""
2339 # fetch list of dirs for all versions of signatures
2342 appversigdirs.append(metadata_get_sigdir(appid, vercode))
2344 appsigdir = metadata_get_sigdir(appid)
2345 if os.path.isdir(appsigdir):
2346 numre = re.compile('[0-9]+')
2347 for ver in os.listdir(appsigdir):
2348 if numre.match(ver):
2349 appversigdir = os.path.join(appsigdir, ver)
2350 appversigdirs.append(appversigdir)
2352 for sigdir in appversigdirs:
2353 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2354 glob.glob(os.path.join(sigdir, '*.EC')) + \
2355 glob.glob(os.path.join(sigdir, '*.RSA'))
2357 raise FDroidException('ambiguous signatures, please make sure there is only one signature in \'{}\'. (The signature has to be the App maintainers signature for version of the APK.)'.format(sigdir))
2359 with open(sig, 'rb') as f:
2360 return signer_fingerprint(f.read())
2364 def metadata_find_signing_files(appid, vercode):
2365 """Gets a list of singed manifests and signatures.
2367 :param appid: app id string
2368 :param vercode: app version code
2369 :returns: a list of triplets for each signing key with following paths:
2370 (signature_file, singed_file, manifest_file)
2373 sigdir = metadata_get_sigdir(appid, vercode)
2374 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2375 glob.glob(os.path.join(sigdir, '*.EC')) + \
2376 glob.glob(os.path.join(sigdir, '*.RSA'))
2377 extre = re.compile('(\.DSA|\.EC|\.RSA)$')
2379 sf = extre.sub('.SF', sig)
2380 if os.path.isfile(sf):
2381 mf = os.path.join(sigdir, 'MANIFEST.MF')
2382 if os.path.isfile(mf):
2383 ret.append((sig, sf, mf))
2387 def metadata_find_developer_signing_files(appid, vercode):
2388 """Get developer signature files for specified app from metadata.
2390 :returns: A triplet of paths for signing files from metadata:
2391 (signature_file, singed_file, manifest_file)
2393 allsigningfiles = metadata_find_signing_files(appid, vercode)
2394 if allsigningfiles and len(allsigningfiles) == 1:
2395 return allsigningfiles[0]
2400 def apk_strip_signatures(signed_apk, strip_manifest=False):
2401 """Removes signatures from APK.
2403 :param signed_apk: path to apk file.
2404 :param strip_manifest: when set to True also the manifest file will
2405 be removed from the APK.
2407 with tempfile.TemporaryDirectory() as tmpdir:
2408 tmp_apk = os.path.join(tmpdir, 'tmp.apk')
2409 shutil.move(signed_apk, tmp_apk)
2410 with ZipFile(tmp_apk, 'r') as in_apk:
2411 with ZipFile(signed_apk, 'w') as out_apk:
2412 for info in in_apk.infolist():
2413 if not apk_sigfile.match(info.filename):
2415 if info.filename != 'META-INF/MANIFEST.MF':
2416 buf = in_apk.read(info.filename)
2417 out_apk.writestr(info, buf)
2419 buf = in_apk.read(info.filename)
2420 out_apk.writestr(info, buf)
2423 def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest):
2424 """Implats a signature from metadata into an APK.
2426 Note: this changes there supplied APK in place. So copy it if you
2427 need the original to be preserved.
2429 :param apkpath: location of the apk
2431 # get list of available signature files in metadata
2432 with tempfile.TemporaryDirectory() as tmpdir:
2433 apkwithnewsig = os.path.join(tmpdir, 'newsig.apk')
2434 with ZipFile(apkpath, 'r') as in_apk:
2435 with ZipFile(apkwithnewsig, 'w') as out_apk:
2436 for sig_file in [signaturefile, signedfile, manifest]:
2437 with open(sig_file, 'rb') as fp:
2439 info = zipfile.ZipInfo('META-INF/' + os.path.basename(sig_file))
2440 info.compress_type = zipfile.ZIP_DEFLATED
2441 info.create_system = 0 # "Windows" aka "FAT", what Android SDK uses
2442 out_apk.writestr(info, buf)
2443 for info in in_apk.infolist():
2444 if not apk_sigfile.match(info.filename):
2445 if info.filename != 'META-INF/MANIFEST.MF':
2446 buf = in_apk.read(info.filename)
2447 out_apk.writestr(info, buf)
2449 p = SdkToolsPopen(['zipalign', '-v', '4', apkwithnewsig, apkpath])
2450 if p.returncode != 0:
2451 raise BuildException("Failed to align application")
2454 def apk_extract_signatures(apkpath, outdir, manifest=True):
2455 """Extracts a signature files from APK and puts them into target directory.
2457 :param apkpath: location of the apk
2458 :param outdir: folder where the extracted signature files will be stored
2459 :param manifest: (optionally) disable extracting manifest file
2461 with ZipFile(apkpath, 'r') as in_apk:
2462 for f in in_apk.infolist():
2463 if apk_sigfile.match(f.filename) or \
2464 (manifest and f.filename == 'META-INF/MANIFEST.MF'):
2465 newpath = os.path.join(outdir, os.path.basename(f.filename))
2466 with open(newpath, 'wb') as out_file:
2467 out_file.write(in_apk.read(f.filename))
2470 def sign_apk(unsigned_path, signed_path, keyalias):
2471 """Sign and zipalign an unsigned APK, then save to a new file, deleting the unsigned
2473 android-18 (4.3) finally added support for reasonable hash
2474 algorithms, like SHA-256, before then, the only options were MD5
2475 and SHA1 :-/ This aims to use SHA-256 when the APK does not target
2476 older Android versions, and is therefore safe to do so.
2478 https://issuetracker.google.com/issues/36956587
2479 https://android-review.googlesource.com/c/platform/libcore/+/44491
2483 if get_minSdkVersion_aapt(unsigned_path) < 18:
2484 signature_algorithm = ['-sigalg', 'SHA1withRSA', '-digestalg', 'SHA1']
2486 signature_algorithm = ['-sigalg', 'SHA256withRSA', '-digestalg', 'SHA-256']
2488 p = FDroidPopen([config['jarsigner'], '-keystore', config['keystore'],
2489 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2490 '-keypass:env', 'FDROID_KEY_PASS']
2491 + signature_algorithm + [unsigned_path, keyalias],
2493 'FDROID_KEY_STORE_PASS': config['keystorepass'],
2494 'FDROID_KEY_PASS': config['keypass'], })
2495 if p.returncode != 0:
2496 raise BuildException(_("Failed to sign application"), p.output)
2498 p = SdkToolsPopen(['zipalign', '-v', '4', unsigned_path, signed_path])
2499 if p.returncode != 0:
2500 raise BuildException(_("Failed to zipalign application"))
2501 os.remove(unsigned_path)
2504 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
2505 """Verify that two apks are the same
2507 One of the inputs is signed, the other is unsigned. The signature metadata
2508 is transferred from the signed to the unsigned apk, and then jarsigner is
2509 used to verify that the signature from the signed apk is also varlid for
2510 the unsigned one. If the APK given as unsigned actually does have a
2511 signature, it will be stripped out and ignored.
2513 There are two SHA1 git commit IDs that fdroidserver includes in the builds
2514 it makes: fdroidserverid and buildserverid. Originally, these were inserted
2515 into AndroidManifest.xml, but that makes the build not reproducible. So
2516 instead they are included as separate files in the APK's META-INF/ folder.
2517 If those files exist in the signed APK, they will be part of the signature
2518 and need to also be included in the unsigned APK for it to validate.
2520 :param signed_apk: Path to a signed apk file
2521 :param unsigned_apk: Path to an unsigned apk file expected to match it
2522 :param tmp_dir: Path to directory for temporary files
2523 :returns: None if the verification is successful, otherwise a string
2524 describing what went wrong.
2527 if not os.path.isfile(signed_apk):
2528 return 'can not verify: file does not exists: {}'.format(signed_apk)
2530 if not os.path.isfile(unsigned_apk):
2531 return 'can not verify: file does not exists: {}'.format(unsigned_apk)
2533 with ZipFile(signed_apk, 'r') as signed:
2534 meta_inf_files = ['META-INF/MANIFEST.MF']
2535 for f in signed.namelist():
2536 if apk_sigfile.match(f) \
2537 or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
2538 meta_inf_files.append(f)
2539 if len(meta_inf_files) < 3:
2540 return "Signature files missing from {0}".format(signed_apk)
2542 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2543 with ZipFile(unsigned_apk, 'r') as unsigned:
2544 # only read the signature from the signed APK, everything else from unsigned
2545 with ZipFile(tmp_apk, 'w') as tmp:
2546 for filename in meta_inf_files:
2547 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2548 for info in unsigned.infolist():
2549 if info.filename in meta_inf_files:
2550 logging.warning('Ignoring %s from %s',
2551 info.filename, unsigned_apk)
2553 if info.filename in tmp.namelist():
2554 return "duplicate filename found: " + info.filename
2555 tmp.writestr(info, unsigned.read(info.filename))
2557 verified = verify_apk_signature(tmp_apk)
2560 logging.info("...NOT verified - {0}".format(tmp_apk))
2561 return compare_apks(signed_apk, tmp_apk, tmp_dir,
2562 os.path.dirname(unsigned_apk))
2564 logging.info("...successfully verified")
2568 def verify_jar_signature(jar):
2569 """Verifies the signature of a given JAR file.
2571 jarsigner is very shitty: unsigned JARs pass as "verified"! So
2572 this has to turn on -strict then check for result 4, since this
2573 does not expect the signature to be from a CA-signed certificate.
2575 :raises: VerificationException() if the JAR's signature could not be verified
2579 error = _('JAR signature failed to verify: {path}').format(path=jar)
2581 output = subprocess.check_output([config['jarsigner'], '-strict', '-verify', jar],
2582 stderr=subprocess.STDOUT)
2583 raise VerificationException(error + '\n' + output.decode('utf-8'))
2584 except subprocess.CalledProcessError as e:
2585 if e.returncode == 4:
2586 logging.debug(_('JAR signature verified: {path}').format(path=jar))
2588 raise VerificationException(error + '\n' + e.output.decode('utf-8'))
2591 def verify_apk_signature(apk, min_sdk_version=None):
2592 """verify the signature on an APK
2594 Try to use apksigner whenever possible since jarsigner is very
2595 shitty: unsigned APKs pass as "verified"! Warning, this does
2596 not work on JARs with apksigner >= 0.7 (build-tools 26.0.1)
2598 :returns: boolean whether the APK was verified
2600 if set_command_in_config('apksigner'):
2601 args = [config['apksigner'], 'verify']
2603 args += ['--min-sdk-version=' + min_sdk_version]
2605 args += ['--verbose']
2607 output = subprocess.check_output(args + [apk])
2609 logging.debug(apk + ': ' + output.decode('utf-8'))
2611 except subprocess.CalledProcessError as e:
2612 logging.error('\n' + apk + ': ' + e.output.decode('utf-8'))
2614 if not config.get('jarsigner_warning_displayed'):
2615 config['jarsigner_warning_displayed'] = True
2616 logging.warning(_("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner"))
2618 verify_jar_signature(apk)
2620 except Exception as e:
2625 def verify_old_apk_signature(apk):
2626 """verify the signature on an archived APK, supporting deprecated algorithms
2628 F-Droid aims to keep every single binary that it ever published. Therefore,
2629 it needs to be able to verify APK signatures that include deprecated/removed
2630 algorithms. For example, jarsigner treats an MD5 signature as unsigned.
2632 jarsigner passes unsigned APKs as "verified"! So this has to turn
2633 on -strict then check for result 4.
2635 :returns: boolean whether the APK was verified
2638 _java_security = os.path.join(os.getcwd(), '.java.security')
2639 with open(_java_security, 'w') as fp:
2640 fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
2644 config['jarsigner'],
2645 '-J-Djava.security.properties=' + _java_security,
2646 '-strict', '-verify', apk
2648 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
2649 except subprocess.CalledProcessError as e:
2650 if e.returncode != 4:
2653 logging.debug(_('JAR signature verified: {path}').format(path=apk))
2656 logging.error(_('Old APK signature failed to verify: {path}').format(path=apk)
2657 + '\n' + output.decode('utf-8'))
2661 apk_badchars = re.compile('''[/ :;'"]''')
2664 def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
2667 Returns None if the apk content is the same (apart from the signing key),
2668 otherwise a string describing what's different, or what went wrong when
2669 trying to do the comparison.
2675 absapk1 = os.path.abspath(apk1)
2676 absapk2 = os.path.abspath(apk2)
2678 if set_command_in_config('diffoscope'):
2679 logfilename = os.path.join(log_dir, os.path.basename(absapk1))
2680 htmlfile = logfilename + '.diffoscope.html'
2681 textfile = logfilename + '.diffoscope.txt'
2682 if subprocess.call([config['diffoscope'],
2683 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2684 '--html', htmlfile, '--text', textfile,
2685 absapk1, absapk2]) != 0:
2686 return("Failed to unpack " + apk1)
2688 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2689 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2690 for d in [apk1dir, apk2dir]:
2691 if os.path.exists(d):
2694 os.mkdir(os.path.join(d, 'jar-xf'))
2696 if subprocess.call(['jar', 'xf',
2697 os.path.abspath(apk1)],
2698 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2699 return("Failed to unpack " + apk1)
2700 if subprocess.call(['jar', 'xf',
2701 os.path.abspath(apk2)],
2702 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2703 return("Failed to unpack " + apk2)
2705 if set_command_in_config('apktool'):
2706 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2708 return("Failed to unpack " + apk1)
2709 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2711 return("Failed to unpack " + apk2)
2713 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2714 lines = p.output.splitlines()
2715 if len(lines) != 1 or 'META-INF' not in lines[0]:
2716 if set_command_in_config('meld'):
2717 p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
2718 return("Unexpected diff output - " + p.output)
2720 # since everything verifies, delete the comparison to keep cruft down
2721 shutil.rmtree(apk1dir)
2722 shutil.rmtree(apk2dir)
2724 # If we get here, it seems like they're the same!
2728 def set_command_in_config(command):
2729 '''Try to find specified command in the path, if it hasn't been
2730 manually set in config.py. If found, it is added to the config
2731 dict. The return value says whether the command is available.
2734 if command in config:
2737 tmp = find_command(command)
2739 config[command] = tmp
2744 def find_command(command):
2745 '''find the full path of a command, or None if it can't be found in the PATH'''
2748 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2750 fpath, fname = os.path.split(command)
2755 for path in os.environ["PATH"].split(os.pathsep):
2756 path = path.strip('"')
2757 exe_file = os.path.join(path, command)
2758 if is_exe(exe_file):
2765 '''generate a random password for when generating keys'''
2766 h = hashlib.sha256()
2767 h.update(os.urandom(16)) # salt
2768 h.update(socket.getfqdn().encode('utf-8'))
2769 passwd = base64.b64encode(h.digest()).strip()
2770 return passwd.decode('utf-8')
2773 def genkeystore(localconfig):
2775 Generate a new key with password provided in :param localconfig and add it to new keystore
2776 :return: hexed public key, public key fingerprint
2778 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2779 keystoredir = os.path.dirname(localconfig['keystore'])
2780 if keystoredir is None or keystoredir == '':
2781 keystoredir = os.path.join(os.getcwd(), keystoredir)
2782 if not os.path.exists(keystoredir):
2783 os.makedirs(keystoredir, mode=0o700)
2786 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
2787 'FDROID_KEY_PASS': localconfig['keypass'],
2789 p = FDroidPopen([config['keytool'], '-genkey',
2790 '-keystore', localconfig['keystore'],
2791 '-alias', localconfig['repo_keyalias'],
2792 '-keyalg', 'RSA', '-keysize', '4096',
2793 '-sigalg', 'SHA256withRSA',
2794 '-validity', '10000',
2795 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2796 '-keypass:env', 'FDROID_KEY_PASS',
2797 '-dname', localconfig['keydname']], envs=env_vars)
2798 if p.returncode != 0:
2799 raise BuildException("Failed to generate key", p.output)
2800 os.chmod(localconfig['keystore'], 0o0600)
2801 if not options.quiet:
2802 # now show the lovely key that was just generated
2803 p = FDroidPopen([config['keytool'], '-list', '-v',
2804 '-keystore', localconfig['keystore'],
2805 '-alias', localconfig['repo_keyalias'],
2806 '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
2807 logging.info(p.output.strip() + '\n\n')
2808 # get the public key
2809 p = FDroidPopenBytes([config['keytool'], '-exportcert',
2810 '-keystore', localconfig['keystore'],
2811 '-alias', localconfig['repo_keyalias'],
2812 '-storepass:env', 'FDROID_KEY_STORE_PASS']
2813 + config['smartcardoptions'],
2814 envs=env_vars, output=False, stderr_to_stdout=False)
2815 if p.returncode != 0 or len(p.output) < 20:
2816 raise BuildException("Failed to get public key", p.output)
2818 fingerprint = get_cert_fingerprint(pubkey)
2819 return hexlify(pubkey), fingerprint
2822 def get_cert_fingerprint(pubkey):
2824 Generate a certificate fingerprint the same way keytool does it
2825 (but with slightly different formatting)
2827 digest = hashlib.sha256(pubkey).digest()
2828 ret = [' '.join("%02X" % b for b in bytearray(digest))]
2829 return " ".join(ret)
2832 def get_certificate(certificate_file):
2834 Extracts a certificate from the given file.
2835 :param certificate_file: file bytes (as string) representing the certificate
2836 :return: A binary representation of the certificate's public key, or None in case of error
2838 content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
2839 if content.getComponentByName('contentType') != rfc2315.signedData:
2841 content = decoder.decode(content.getComponentByName('content'),
2842 asn1Spec=rfc2315.SignedData())[0]
2844 certificates = content.getComponentByName('certificates')
2845 cert = certificates[0].getComponentByName('certificate')
2847 logging.error("Certificates not found.")
2849 return encoder.encode(cert)
2852 def load_stats_fdroid_signing_key_fingerprints():
2853 """Load list of signing-key fingerprints stored by fdroid publish from file.
2855 :returns: list of dictionanryies containing the singing-key fingerprints.
2857 jar_file = os.path.join('stats', 'publishsigkeys.jar')
2858 if not os.path.isfile(jar_file):
2860 cmd = [config['jarsigner'], '-strict', '-verify', jar_file]
2861 p = FDroidPopen(cmd, output=False)
2862 if p.returncode != 4:
2863 raise FDroidException("Signature validation of '{}' failed! "
2864 "Please run publish again to rebuild this file.".format(jar_file))
2866 jar_sigkey = apk_signer_fingerprint(jar_file)
2867 repo_key_sig = config.get('repo_key_sha256')
2869 if jar_sigkey != repo_key_sig:
2870 raise FDroidException("Signature key fingerprint of file '{}' does not match repo_key_sha256 in config.py (found fingerprint: '{}')".format(jar_file, jar_sigkey))
2872 logging.warning("repo_key_sha256 not in config.py, setting it to the signature key fingerprint of '{}'".format(jar_file))
2873 config['repo_key_sha256'] = jar_sigkey
2874 write_to_config(config, 'repo_key_sha256')
2876 with zipfile.ZipFile(jar_file, 'r') as f:
2877 return json.loads(str(f.read('publishsigkeys.json'), 'utf-8'))
2880 def write_to_config(thisconfig, key, value=None, config_file=None):
2881 '''write a key/value to the local config.py
2883 NOTE: only supports writing string variables.
2885 :param thisconfig: config dictionary
2886 :param key: variable name in config.py to be overwritten/added
2887 :param value: optional value to be written, instead of fetched
2888 from 'thisconfig' dictionary.
2891 origkey = key + '_orig'
2892 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2893 cfg = config_file if config_file else 'config.py'
2895 # load config file, create one if it doesn't exist
2896 if not os.path.exists(cfg):
2897 open(cfg, 'a').close()
2898 logging.info("Creating empty " + cfg)
2899 with open(cfg, 'r', encoding="utf-8") as f:
2900 lines = f.readlines()
2902 # make sure the file ends with a carraige return
2904 if not lines[-1].endswith('\n'):
2907 # regex for finding and replacing python string variable
2908 # definitions/initializations
2909 pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
2910 repl = key + ' = "' + value + '"'
2911 pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
2912 repl2 = key + " = '" + value + "'"
2914 # If we replaced this line once, we make sure won't be a
2915 # second instance of this line for this key in the document.
2918 with open(cfg, 'w', encoding="utf-8") as f:
2920 if pattern.match(line) or pattern2.match(line):
2922 line = pattern.sub(repl, line)
2923 line = pattern2.sub(repl2, line)
2934 def parse_xml(path):
2935 return XMLElementTree.parse(path).getroot()
2938 def string_is_integer(string):
2946 def local_rsync(options, fromdir, todir):
2947 '''Rsync method for local to local copying of things
2949 This is an rsync wrapper with all the settings for safe use within
2950 the various fdroidserver use cases. This uses stricter rsync
2951 checking on all files since people using offline mode are already
2952 prioritizing security above ease and speed.
2955 rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
2956 '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
2957 if not options.no_checksum:
2958 rsyncargs.append('--checksum')
2960 rsyncargs += ['--verbose']
2962 rsyncargs += ['--quiet']
2963 logging.debug(' '.join(rsyncargs + [fromdir, todir]))
2964 if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
2965 raise FDroidException()
2968 def get_per_app_repos():
2969 '''per-app repos are dirs named with the packageName of a single app'''
2971 # Android packageNames are Java packages, they may contain uppercase or
2972 # lowercase letters ('A' through 'Z'), numbers, and underscores
2973 # ('_'). However, individual package name parts may only start with
2974 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2975 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2978 for root, dirs, files in os.walk(os.getcwd()):
2980 print('checking', root, 'for', d)
2981 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2982 # standard parts of an fdroid repo, so never packageNames
2985 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
2991 def is_repo_file(filename):
2992 '''Whether the file in a repo is a build product to be delivered to users'''
2993 if isinstance(filename, str):
2994 filename = filename.encode('utf-8', errors="surrogateescape")
2995 return os.path.isfile(filename) \
2996 and not filename.endswith(b'.asc') \
2997 and not filename.endswith(b'.sig') \
2998 and os.path.basename(filename) not in [
3000 b'index_unsigned.jar',
3009 def get_examples_dir():
3010 '''Return the dir where the fdroidserver example files are available'''
3012 tmp = os.path.dirname(sys.argv[0])
3013 if os.path.basename(tmp) == 'bin':
3014 egg_links = glob.glob(os.path.join(tmp, '..',
3015 'local/lib/python3.*/site-packages/fdroidserver.egg-link'))
3017 # installed from local git repo
3018 examplesdir = os.path.join(open(egg_links[0]).readline().rstrip(), 'examples')
3021 examplesdir = os.path.dirname(os.path.dirname(__file__)) + '/share/doc/fdroidserver/examples'
3022 if not os.path.exists(examplesdir): # use UNIX layout
3023 examplesdir = os.path.dirname(tmp) + '/share/doc/fdroidserver/examples'
3025 # we're running straight out of the git repo
3026 prefix = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
3027 examplesdir = prefix + '/examples'
3032 def get_wiki_timestamp(timestamp=None):
3033 """Return current time in the standard format for posting to the wiki"""
3035 if timestamp is None:
3036 timestamp = time.gmtime()
3037 return time.strftime("%Y-%m-%d %H:%M:%SZ", timestamp)
3040 def get_android_tools_versions(ndk_path=None):
3041 '''get a list of the versions of all installed Android SDK/NDK components'''
3044 sdk_path = config['sdk_path']
3045 if sdk_path[-1] != '/':
3049 ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT')
3050 if os.path.isfile(ndk_release_txt):
3051 with open(ndk_release_txt, 'r') as fp:
3052 components.append((os.path.basename(ndk_path), fp.read()[:-1]))
3054 pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE)
3055 for root, dirs, files in os.walk(sdk_path):
3056 if 'source.properties' in files:
3057 source_properties = os.path.join(root, 'source.properties')
3058 with open(source_properties, 'r') as fp:
3059 m = pattern.search(fp.read())
3061 components.append((root[len(sdk_path):], m.group(1)))
3066 def get_android_tools_version_log(ndk_path=None):
3067 '''get a list of the versions of all installed Android SDK/NDK components'''
3068 log = '== Installed Android Tools ==\n\n'
3069 components = get_android_tools_versions(ndk_path)
3070 for name, version in sorted(components):
3071 log += '* ' + name + ' (' + version + ')\n'