3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
40 import xml.etree.ElementTree as XMLElementTree
42 from binascii import hexlify
43 from datetime import datetime, timedelta
44 from distutils.version import LooseVersion
45 from queue import Queue
46 from zipfile import ZipFile
48 from pyasn1.codec.der import decoder, encoder
49 from pyasn1_modules import rfc2315
50 from pyasn1.error import PyAsn1Error
52 from distutils.util import strtobool
54 import fdroidserver.metadata
55 from fdroidserver import _
56 from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException,\
57 BuildException, VerificationException
58 from .asynchronousfilereader import AsynchronousFileReader
60 # this is the build-tools version, aapt has a separate version that
61 # has to be manually set in test_aapt_version()
62 MINIMUM_AAPT_VERSION = '26.0.0'
64 # A signature block file with a .DSA, .RSA, or .EC extension
65 CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
66 APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
67 STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
69 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
78 'sdk_path': "$ANDROID_HOME",
82 'r12b': "$ANDROID_NDK",
88 'build_tools': MINIMUM_AAPT_VERSION,
89 'force_build_tools': False,
94 'accepted_formats': ['txt', 'yml'],
95 'sync_from_local_copy_dir': False,
96 'allow_disabled_algorithms': False,
97 'per_app_repos': False,
98 'make_current_version_link': True,
99 'current_version_name_source': 'Name',
100 'update_stats': False,
102 'stats_server': None,
104 'stats_to_carbon': False,
106 'build_server_always': False,
107 'keystore': 'keystore.jks',
108 'smartcardoptions': [],
118 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
119 'repo_name': "My First FDroid Repo Demo",
120 'repo_icon': "fdroid-icon.png",
121 'repo_description': '''
122 This is a repository of apps to be used with FDroid. Applications in this
123 repository are either official binaries built by the original application
124 developers, or are binaries built from source by the admin of f-droid.org
125 using the tools on https://gitlab.com/u/fdroid.
131 def setup_global_opts(parser):
132 try: # the buildserver VM might not have PIL installed
133 from PIL import PngImagePlugin
134 logger = logging.getLogger(PngImagePlugin.__name__)
135 logger.setLevel(logging.INFO) # tame the "STREAM" debug messages
139 parser.add_argument("-v", "--verbose", action="store_true", default=False,
140 help=_("Spew out even more information than normal"))
141 parser.add_argument("-q", "--quiet", action="store_true", default=False,
142 help=_("Restrict output to warnings and errors"))
145 def _add_java_paths_to_config(pathlist, thisconfig):
146 def path_version_key(s):
148 for u in re.split('[^0-9]+', s):
150 versionlist.append(int(u))
155 for d in sorted(pathlist, key=path_version_key):
156 if os.path.islink(d):
158 j = os.path.basename(d)
159 # the last one found will be the canonical one, so order appropriately
161 r'^1\.([6-9])\.0\.jdk$', # OSX
162 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
163 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
164 r'^jdk([6-9])-openjdk$', # Arch
165 r'^java-([6-9])-openjdk$', # Arch
166 r'^java-([6-9])-jdk$', # Arch (oracle)
167 r'^java-1\.([6-9])\.0-.*$', # RedHat
168 r'^java-([6-9])-oracle$', # Debian WebUpd8
169 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
170 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
172 m = re.match(regex, j)
175 for p in [d, os.path.join(d, 'Contents', 'Home')]:
176 if os.path.exists(os.path.join(p, 'bin', 'javac')):
177 thisconfig['java_paths'][m.group(1)] = p
180 def fill_config_defaults(thisconfig):
181 for k, v in default_config.items():
182 if k not in thisconfig:
185 # Expand paths (~users and $vars)
186 def expand_path(path):
190 path = os.path.expanduser(path)
191 path = os.path.expandvars(path)
196 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
201 thisconfig[k + '_orig'] = v
203 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
204 if thisconfig['java_paths'] is None:
205 thisconfig['java_paths'] = dict()
207 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
208 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
209 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
210 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
211 if os.getenv('JAVA_HOME') is not None:
212 pathlist.append(os.getenv('JAVA_HOME'))
213 if os.getenv('PROGRAMFILES') is not None:
214 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
215 _add_java_paths_to_config(pathlist, thisconfig)
217 for java_version in ('7', '8', '9'):
218 if java_version not in thisconfig['java_paths']:
220 java_home = thisconfig['java_paths'][java_version]
221 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
222 if os.path.exists(jarsigner):
223 thisconfig['jarsigner'] = jarsigner
224 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
225 break # Java7 is preferred, so quit if found
227 for k in ['ndk_paths', 'java_paths']:
233 thisconfig[k][k2] = exp
234 thisconfig[k][k2 + '_orig'] = v
237 def regsub_file(pattern, repl, path):
238 with open(path, 'rb') as f:
240 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
241 with open(path, 'wb') as f:
245 def read_config(opts, config_file='config.py'):
246 """Read the repository config
248 The config is read from config_file, which is in the current
249 directory when any of the repo management commands are used. If
250 there is a local metadata file in the git repo, then config.py is
251 not required, just use defaults.
254 global config, options
256 if config is not None:
263 if os.path.isfile(config_file):
264 logging.debug(_("Reading '{config_file}'").format(config_file=config_file))
265 with io.open(config_file, "rb") as f:
266 code = compile(f.read(), config_file, 'exec')
267 exec(code, None, config)
269 logging.warning(_("No 'config.py' found, using defaults."))
271 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
273 if not type(config[k]) in (str, list, tuple):
275 _("'{field}' will be in random order! Use () or [] brackets if order is important!")
278 # smartcardoptions must be a list since its command line args for Popen
279 if 'smartcardoptions' in config:
280 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
281 elif 'keystore' in config and config['keystore'] == 'NONE':
282 # keystore='NONE' means use smartcard, these are required defaults
283 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
284 'SunPKCS11-OpenSC', '-providerClass',
285 'sun.security.pkcs11.SunPKCS11',
286 '-providerArg', 'opensc-fdroid.cfg']
288 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
289 st = os.stat(config_file)
290 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
291 logging.warning(_("unsafe permissions on '{config_file}' (should be 0600)!")
292 .format(config_file=config_file))
294 fill_config_defaults(config)
296 for k in ["repo_description", "archive_description"]:
298 config[k] = clean_description(config[k])
300 if 'serverwebroot' in config:
301 if isinstance(config['serverwebroot'], str):
302 roots = [config['serverwebroot']]
303 elif all(isinstance(item, str) for item in config['serverwebroot']):
304 roots = config['serverwebroot']
306 raise TypeError(_('only accepts strings, lists, and tuples'))
308 for rootstr in roots:
309 # since this is used with rsync, where trailing slashes have
310 # meaning, ensure there is always a trailing slash
311 if rootstr[-1] != '/':
313 rootlist.append(rootstr.replace('//', '/'))
314 config['serverwebroot'] = rootlist
316 if 'servergitmirrors' in config:
317 if isinstance(config['servergitmirrors'], str):
318 roots = [config['servergitmirrors']]
319 elif all(isinstance(item, str) for item in config['servergitmirrors']):
320 roots = config['servergitmirrors']
322 raise TypeError(_('only accepts strings, lists, and tuples'))
323 config['servergitmirrors'] = roots
328 def assert_config_keystore(config):
329 """Check weather keystore is configured correctly and raise exception if not."""
332 if 'repo_keyalias' not in config:
334 logging.critical(_("'repo_keyalias' not found in config.py!"))
335 if 'keystore' not in config:
337 logging.critical(_("'keystore' not found in config.py!"))
338 elif not os.path.exists(config['keystore']):
340 logging.critical("'" + config['keystore'] + "' does not exist!")
341 if 'keystorepass' not in config:
343 logging.critical(_("'keystorepass' not found in config.py!"))
344 if 'keypass' not in config:
346 logging.critical(_("'keypass' not found in config.py!"))
348 raise FDroidException("This command requires a signing key, " +
349 "you can create one using: fdroid update --create-key")
352 def find_sdk_tools_cmd(cmd):
353 '''find a working path to a tool from the Android SDK'''
356 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
357 # try to find a working path to this command, in all the recent possible paths
358 if 'build_tools' in config:
359 build_tools = os.path.join(config['sdk_path'], 'build-tools')
360 # if 'build_tools' was manually set and exists, check only that one
361 configed_build_tools = os.path.join(build_tools, config['build_tools'])
362 if os.path.exists(configed_build_tools):
363 tooldirs.append(configed_build_tools)
365 # no configed version, so hunt known paths for it
366 for f in sorted(os.listdir(build_tools), reverse=True):
367 if os.path.isdir(os.path.join(build_tools, f)):
368 tooldirs.append(os.path.join(build_tools, f))
369 tooldirs.append(build_tools)
370 sdk_tools = os.path.join(config['sdk_path'], 'tools')
371 if os.path.exists(sdk_tools):
372 tooldirs.append(sdk_tools)
373 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
374 if os.path.exists(sdk_platform_tools):
375 tooldirs.append(sdk_platform_tools)
376 tooldirs.append('/usr/bin')
378 path = os.path.join(d, cmd)
379 if os.path.isfile(path):
381 test_aapt_version(path)
383 # did not find the command, exit with error message
384 ensure_build_tools_exists(config)
387 def test_aapt_version(aapt):
388 '''Check whether the version of aapt is new enough'''
389 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
390 if output is None or output == '':
391 logging.error(_("'{path}' failed to execute!").format(path=aapt))
393 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
398 # the Debian package has the version string like "v0.2-23.0.2"
401 if LooseVersion(bugfix) < LooseVersion(MINIMUM_AAPT_VERSION):
403 elif LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.4062713'):
406 logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-{version} or newer!")
407 .format(aapt=aapt, version=MINIMUM_AAPT_VERSION))
409 logging.warning(_('Unknown version of aapt, might cause problems: ') + output)
412 def test_sdk_exists(thisconfig):
413 if 'sdk_path' not in thisconfig:
414 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
415 test_aapt_version(thisconfig['aapt'])
418 logging.error(_("'sdk_path' not set in 'config.py'!"))
420 if thisconfig['sdk_path'] == default_config['sdk_path']:
421 logging.error(_('No Android SDK found!'))
422 logging.error(_('You can use ANDROID_HOME to set the path to your SDK, i.e.:'))
423 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
425 if not os.path.exists(thisconfig['sdk_path']):
426 logging.critical(_("Android SDK path '{path}' does not exist!")
427 .format(path=thisconfig['sdk_path']))
429 if not os.path.isdir(thisconfig['sdk_path']):
430 logging.critical(_("Android SDK path '{path}' is not a directory!")
431 .format(path=thisconfig['sdk_path']))
433 for d in ['build-tools', 'platform-tools', 'tools']:
434 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
435 logging.critical(_("Android SDK '{path}' does not have '{dirname}' installed!")
436 .format(path=thisconfig['sdk_path'], dirname=d))
441 def ensure_build_tools_exists(thisconfig):
442 if not test_sdk_exists(thisconfig):
443 raise FDroidException(_("Android SDK not found!"))
444 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
445 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
446 if not os.path.isdir(versioned_build_tools):
447 raise FDroidException(
448 _("Android build-tools path '{path}' does not exist!")
449 .format(path=versioned_build_tools))
452 def get_local_metadata_files():
453 '''get any metadata files local to an app's source repo
455 This tries to ignore anything that does not count as app metdata,
456 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
459 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
462 def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False):
464 :param appids: arguments in the form of multiple appid:[vc] strings
465 :returns: a dictionary with the set of vercodes specified for each package
468 if not appid_versionCode_pairs:
471 for p in appid_versionCode_pairs:
472 if allow_vercodes and ':' in p:
473 package, vercode = p.split(':')
475 package, vercode = p, None
476 if package not in vercodes:
477 vercodes[package] = [vercode] if vercode else []
479 elif vercode and vercode not in vercodes[package]:
480 vercodes[package] += [vercode] if vercode else []
485 def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
486 """Build a list of App instances for processing
488 On top of what read_pkg_args does, this returns the whole app
489 metadata, but limiting the builds list to the builds matching the
490 appid_versionCode_pairs and vercodes specified. If no appid_versionCode_pairs are specified, then
491 all App and Build instances are returned.
495 vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes)
501 for appid, app in allapps.items():
502 if appid in vercodes:
505 if len(apps) != len(vercodes):
508 logging.critical(_("No such package: %s") % p)
509 raise FDroidException(_("Found invalid appids in arguments"))
511 raise FDroidException(_("No packages specified"))
514 for appid, app in apps.items():
518 app.builds = [b for b in app.builds if b.versionCode in vc]
519 if len(app.builds) != len(vercodes[appid]):
521 allvcs = [b.versionCode for b in app.builds]
522 for v in vercodes[appid]:
524 logging.critical(_("No such versionCode {versionCode} for app {appid}")
525 .format(versionCode=v, appid=appid))
528 raise FDroidException(_("Found invalid versionCodes for some apps"))
533 def get_extension(filename):
534 base, ext = os.path.splitext(filename)
537 return base, ext.lower()[1:]
540 def has_extension(filename, ext):
541 _ignored, f_ext = get_extension(filename)
545 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
548 def clean_description(description):
549 'Remove unneeded newlines and spaces from a block of description text'
551 # this is split up by paragraph to make removing the newlines easier
552 for paragraph in re.split(r'\n\n', description):
553 paragraph = re.sub('\r', '', paragraph)
554 paragraph = re.sub('\n', ' ', paragraph)
555 paragraph = re.sub(' {2,}', ' ', paragraph)
556 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
557 returnstring += paragraph + '\n\n'
558 return returnstring.rstrip('\n')
561 def publishednameinfo(filename):
562 filename = os.path.basename(filename)
563 m = publish_name_regex.match(filename)
565 result = (m.group(1), m.group(2))
566 except AttributeError:
567 raise FDroidException(_("Invalid name for published file: %s") % filename)
571 apk_release_filename = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)\.apk')
572 apk_release_filename_with_sigfp = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)_(?P<sigfp>[0-9a-f]{7})\.apk')
575 def apk_parse_release_filename(apkname):
576 """Parses the name of an APK file according the F-Droids APK naming
577 scheme and returns the tokens.
579 WARNING: Returned values don't necessarily represent the APKs actual
580 properties, the are just paresed from the file name.
582 :returns: A triplet containing (appid, versionCode, signer), where appid
583 should be the package name, versionCode should be the integer
584 represion of the APKs version and signer should be the first 7 hex
585 digists of the sha256 signing key fingerprint which was used to sign
588 m = apk_release_filename_with_sigfp.match(apkname)
590 return m.group('appid'), m.group('vercode'), m.group('sigfp')
591 m = apk_release_filename.match(apkname)
593 return m.group('appid'), m.group('vercode'), None
594 return None, None, None
597 def get_release_filename(app, build):
599 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
601 return "%s_%s.apk" % (app.id, build.versionCode)
604 def get_toolsversion_logname(app, build):
605 return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
608 def getsrcname(app, build):
609 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
621 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
624 def get_build_dir(app):
625 '''get the dir that this app will be built in'''
627 if app.RepoType == 'srclib':
628 return os.path.join('build', 'srclib', app.Repo)
630 return os.path.join('build', app.id)
634 '''checkout code from VCS and return instance of vcs and the build dir'''
635 build_dir = get_build_dir(app)
637 # Set up vcs interface and make sure we have the latest code...
638 logging.debug("Getting {0} vcs interface for {1}"
639 .format(app.RepoType, app.Repo))
640 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
644 vcs = getvcs(app.RepoType, remote, build_dir)
646 return vcs, build_dir
649 def getvcs(vcstype, remote, local):
651 return vcs_git(remote, local)
652 if vcstype == 'git-svn':
653 return vcs_gitsvn(remote, local)
655 return vcs_hg(remote, local)
657 return vcs_bzr(remote, local)
658 if vcstype == 'srclib':
659 if local != os.path.join('build', 'srclib', remote):
660 raise VCSException("Error: srclib paths are hard-coded!")
661 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
663 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
664 raise VCSException("Invalid vcs type " + vcstype)
667 def getsrclibvcs(name):
668 if name not in fdroidserver.metadata.srclibs:
669 raise VCSException("Missing srclib " + name)
670 return fdroidserver.metadata.srclibs[name]['Repo Type']
675 def __init__(self, remote, local):
677 # svn, git-svn and bzr may require auth
679 if self.repotype() in ('git-svn', 'bzr'):
681 if self.repotype == 'git-svn':
682 raise VCSException("Authentication is not supported for git-svn")
683 self.username, remote = remote.split('@')
684 if ':' not in self.username:
685 raise VCSException(_("Password required with username"))
686 self.username, self.password = self.username.split(':')
690 self.clone_failed = False
691 self.refreshed = False
697 def clientversion(self):
698 versionstr = FDroidPopen(self.clientversioncmd()).output
699 return versionstr[0:versionstr.find('\n')]
701 def clientversioncmd(self):
704 def gotorevision(self, rev, refresh=True):
705 """Take the local repository to a clean version of the given
706 revision, which is specificed in the VCS's native
707 format. Beforehand, the repository can be dirty, or even
708 non-existent. If the repository does already exist locally, it
709 will be updated from the origin, but only once in the lifetime
710 of the vcs object. None is acceptable for 'rev' if you know
711 you are cloning a clean copy of the repo - otherwise it must
712 specify a valid revision.
715 if self.clone_failed:
716 raise VCSException(_("Downloading the repository already failed once, not trying again."))
718 # The .fdroidvcs-id file for a repo tells us what VCS type
719 # and remote that directory was created from, allowing us to drop it
720 # automatically if either of those things changes.
721 fdpath = os.path.join(self.local, '..',
722 '.fdroidvcs-' + os.path.basename(self.local))
723 fdpath = os.path.normpath(fdpath)
724 cdata = self.repotype() + ' ' + self.remote
727 if os.path.exists(self.local):
728 if os.path.exists(fdpath):
729 with open(fdpath, 'r') as f:
730 fsdata = f.read().strip()
735 logging.info("Repository details for %s changed - deleting" % (
739 logging.info("Repository details for %s missing - deleting" % (
742 shutil.rmtree(self.local)
746 self.refreshed = True
749 self.gotorevisionx(rev)
750 except FDroidException as e:
753 # If necessary, write the .fdroidvcs file.
754 if writeback and not self.clone_failed:
755 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
756 with open(fdpath, 'w+') as f:
762 def gotorevisionx(self, rev): # pylint: disable=unused-argument
763 """Derived classes need to implement this.
765 It's called once basic checking has been performed.
767 raise VCSException("This VCS type doesn't define gotorevisionx")
769 # Initialise and update submodules
770 def initsubmodules(self):
771 raise VCSException('Submodules not supported for this vcs type')
773 # Get a list of all known tags
775 if not self._gettags:
776 raise VCSException('gettags not supported for this vcs type')
778 for tag in self._gettags():
779 if re.match('[-A-Za-z0-9_. /]+$', tag):
783 def latesttags(self):
784 """Get a list of all the known tags, sorted from newest to oldest"""
785 raise VCSException('latesttags not supported for this vcs type')
788 """Get current commit reference (hash, revision, etc)"""
789 raise VCSException('getref not supported for this vcs type')
792 """Returns the srclib (name, path) used in setting up the current revision, or None."""
801 def clientversioncmd(self):
802 return ['git', '--version']
804 def git(self, args, envs=dict(), cwd=None, output=True):
805 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
807 While fetch/pull/clone respect the command line option flags,
808 it seems that submodule commands do not. They do seem to
809 follow whatever is in env vars, if the version of git is new
810 enough. So we just throw the kitchen sink at it to see what
813 Also, because of CVE-2017-1000117, block all SSH URLs.
816 # supported in git >= 2.3
818 '-c', 'core.askpass=/bin/true',
819 '-c', 'core.sshCommand=/bin/false',
820 '-c', 'url.https://.insteadOf=ssh://',
822 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
823 git_config.append('-c')
824 git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':')
825 git_config.append('-c')
826 git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain)
827 git_config.append('-c')
828 git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain)
830 'GIT_TERMINAL_PROMPT': '0',
831 'GIT_ASKPASS': '/bin/true',
832 'SSH_ASKPASS': '/bin/true',
833 'GIT_SSH': '/bin/false', # for git < 2.3
835 return FDroidPopen(['git', ] + git_config + args,
836 envs=envs, cwd=cwd, output=output)
839 """If the local directory exists, but is somehow not a git repository,
840 git will traverse up the directory tree until it finds one
841 that is (i.e. fdroidserver) and then we'll proceed to destroy
842 it! This is called as a safety check.
846 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
847 result = p.output.rstrip()
848 if not result.endswith(self.local):
849 raise VCSException('Repository mismatch')
851 def gotorevisionx(self, rev):
852 if not os.path.exists(self.local):
854 p = self.git(['clone', '--', self.remote, self.local])
855 if p.returncode != 0:
856 self.clone_failed = True
857 raise VCSException("Git clone failed", p.output)
861 # Discard any working tree changes
862 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
863 'git', 'reset', '--hard'], cwd=self.local, output=False)
864 if p.returncode != 0:
865 raise VCSException(_("Git reset failed"), p.output)
866 # Remove untracked files now, in case they're tracked in the target
867 # revision (it happens!)
868 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
869 'git', 'clean', '-dffx'], cwd=self.local, output=False)
870 if p.returncode != 0:
871 raise VCSException(_("Git clean failed"), p.output)
872 if not self.refreshed:
873 # Get latest commits and tags from remote
874 p = self.git(['fetch', 'origin'], cwd=self.local)
875 if p.returncode != 0:
876 raise VCSException(_("Git fetch failed"), p.output)
877 p = self.git(['fetch', '--prune', '--tags', 'origin'], output=False, cwd=self.local)
878 if p.returncode != 0:
879 raise VCSException(_("Git fetch failed"), p.output)
880 # Recreate origin/HEAD as git clone would do it, in case it disappeared
881 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
882 if p.returncode != 0:
883 lines = p.output.splitlines()
884 if 'Multiple remote HEAD branches' not in lines[0]:
885 raise VCSException(_("Git remote set-head failed"), p.output)
886 branch = lines[1].split(' ')[-1]
887 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--', branch],
888 cwd=self.local, output=False)
889 if p2.returncode != 0:
890 raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output)
891 self.refreshed = True
892 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
893 # a github repo. Most of the time this is the same as origin/master.
894 rev = rev or 'origin/HEAD'
895 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
896 if p.returncode != 0:
897 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
898 # Get rid of any uncontrolled files left behind
899 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
900 if p.returncode != 0:
901 raise VCSException(_("Git clean failed"), p.output)
903 def initsubmodules(self):
905 submfile = os.path.join(self.local, '.gitmodules')
906 if not os.path.isfile(submfile):
907 raise NoSubmodulesException(_("No git submodules available"))
909 # fix submodules not accessible without an account and public key auth
910 with open(submfile, 'r') as f:
911 lines = f.readlines()
912 with open(submfile, 'w') as f:
914 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
915 line = re.sub('git@' + domain + ':', 'https://u:p@' + domain + '/', line)
918 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
919 if p.returncode != 0:
920 raise VCSException(_("Git submodule sync failed"), p.output)
921 p = self.git(['submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
922 if p.returncode != 0:
923 raise VCSException(_("Git submodule update failed"), p.output)
927 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
928 return p.output.splitlines()
930 tag_format = re.compile(r'tag: ([^),]*)')
932 def latesttags(self):
934 p = FDroidPopen(['git', 'log', '--tags',
935 '--simplify-by-decoration', '--pretty=format:%d'],
936 cwd=self.local, output=False)
938 for line in p.output.splitlines():
939 for tag in self.tag_format.findall(line):
944 class vcs_gitsvn(vcs):
949 def clientversioncmd(self):
950 return ['git', 'svn', '--version']
953 """If the local directory exists, but is somehow not a git repository,
954 git will traverse up the directory tree until it finds one that
955 is (i.e. fdroidserver) and then we'll proceed to destory it!
956 This is called as a safety check.
959 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
960 result = p.output.rstrip()
961 if not result.endswith(self.local):
962 raise VCSException('Repository mismatch')
964 def git(self, args, envs=dict(), cwd=None, output=True):
965 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
967 AskPass is set to /bin/true to let the process try to connect
968 without a username/password.
970 The SSH command is set to /bin/false to block all SSH URLs
971 (supported in git >= 2.3). This protects against
976 '-c', 'core.askpass=/bin/true',
977 '-c', 'core.sshCommand=/bin/false',
980 'GIT_TERMINAL_PROMPT': '0',
981 'GIT_ASKPASS': '/bin/true',
982 'SSH_ASKPASS': '/bin/true',
983 'GIT_SSH': '/bin/false', # for git < 2.3
984 'SVN_SSH': '/bin/false',
986 return FDroidPopen(['git', ] + git_config + args,
987 envs=envs, cwd=cwd, output=output)
989 def gotorevisionx(self, rev):
990 if not os.path.exists(self.local):
992 gitsvn_args = ['svn', 'clone']
994 if ';' in self.remote:
995 remote_split = self.remote.split(';')
996 for i in remote_split[1:]:
997 if i.startswith('trunk='):
998 gitsvn_args.extend(['-T', i[6:]])
999 elif i.startswith('tags='):
1000 gitsvn_args.extend(['-t', i[5:]])
1001 elif i.startswith('branches='):
1002 gitsvn_args.extend(['-b', i[9:]])
1003 remote = remote_split[0]
1005 remote = self.remote
1007 if not remote.startswith('https://'):
1008 raise VCSException(_('HTTPS must be used with Subversion URLs!'))
1010 # git-svn sucks at certificate validation, this throws useful errors:
1012 r = requests.head(remote)
1013 r.raise_for_status()
1015 gitsvn_args.extend(['--', remote, self.local])
1016 p = self.git(gitsvn_args)
1017 if p.returncode != 0:
1018 self.clone_failed = True
1019 raise VCSException(_('git svn clone failed'), p.output)
1023 # Discard any working tree changes
1024 p = self.git(['reset', '--hard'], cwd=self.local, output=False)
1025 if p.returncode != 0:
1026 raise VCSException("Git reset failed", p.output)
1027 # Remove untracked files now, in case they're tracked in the target
1028 # revision (it happens!)
1029 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1030 if p.returncode != 0:
1031 raise VCSException("Git clean failed", p.output)
1032 if not self.refreshed:
1033 # Get new commits, branches and tags from repo
1034 p = self.git(['svn', 'fetch'], cwd=self.local, output=False)
1035 if p.returncode != 0:
1036 raise VCSException("Git svn fetch failed")
1037 p = self.git(['svn', 'rebase'], cwd=self.local, output=False)
1038 if p.returncode != 0:
1039 raise VCSException("Git svn rebase failed", p.output)
1040 self.refreshed = True
1042 rev = rev or 'master'
1044 nospaces_rev = rev.replace(' ', '%20')
1045 # Try finding a svn tag
1046 for treeish in ['origin/', '']:
1047 p = self.git(['checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
1048 if p.returncode == 0:
1050 if p.returncode != 0:
1051 # No tag found, normal svn rev translation
1052 # Translate svn rev into git format
1053 rev_split = rev.split('/')
1056 for treeish in ['origin/', '']:
1057 if len(rev_split) > 1:
1058 treeish += rev_split[0]
1059 svn_rev = rev_split[1]
1062 # if no branch is specified, then assume trunk (i.e. 'master' branch):
1066 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
1068 p = self.git(['svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
1069 git_rev = p.output.rstrip()
1071 if p.returncode == 0 and git_rev:
1074 if p.returncode != 0 or not git_rev:
1075 # Try a plain git checkout as a last resort
1076 p = self.git(['checkout', rev], cwd=self.local, output=False)
1077 if p.returncode != 0:
1078 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
1080 # Check out the git rev equivalent to the svn rev
1081 p = self.git(['checkout', git_rev], cwd=self.local, output=False)
1082 if p.returncode != 0:
1083 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
1085 # Get rid of any uncontrolled files left behind
1086 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1087 if p.returncode != 0:
1088 raise VCSException(_("Git clean failed"), p.output)
1092 for treeish in ['origin/', '']:
1093 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
1094 if os.path.isdir(d):
1095 return os.listdir(d)
1099 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
1100 if p.returncode != 0:
1102 return p.output.strip()
1110 def clientversioncmd(self):
1111 return ['hg', '--version']
1113 def gotorevisionx(self, rev):
1114 if not os.path.exists(self.local):
1115 p = FDroidPopen(['hg', 'clone', '--ssh', 'false', '--', self.remote, self.local],
1117 if p.returncode != 0:
1118 self.clone_failed = True
1119 raise VCSException("Hg clone failed", p.output)
1121 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
1122 if p.returncode != 0:
1123 raise VCSException("Hg status failed", p.output)
1124 for line in p.output.splitlines():
1125 if not line.startswith('? '):
1126 raise VCSException("Unexpected output from hg status -uS: " + line)
1127 FDroidPopen(['rm', '-rf', '--', line[2:]], cwd=self.local, output=False)
1128 if not self.refreshed:
1129 p = FDroidPopen(['hg', 'pull', '--ssh', 'false'], cwd=self.local, output=False)
1130 if p.returncode != 0:
1131 raise VCSException("Hg pull failed", p.output)
1132 self.refreshed = True
1134 rev = rev or 'default'
1137 p = FDroidPopen(['hg', 'update', '-C', '--', rev], cwd=self.local, output=False)
1138 if p.returncode != 0:
1139 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
1140 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1141 # Also delete untracked files, we have to enable purge extension for that:
1142 if "'purge' is provided by the following extension" in p.output:
1143 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
1144 myfile.write("\n[extensions]\nhgext.purge=\n")
1145 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1146 if p.returncode != 0:
1147 raise VCSException("HG purge failed", p.output)
1148 elif p.returncode != 0:
1149 raise VCSException("HG purge failed", p.output)
1152 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
1153 return p.output.splitlines()[1:]
1161 def clientversioncmd(self):
1162 return ['bzr', '--version']
1164 def bzr(self, args, envs=dict(), cwd=None, output=True):
1165 '''Prevent bzr from ever using SSH to avoid security vulns'''
1169 return FDroidPopen(['bzr', ] + args, envs=envs, cwd=cwd, output=output)
1171 def gotorevisionx(self, rev):
1172 if not os.path.exists(self.local):
1173 p = self.bzr(['branch', self.remote, self.local], output=False)
1174 if p.returncode != 0:
1175 self.clone_failed = True
1176 raise VCSException("Bzr branch failed", p.output)
1178 p = self.bzr(['clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
1179 if p.returncode != 0:
1180 raise VCSException("Bzr revert failed", p.output)
1181 if not self.refreshed:
1182 p = self.bzr(['pull'], cwd=self.local, output=False)
1183 if p.returncode != 0:
1184 raise VCSException("Bzr update failed", p.output)
1185 self.refreshed = True
1187 revargs = list(['-r', rev] if rev else [])
1188 p = self.bzr(['revert'] + revargs, cwd=self.local, output=False)
1189 if p.returncode != 0:
1190 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1193 p = self.bzr(['tags'], cwd=self.local, output=False)
1194 return [tag.split(' ')[0].strip() for tag in
1195 p.output.splitlines()]
1198 def unescape_string(string):
1201 if string[0] == '"' and string[-1] == '"':
1204 return string.replace("\\'", "'")
1207 def retrieve_string(app_dir, string, xmlfiles=None):
1209 if not string.startswith('@string/'):
1210 return unescape_string(string)
1212 if xmlfiles is None:
1215 os.path.join(app_dir, 'res'),
1216 os.path.join(app_dir, 'src', 'main', 'res'),
1218 for root, dirs, files in os.walk(res_dir):
1219 if os.path.basename(root) == 'values':
1220 xmlfiles += [os.path.join(root, x) for x in files if x.endswith('.xml')]
1222 name = string[len('@string/'):]
1224 def element_content(element):
1225 if element.text is None:
1227 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1228 return s.decode('utf-8').strip()
1230 for path in xmlfiles:
1231 if not os.path.isfile(path):
1233 xml = parse_xml(path)
1234 element = xml.find('string[@name="' + name + '"]')
1235 if element is not None:
1236 content = element_content(element)
1237 return retrieve_string(app_dir, content, xmlfiles)
1242 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1243 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1246 def manifest_paths(app_dir, flavours):
1247 '''Return list of existing files that will be used to find the highest vercode'''
1249 possible_manifests = \
1250 [os.path.join(app_dir, 'AndroidManifest.xml'),
1251 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1252 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1253 os.path.join(app_dir, 'build.gradle')]
1255 for flavour in flavours:
1256 if flavour == 'yes':
1258 possible_manifests.append(
1259 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1261 return [path for path in possible_manifests if os.path.isfile(path)]
1264 def fetch_real_name(app_dir, flavours):
1265 '''Retrieve the package name. Returns the name, or None if not found.'''
1266 for path in manifest_paths(app_dir, flavours):
1267 if not has_extension(path, 'xml') or not os.path.isfile(path):
1269 logging.debug("fetch_real_name: Checking manifest at " + path)
1270 xml = parse_xml(path)
1271 app = xml.find('application')
1274 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1276 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1277 result = retrieve_string_singleline(app_dir, label)
1279 result = result.strip()
1284 def get_library_references(root_dir):
1286 proppath = os.path.join(root_dir, 'project.properties')
1287 if not os.path.isfile(proppath):
1289 with open(proppath, 'r', encoding='iso-8859-1') as f:
1291 if not line.startswith('android.library.reference.'):
1293 path = line.split('=')[1].strip()
1294 relpath = os.path.join(root_dir, path)
1295 if not os.path.isdir(relpath):
1297 logging.debug("Found subproject at %s" % path)
1298 libraries.append(path)
1302 def ant_subprojects(root_dir):
1303 subprojects = get_library_references(root_dir)
1304 for subpath in subprojects:
1305 subrelpath = os.path.join(root_dir, subpath)
1306 for p in get_library_references(subrelpath):
1307 relp = os.path.normpath(os.path.join(subpath, p))
1308 if relp not in subprojects:
1309 subprojects.insert(0, relp)
1313 def remove_debuggable_flags(root_dir):
1314 # Remove forced debuggable flags
1315 logging.debug("Removing debuggable flags from %s" % root_dir)
1316 for root, dirs, files in os.walk(root_dir):
1317 if 'AndroidManifest.xml' in files and os.path.isfile(os.path.join(root, 'AndroidManifest.xml')):
1318 regsub_file(r'android:debuggable="[^"]*"',
1320 os.path.join(root, 'AndroidManifest.xml'))
1323 vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search
1324 vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1325 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1328 def app_matches_packagename(app, package):
1331 appid = app.UpdateCheckName or app.id
1332 if appid is None or appid == "Ignore":
1334 return appid == package
1337 def parse_androidmanifests(paths, app):
1339 Extract some information from the AndroidManifest.xml at the given path.
1340 Returns (version, vercode, package), any or all of which might be None.
1341 All values returned are strings.
1344 ignoreversions = app.UpdateCheckIgnore
1345 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1348 return (None, None, None)
1356 if not os.path.isfile(path):
1359 logging.debug(_("Parsing manifest at '{path}'").format(path=path))
1365 if app.builds and 'gradle' in app.builds[-1] and app.builds[-1].gradle:
1366 flavour = app.builds[-1].gradle[-1]
1368 if has_extension(path, 'gradle'):
1369 with open(path, 'r') as f:
1370 inside_flavour_group = 0
1371 inside_required_flavour = 0
1373 if gradle_comment.match(line):
1376 if inside_flavour_group > 0:
1377 if inside_required_flavour > 0:
1378 matches = psearch_g(line)
1380 s = matches.group(2)
1381 if app_matches_packagename(app, s):
1384 matches = vnsearch_g(line)
1386 version = matches.group(2)
1388 matches = vcsearch_g(line)
1390 vercode = matches.group(1)
1393 inside_required_flavour += 1
1395 inside_required_flavour -= 1
1397 if flavour and (flavour in line):
1398 inside_required_flavour = 1
1401 inside_flavour_group += 1
1403 inside_flavour_group -= 1
1405 if "productFlavors" in line:
1406 inside_flavour_group = 1
1408 matches = psearch_g(line)
1410 s = matches.group(2)
1411 if app_matches_packagename(app, s):
1414 matches = vnsearch_g(line)
1416 version = matches.group(2)
1418 matches = vcsearch_g(line)
1420 vercode = matches.group(1)
1423 xml = parse_xml(path)
1424 if "package" in xml.attrib:
1425 s = xml.attrib["package"]
1426 if app_matches_packagename(app, s):
1428 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1429 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1430 base_dir = os.path.dirname(path)
1431 version = retrieve_string_singleline(base_dir, version)
1432 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1433 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1434 if string_is_integer(a):
1437 logging.warning(_("Problem with xml at '{path}'").format(path=path))
1439 # Remember package name, may be defined separately from version+vercode
1441 package = max_package
1443 logging.debug("..got package={0}, version={1}, vercode={2}"
1444 .format(package, version, vercode))
1446 # Always grab the package name and version name in case they are not
1447 # together with the highest version code
1448 if max_package is None and package is not None:
1449 max_package = package
1450 if max_version is None and version is not None:
1451 max_version = version
1453 if vercode is not None \
1454 and (max_vercode is None or vercode > max_vercode):
1455 if not ignoresearch or not ignoresearch(version):
1456 if version is not None:
1457 max_version = version
1458 if vercode is not None:
1459 max_vercode = vercode
1460 if package is not None:
1461 max_package = package
1463 max_version = "Ignore"
1465 if max_version is None:
1466 max_version = "Unknown"
1468 if max_package and not is_valid_package_name(max_package):
1469 raise FDroidException(_("Invalid package name {0}").format(max_package))
1471 return (max_version, max_vercode, max_package)
1474 def is_valid_package_name(name):
1475 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1478 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1479 raw=False, prepare=True, preponly=False, refresh=True,
1481 """Get the specified source library.
1483 Returns the path to it. Normally this is the path to be used when
1484 referencing it, which may be a subdirectory of the actual project. If
1485 you want the base directory of the project, pass 'basepath=True'.
1494 name, ref = spec.split('@')
1496 number, name = name.split(':', 1)
1498 name, subdir = name.split('/', 1)
1500 if name not in fdroidserver.metadata.srclibs:
1501 raise VCSException('srclib ' + name + ' not found.')
1503 srclib = fdroidserver.metadata.srclibs[name]
1505 sdir = os.path.join(srclib_dir, name)
1508 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1509 vcs.srclib = (name, number, sdir)
1511 vcs.gotorevision(ref, refresh)
1518 libdir = os.path.join(sdir, subdir)
1519 elif srclib["Subdir"]:
1520 for subdir in srclib["Subdir"]:
1521 libdir_candidate = os.path.join(sdir, subdir)
1522 if os.path.exists(libdir_candidate):
1523 libdir = libdir_candidate
1529 remove_signing_keys(sdir)
1530 remove_debuggable_flags(sdir)
1534 if srclib["Prepare"]:
1535 cmd = replace_config_vars(srclib["Prepare"], build)
1537 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=libdir)
1538 if p.returncode != 0:
1539 raise BuildException("Error running prepare command for srclib %s"
1545 return (name, number, libdir)
1548 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1551 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1552 """ Prepare the source code for a particular build
1554 :param vcs: the appropriate vcs object for the application
1555 :param app: the application details from the metadata
1556 :param build: the build details from the metadata
1557 :param build_dir: the path to the build directory, usually 'build/app.id'
1558 :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
1559 :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
1561 Returns the (root, srclibpaths) where:
1562 :param root: is the root directory, which may be the same as 'build_dir' or may
1563 be a subdirectory of it.
1564 :param srclibpaths: is information on the srclibs being used
1567 # Optionally, the actual app source can be in a subdirectory
1569 root_dir = os.path.join(build_dir, build.subdir)
1571 root_dir = build_dir
1573 # Get a working copy of the right revision
1574 logging.info("Getting source for revision " + build.commit)
1575 vcs.gotorevision(build.commit, refresh)
1577 # Initialise submodules if required
1578 if build.submodules:
1579 logging.info(_("Initialising submodules"))
1580 vcs.initsubmodules()
1582 # Check that a subdir (if we're using one) exists. This has to happen
1583 # after the checkout, since it might not exist elsewhere
1584 if not os.path.exists(root_dir):
1585 raise BuildException('Missing subdir ' + root_dir)
1587 # Run an init command if one is required
1589 cmd = replace_config_vars(build.init, build)
1590 logging.info("Running 'init' commands in %s" % root_dir)
1592 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir)
1593 if p.returncode != 0:
1594 raise BuildException("Error running init command for %s:%s" %
1595 (app.id, build.versionName), p.output)
1597 # Apply patches if any
1599 logging.info("Applying patches")
1600 for patch in build.patch:
1601 patch = patch.strip()
1602 logging.info("Applying " + patch)
1603 patch_path = os.path.join('metadata', app.id, patch)
1604 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1605 if p.returncode != 0:
1606 raise BuildException("Failed to apply patch %s" % patch_path)
1608 # Get required source libraries
1611 logging.info("Collecting source libraries")
1612 for lib in build.srclibs:
1613 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1614 refresh=refresh, build=build))
1616 for name, number, libpath in srclibpaths:
1617 place_srclib(root_dir, int(number) if number else None, libpath)
1619 basesrclib = vcs.getsrclib()
1620 # If one was used for the main source, add that too.
1622 srclibpaths.append(basesrclib)
1624 # Update the local.properties file
1625 localprops = [os.path.join(build_dir, 'local.properties')]
1627 parts = build.subdir.split(os.sep)
1630 cur = os.path.join(cur, d)
1631 localprops += [os.path.join(cur, 'local.properties')]
1632 for path in localprops:
1634 if os.path.isfile(path):
1635 logging.info("Updating local.properties file at %s" % path)
1636 with open(path, 'r', encoding='iso-8859-1') as f:
1640 logging.info("Creating local.properties file at %s" % path)
1641 # Fix old-fashioned 'sdk-location' by copying
1642 # from sdk.dir, if necessary
1644 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1645 re.S | re.M).group(1)
1646 props += "sdk-location=%s\n" % sdkloc
1648 props += "sdk.dir=%s\n" % config['sdk_path']
1649 props += "sdk-location=%s\n" % config['sdk_path']
1650 ndk_path = build.ndk_path()
1651 # if for any reason the path isn't valid or the directory
1652 # doesn't exist, some versions of Gradle will error with a
1653 # cryptic message (even if the NDK is not even necessary).
1654 # https://gitlab.com/fdroid/fdroidserver/issues/171
1655 if ndk_path and os.path.exists(ndk_path):
1657 props += "ndk.dir=%s\n" % ndk_path
1658 props += "ndk-location=%s\n" % ndk_path
1659 # Add java.encoding if necessary
1661 props += "java.encoding=%s\n" % build.encoding
1662 with open(path, 'w', encoding='iso-8859-1') as f:
1666 if build.build_method() == 'gradle':
1667 flavours = build.gradle
1670 n = build.target.split('-')[1]
1671 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1672 r'compileSdkVersion %s' % n,
1673 os.path.join(root_dir, 'build.gradle'))
1675 # Remove forced debuggable flags
1676 remove_debuggable_flags(root_dir)
1678 # Insert version code and number into the manifest if necessary
1679 if build.forceversion:
1680 logging.info("Changing the version name")
1681 for path in manifest_paths(root_dir, flavours):
1682 if not os.path.isfile(path):
1684 if has_extension(path, 'xml'):
1685 regsub_file(r'android:versionName="[^"]*"',
1686 r'android:versionName="%s"' % build.versionName,
1688 elif has_extension(path, 'gradle'):
1689 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1690 r"""\1versionName '%s'""" % build.versionName,
1693 if build.forcevercode:
1694 logging.info("Changing the version code")
1695 for path in manifest_paths(root_dir, flavours):
1696 if not os.path.isfile(path):
1698 if has_extension(path, 'xml'):
1699 regsub_file(r'android:versionCode="[^"]*"',
1700 r'android:versionCode="%s"' % build.versionCode,
1702 elif has_extension(path, 'gradle'):
1703 regsub_file(r'versionCode[ =]+[0-9]+',
1704 r'versionCode %s' % build.versionCode,
1707 # Delete unwanted files
1709 logging.info(_("Removing specified files"))
1710 for part in getpaths(build_dir, build.rm):
1711 dest = os.path.join(build_dir, part)
1712 logging.info("Removing {0}".format(part))
1713 if os.path.lexists(dest):
1714 # rmtree can only handle directories that are not symlinks, so catch anything else
1715 if not os.path.isdir(dest) or os.path.islink(dest):
1720 logging.info("...but it didn't exist")
1722 remove_signing_keys(build_dir)
1724 # Add required external libraries
1726 logging.info("Collecting prebuilt libraries")
1727 libsdir = os.path.join(root_dir, 'libs')
1728 if not os.path.exists(libsdir):
1730 for lib in build.extlibs:
1732 logging.info("...installing extlib {0}".format(lib))
1733 libf = os.path.basename(lib)
1734 libsrc = os.path.join(extlib_dir, lib)
1735 if not os.path.exists(libsrc):
1736 raise BuildException("Missing extlib file {0}".format(libsrc))
1737 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1739 # Run a pre-build command if one is required
1741 logging.info("Running 'prebuild' commands in %s" % root_dir)
1743 cmd = replace_config_vars(build.prebuild, build)
1745 # Substitute source library paths into prebuild commands
1746 for name, number, libpath in srclibpaths:
1747 libpath = os.path.relpath(libpath, root_dir)
1748 cmd = cmd.replace('$$' + name + '$$', libpath)
1750 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir)
1751 if p.returncode != 0:
1752 raise BuildException("Error running prebuild command for %s:%s" %
1753 (app.id, build.versionName), p.output)
1755 # Generate (or update) the ant build file, build.xml...
1756 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1757 parms = ['android', 'update', 'lib-project']
1758 lparms = ['android', 'update', 'project']
1761 parms += ['-t', build.target]
1762 lparms += ['-t', build.target]
1763 if build.androidupdate:
1764 update_dirs = build.androidupdate
1766 update_dirs = ant_subprojects(root_dir) + ['.']
1768 for d in update_dirs:
1769 subdir = os.path.join(root_dir, d)
1771 logging.debug("Updating main project")
1772 cmd = parms + ['-p', d]
1774 logging.debug("Updating subproject %s" % d)
1775 cmd = lparms + ['-p', d]
1776 p = SdkToolsPopen(cmd, cwd=root_dir)
1777 # Check to see whether an error was returned without a proper exit
1778 # code (this is the case for the 'no target set or target invalid'
1780 if p.returncode != 0 or p.output.startswith("Error: "):
1781 raise BuildException("Failed to update project at %s" % d, p.output)
1782 # Clean update dirs via ant
1784 logging.info("Cleaning subproject %s" % d)
1785 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1787 return (root_dir, srclibpaths)
1790 def getpaths_map(build_dir, globpaths):
1791 """Extend via globbing the paths from a field and return them as a map from original path to resulting paths"""
1795 full_path = os.path.join(build_dir, p)
1796 full_path = os.path.normpath(full_path)
1797 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1799 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1803 def getpaths(build_dir, globpaths):
1804 """Extend via globbing the paths from a field and return them as a set"""
1805 paths_map = getpaths_map(build_dir, globpaths)
1807 for k, v in paths_map.items():
1814 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1817 def check_system_clock(dt_obj, path):
1818 """Check if system clock is updated based on provided date
1820 If an APK has files newer than the system time, suggest updating
1821 the system clock. This is useful for offline systems, used for
1822 signing, which do not have another source of clock sync info. It
1823 has to be more than 24 hours newer because ZIP/APK files do not
1827 checkdt = dt_obj - timedelta(1)
1828 if datetime.today() < checkdt:
1829 logging.warning(_('System clock is older than date in {path}!').format(path=path)
1830 + '\n' + _('Set clock to that time using:') + '\n'
1831 + 'sudo date -s "' + str(dt_obj) + '"')
1835 """permanent store of existing APKs with the date they were added
1837 This is currently the only way to permanently store the "updated"
1842 '''Load filename/date info about previously seen APKs
1844 Since the appid and date strings both will never have spaces,
1845 this is parsed as a list from the end to allow the filename to
1846 have any combo of spaces.
1849 self.path = os.path.join('stats', 'known_apks.txt')
1851 if os.path.isfile(self.path):
1852 with open(self.path, 'r', encoding='utf8') as f:
1854 t = line.rstrip().split(' ')
1856 self.apks[t[0]] = (t[1], None)
1859 date = datetime.strptime(t[-1], '%Y-%m-%d')
1860 filename = line[0:line.rfind(appid) - 1]
1861 self.apks[filename] = (appid, date)
1862 check_system_clock(date, self.path)
1863 self.changed = False
1865 def writeifchanged(self):
1866 if not self.changed:
1869 if not os.path.exists('stats'):
1873 for apk, app in self.apks.items():
1875 line = apk + ' ' + appid
1877 line += ' ' + added.strftime('%Y-%m-%d')
1880 with open(self.path, 'w', encoding='utf8') as f:
1881 for line in sorted(lst, key=natural_key):
1882 f.write(line + '\n')
1884 def recordapk(self, apkName, app, default_date=None):
1886 Record an apk (if it's new, otherwise does nothing)
1887 Returns the date it was added as a datetime instance
1889 if apkName not in self.apks:
1890 if default_date is None:
1891 default_date = datetime.utcnow()
1892 self.apks[apkName] = (app, default_date)
1894 _ignored, added = self.apks[apkName]
1897 def getapp(self, apkname):
1898 """Look up information - given the 'apkname', returns (app id, date added/None).
1900 Or returns None for an unknown apk.
1902 if apkname in self.apks:
1903 return self.apks[apkname]
1906 def getlatest(self, num):
1907 """Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first"""
1909 for apk, app in self.apks.items():
1913 if apps[appid] > added:
1917 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1918 lst = [app for app, _ignored in sortedapps]
1923 def get_file_extension(filename):
1924 """get the normalized file extension, can be blank string but never None"""
1925 if isinstance(filename, bytes):
1926 filename = filename.decode('utf-8')
1927 return os.path.splitext(filename)[1].lower()[1:]
1930 def get_apk_debuggable_aapt(apkfile):
1931 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1933 if p.returncode != 0:
1934 raise FDroidException(_("Failed to get APK manifest information"))
1935 for line in p.output.splitlines():
1936 if 'android:debuggable' in line and not line.endswith('0x0'):
1941 def get_apk_debuggable_androguard(apkfile):
1943 from androguard.core.bytecodes.apk import APK
1945 raise FDroidException("androguard library is not installed and aapt not present")
1947 apkobject = APK(apkfile)
1948 if apkobject.is_valid_APK():
1949 debuggable = apkobject.get_element("application", "debuggable")
1950 if debuggable is not None:
1951 return bool(strtobool(debuggable))
1955 def isApkAndDebuggable(apkfile):
1956 """Returns True if the given file is an APK and is debuggable
1958 :param apkfile: full path to the apk to check"""
1960 if get_file_extension(apkfile) != 'apk':
1963 if SdkToolsPopen(['aapt', 'version'], output=False):
1964 return get_apk_debuggable_aapt(apkfile)
1966 return get_apk_debuggable_androguard(apkfile)
1969 def get_apk_id_aapt(apkfile):
1970 """Extrat identification information from APK using aapt.
1972 :param apkfile: path to an APK file.
1973 :returns: triplet (appid, version code, version name)
1975 r = re.compile("package: name='(?P<appid>.*)' versionCode='(?P<vercode>.*)' versionName='(?P<vername>.*)' platformBuildVersionName='.*'")
1976 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1977 for line in p.output.splitlines():
1980 return m.group('appid'), m.group('vercode'), m.group('vername')
1981 raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'")
1982 .format(apkfilename=apkfile))
1985 def get_minSdkVersion_aapt(apkfile):
1986 """Extract the minimum supported Android SDK from an APK using aapt
1988 :param apkfile: path to an APK file.
1989 :returns: the integer representing the SDK version
1991 r = re.compile(r"^sdkVersion:'([0-9]+)'")
1992 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1993 for line in p.output.splitlines():
1996 return int(m.group(1))
1997 raise FDroidException(_('Reading minSdkVersion failed: "{apkfilename}"')
1998 .format(apkfilename=apkfile))
2003 self.returncode = None
2007 def SdkToolsPopen(commands, cwd=None, output=True):
2009 if cmd not in config:
2010 config[cmd] = find_sdk_tools_cmd(commands[0])
2011 abscmd = config[cmd]
2013 raise FDroidException(_("Could not find '{command}' on your system").format(command=cmd))
2015 test_aapt_version(config['aapt'])
2016 return FDroidPopen([abscmd] + commands[1:],
2017 cwd=cwd, output=output)
2020 def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2022 Run a command and capture the possibly huge output as bytes.
2024 :param commands: command and argument list like in subprocess.Popen
2025 :param cwd: optionally specifies a working directory
2026 :param envs: a optional dictionary of environment variables and their values
2027 :returns: A PopenResult.
2032 set_FDroidPopen_env()
2034 process_env = env.copy()
2035 if envs is not None and len(envs) > 0:
2036 process_env.update(envs)
2039 cwd = os.path.normpath(cwd)
2040 logging.debug("Directory: %s" % cwd)
2041 logging.debug("> %s" % ' '.join(commands))
2043 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
2044 result = PopenResult()
2047 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
2048 stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
2049 stderr=stderr_param)
2050 except OSError as e:
2051 raise BuildException("OSError while trying to execute " +
2052 ' '.join(commands) + ': ' + str(e))
2054 # TODO are these AsynchronousFileReader threads always exiting?
2055 if not stderr_to_stdout and options.verbose:
2056 stderr_queue = Queue()
2057 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
2059 while not stderr_reader.eof():
2060 while not stderr_queue.empty():
2061 line = stderr_queue.get()
2062 sys.stderr.buffer.write(line)
2067 stdout_queue = Queue()
2068 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
2071 # Check the queue for output (until there is no more to get)
2072 while not stdout_reader.eof():
2073 while not stdout_queue.empty():
2074 line = stdout_queue.get()
2075 if output and options.verbose:
2076 # Output directly to console
2077 sys.stderr.buffer.write(line)
2083 result.returncode = p.wait()
2084 result.output = buf.getvalue()
2086 # make sure all filestreams of the subprocess are closed
2087 for streamvar in ['stdin', 'stdout', 'stderr']:
2088 if hasattr(p, streamvar):
2089 stream = getattr(p, streamvar)
2095 def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2097 Run a command and capture the possibly huge output as a str.
2099 :param commands: command and argument list like in subprocess.Popen
2100 :param cwd: optionally specifies a working directory
2101 :param envs: a optional dictionary of environment variables and their values
2102 :returns: A PopenResult.
2104 result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
2105 result.output = result.output.decode('utf-8', 'ignore')
2109 gradle_comment = re.compile(r'[ ]*//')
2110 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
2111 gradle_line_matches = [
2112 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
2113 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
2114 re.compile(r'.*\.readLine\(.*'),
2118 def remove_signing_keys(build_dir):
2119 for root, dirs, files in os.walk(build_dir):
2120 if 'build.gradle' in files:
2121 path = os.path.join(root, 'build.gradle')
2123 with open(path, "r", encoding='utf8') as o:
2124 lines = o.readlines()
2130 with open(path, "w", encoding='utf8') as o:
2131 while i < len(lines):
2134 while line.endswith('\\\n'):
2135 line = line.rstrip('\\\n') + lines[i]
2138 if gradle_comment.match(line):
2143 opened += line.count('{')
2144 opened -= line.count('}')
2147 if gradle_signing_configs.match(line):
2152 if any(s.match(line) for s in gradle_line_matches):
2160 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
2163 'project.properties',
2165 'default.properties',
2166 'ant.properties', ]:
2167 if propfile in files:
2168 path = os.path.join(root, propfile)
2170 with open(path, "r", encoding='iso-8859-1') as o:
2171 lines = o.readlines()
2175 with open(path, "w", encoding='iso-8859-1') as o:
2177 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
2184 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
2187 def set_FDroidPopen_env(build=None):
2189 set up the environment variables for the build environment
2191 There is only a weak standard, the variables used by gradle, so also set
2192 up the most commonly used environment variables for SDK and NDK. Also, if
2193 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
2195 global env, orig_path
2199 orig_path = env['PATH']
2200 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
2201 env[n] = config['sdk_path']
2202 for k, v in config['java_paths'].items():
2203 env['JAVA%s_HOME' % k] = v
2205 missinglocale = True
2206 for k, v in env.items():
2207 if k == 'LANG' and v != 'C':
2208 missinglocale = False
2210 missinglocale = False
2212 env['LANG'] = 'en_US.UTF-8'
2214 if build is not None:
2215 path = build.ndk_path()
2216 paths = orig_path.split(os.pathsep)
2217 if path not in paths:
2218 paths = [path] + paths
2219 env['PATH'] = os.pathsep.join(paths)
2220 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
2221 env[n] = build.ndk_path()
2224 def replace_build_vars(cmd, build):
2225 cmd = cmd.replace('$$COMMIT$$', build.commit)
2226 cmd = cmd.replace('$$VERSION$$', build.versionName)
2227 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
2231 def replace_config_vars(cmd, build):
2232 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
2233 cmd = cmd.replace('$$NDK$$', build.ndk_path())
2234 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
2235 if build is not None:
2236 cmd = replace_build_vars(cmd, build)
2240 def place_srclib(root_dir, number, libpath):
2243 relpath = os.path.relpath(libpath, root_dir)
2244 proppath = os.path.join(root_dir, 'project.properties')
2247 if os.path.isfile(proppath):
2248 with open(proppath, "r", encoding='iso-8859-1') as o:
2249 lines = o.readlines()
2251 with open(proppath, "w", encoding='iso-8859-1') as o:
2254 if line.startswith('android.library.reference.%d=' % number):
2255 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2260 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2263 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
2266 def signer_fingerprint_short(sig):
2267 """Obtain shortened sha256 signing-key fingerprint for pkcs7 signature.
2269 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2270 for a given pkcs7 signature.
2272 :param sig: Contents of an APK signing certificate.
2273 :returns: shortened signing-key fingerprint.
2275 return signer_fingerprint(sig)[:7]
2278 def signer_fingerprint(sig):
2279 """Obtain sha256 signing-key fingerprint for pkcs7 signature.
2281 Extracts hexadecimal sha256 signing-key fingerprint string
2282 for a given pkcs7 signature.
2284 :param: Contents of an APK signature.
2285 :returns: shortened signature fingerprint.
2287 cert_encoded = get_certificate(sig)
2288 return hashlib.sha256(cert_encoded).hexdigest()
2291 def apk_signer_fingerprint(apk_path):
2292 """Obtain sha256 signing-key fingerprint for APK.
2294 Extracts hexadecimal sha256 signing-key fingerprint string
2297 :param apkpath: path to APK
2298 :returns: signature fingerprint
2301 with zipfile.ZipFile(apk_path, 'r') as apk:
2302 certs = [n for n in apk.namelist() if CERT_PATH_REGEX.match(n)]
2305 logging.error("Found no signing certificates on %s" % apk_path)
2308 logging.error("Found multiple signing certificates on %s" % apk_path)
2311 cert = apk.read(certs[0])
2312 return signer_fingerprint(cert)
2315 def apk_signer_fingerprint_short(apk_path):
2316 """Obtain shortened sha256 signing-key fingerprint for APK.
2318 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2319 for a given pkcs7 APK.
2321 :param apk_path: path to APK
2322 :returns: shortened signing-key fingerprint
2324 return apk_signer_fingerprint(apk_path)[:7]
2327 def metadata_get_sigdir(appid, vercode=None):
2328 """Get signature directory for app"""
2330 return os.path.join('metadata', appid, 'signatures', vercode)
2332 return os.path.join('metadata', appid, 'signatures')
2335 def metadata_find_developer_signature(appid, vercode=None):
2336 """Tires to find the developer signature for given appid.
2338 This picks the first signature file found in metadata an returns its
2341 :returns: sha256 signing key fingerprint of the developer signing key.
2342 None in case no signature can not be found."""
2344 # fetch list of dirs for all versions of signatures
2347 appversigdirs.append(metadata_get_sigdir(appid, vercode))
2349 appsigdir = metadata_get_sigdir(appid)
2350 if os.path.isdir(appsigdir):
2351 numre = re.compile('[0-9]+')
2352 for ver in os.listdir(appsigdir):
2353 if numre.match(ver):
2354 appversigdir = os.path.join(appsigdir, ver)
2355 appversigdirs.append(appversigdir)
2357 for sigdir in appversigdirs:
2358 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2359 glob.glob(os.path.join(sigdir, '*.EC')) + \
2360 glob.glob(os.path.join(sigdir, '*.RSA'))
2362 raise FDroidException('ambiguous signatures, please make sure there is only one signature in \'{}\'. (The signature has to be the App maintainers signature for version of the APK.)'.format(sigdir))
2364 with open(sig, 'rb') as f:
2365 return signer_fingerprint(f.read())
2369 def metadata_find_signing_files(appid, vercode):
2370 """Gets a list of singed manifests and signatures.
2372 :param appid: app id string
2373 :param vercode: app version code
2374 :returns: a list of triplets for each signing key with following paths:
2375 (signature_file, singed_file, manifest_file)
2378 sigdir = metadata_get_sigdir(appid, vercode)
2379 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2380 glob.glob(os.path.join(sigdir, '*.EC')) + \
2381 glob.glob(os.path.join(sigdir, '*.RSA'))
2382 extre = re.compile('(\.DSA|\.EC|\.RSA)$')
2384 sf = extre.sub('.SF', sig)
2385 if os.path.isfile(sf):
2386 mf = os.path.join(sigdir, 'MANIFEST.MF')
2387 if os.path.isfile(mf):
2388 ret.append((sig, sf, mf))
2392 def metadata_find_developer_signing_files(appid, vercode):
2393 """Get developer signature files for specified app from metadata.
2395 :returns: A triplet of paths for signing files from metadata:
2396 (signature_file, singed_file, manifest_file)
2398 allsigningfiles = metadata_find_signing_files(appid, vercode)
2399 if allsigningfiles and len(allsigningfiles) == 1:
2400 return allsigningfiles[0]
2405 def apk_strip_signatures(signed_apk, strip_manifest=False):
2406 """Removes signatures from APK.
2408 :param signed_apk: path to apk file.
2409 :param strip_manifest: when set to True also the manifest file will
2410 be removed from the APK.
2412 with tempfile.TemporaryDirectory() as tmpdir:
2413 tmp_apk = os.path.join(tmpdir, 'tmp.apk')
2414 shutil.move(signed_apk, tmp_apk)
2415 with ZipFile(tmp_apk, 'r') as in_apk:
2416 with ZipFile(signed_apk, 'w') as out_apk:
2417 for info in in_apk.infolist():
2418 if not apk_sigfile.match(info.filename):
2420 if info.filename != 'META-INF/MANIFEST.MF':
2421 buf = in_apk.read(info.filename)
2422 out_apk.writestr(info, buf)
2424 buf = in_apk.read(info.filename)
2425 out_apk.writestr(info, buf)
2428 def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest):
2429 """Implats a signature from metadata into an APK.
2431 Note: this changes there supplied APK in place. So copy it if you
2432 need the original to be preserved.
2434 :param apkpath: location of the apk
2436 # get list of available signature files in metadata
2437 with tempfile.TemporaryDirectory() as tmpdir:
2438 apkwithnewsig = os.path.join(tmpdir, 'newsig.apk')
2439 with ZipFile(apkpath, 'r') as in_apk:
2440 with ZipFile(apkwithnewsig, 'w') as out_apk:
2441 for sig_file in [signaturefile, signedfile, manifest]:
2442 with open(sig_file, 'rb') as fp:
2444 info = zipfile.ZipInfo('META-INF/' + os.path.basename(sig_file))
2445 info.compress_type = zipfile.ZIP_DEFLATED
2446 info.create_system = 0 # "Windows" aka "FAT", what Android SDK uses
2447 out_apk.writestr(info, buf)
2448 for info in in_apk.infolist():
2449 if not apk_sigfile.match(info.filename):
2450 if info.filename != 'META-INF/MANIFEST.MF':
2451 buf = in_apk.read(info.filename)
2452 out_apk.writestr(info, buf)
2454 p = SdkToolsPopen(['zipalign', '-v', '4', apkwithnewsig, apkpath])
2455 if p.returncode != 0:
2456 raise BuildException("Failed to align application")
2459 def apk_extract_signatures(apkpath, outdir, manifest=True):
2460 """Extracts a signature files from APK and puts them into target directory.
2462 :param apkpath: location of the apk
2463 :param outdir: folder where the extracted signature files will be stored
2464 :param manifest: (optionally) disable extracting manifest file
2466 with ZipFile(apkpath, 'r') as in_apk:
2467 for f in in_apk.infolist():
2468 if apk_sigfile.match(f.filename) or \
2469 (manifest and f.filename == 'META-INF/MANIFEST.MF'):
2470 newpath = os.path.join(outdir, os.path.basename(f.filename))
2471 with open(newpath, 'wb') as out_file:
2472 out_file.write(in_apk.read(f.filename))
2475 def sign_apk(unsigned_path, signed_path, keyalias):
2476 """Sign and zipalign an unsigned APK, then save to a new file, deleting the unsigned
2478 android-18 (4.3) finally added support for reasonable hash
2479 algorithms, like SHA-256, before then, the only options were MD5
2480 and SHA1 :-/ This aims to use SHA-256 when the APK does not target
2481 older Android versions, and is therefore safe to do so.
2483 https://issuetracker.google.com/issues/36956587
2484 https://android-review.googlesource.com/c/platform/libcore/+/44491
2488 if get_minSdkVersion_aapt(unsigned_path) < 18:
2489 signature_algorithm = ['-sigalg', 'SHA1withRSA', '-digestalg', 'SHA1']
2491 signature_algorithm = ['-sigalg', 'SHA256withRSA', '-digestalg', 'SHA-256']
2493 p = FDroidPopen([config['jarsigner'], '-keystore', config['keystore'],
2494 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2495 '-keypass:env', 'FDROID_KEY_PASS']
2496 + signature_algorithm + [unsigned_path, keyalias],
2498 'FDROID_KEY_STORE_PASS': config['keystorepass'],
2499 'FDROID_KEY_PASS': config['keypass'], })
2500 if p.returncode != 0:
2501 raise BuildException(_("Failed to sign application"), p.output)
2503 p = SdkToolsPopen(['zipalign', '-v', '4', unsigned_path, signed_path])
2504 if p.returncode != 0:
2505 raise BuildException(_("Failed to zipalign application"))
2506 os.remove(unsigned_path)
2509 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
2510 """Verify that two apks are the same
2512 One of the inputs is signed, the other is unsigned. The signature metadata
2513 is transferred from the signed to the unsigned apk, and then jarsigner is
2514 used to verify that the signature from the signed apk is also varlid for
2515 the unsigned one. If the APK given as unsigned actually does have a
2516 signature, it will be stripped out and ignored.
2518 There are two SHA1 git commit IDs that fdroidserver includes in the builds
2519 it makes: fdroidserverid and buildserverid. Originally, these were inserted
2520 into AndroidManifest.xml, but that makes the build not reproducible. So
2521 instead they are included as separate files in the APK's META-INF/ folder.
2522 If those files exist in the signed APK, they will be part of the signature
2523 and need to also be included in the unsigned APK for it to validate.
2525 :param signed_apk: Path to a signed apk file
2526 :param unsigned_apk: Path to an unsigned apk file expected to match it
2527 :param tmp_dir: Path to directory for temporary files
2528 :returns: None if the verification is successful, otherwise a string
2529 describing what went wrong.
2532 if not os.path.isfile(signed_apk):
2533 return 'can not verify: file does not exists: {}'.format(signed_apk)
2535 if not os.path.isfile(unsigned_apk):
2536 return 'can not verify: file does not exists: {}'.format(unsigned_apk)
2538 with ZipFile(signed_apk, 'r') as signed:
2539 meta_inf_files = ['META-INF/MANIFEST.MF']
2540 for f in signed.namelist():
2541 if apk_sigfile.match(f) \
2542 or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
2543 meta_inf_files.append(f)
2544 if len(meta_inf_files) < 3:
2545 return "Signature files missing from {0}".format(signed_apk)
2547 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2548 with ZipFile(unsigned_apk, 'r') as unsigned:
2549 # only read the signature from the signed APK, everything else from unsigned
2550 with ZipFile(tmp_apk, 'w') as tmp:
2551 for filename in meta_inf_files:
2552 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2553 for info in unsigned.infolist():
2554 if info.filename in meta_inf_files:
2555 logging.warning('Ignoring %s from %s',
2556 info.filename, unsigned_apk)
2558 if info.filename in tmp.namelist():
2559 return "duplicate filename found: " + info.filename
2560 tmp.writestr(info, unsigned.read(info.filename))
2562 verified = verify_apk_signature(tmp_apk)
2565 logging.info("...NOT verified - {0}".format(tmp_apk))
2566 return compare_apks(signed_apk, tmp_apk, tmp_dir,
2567 os.path.dirname(unsigned_apk))
2569 logging.info("...successfully verified")
2573 def verify_jar_signature(jar):
2574 """Verifies the signature of a given JAR file.
2576 jarsigner is very shitty: unsigned JARs pass as "verified"! So
2577 this has to turn on -strict then check for result 4, since this
2578 does not expect the signature to be from a CA-signed certificate.
2580 :raises: VerificationException() if the JAR's signature could not be verified
2584 error = _('JAR signature failed to verify: {path}').format(path=jar)
2586 output = subprocess.check_output([config['jarsigner'], '-strict', '-verify', jar],
2587 stderr=subprocess.STDOUT)
2588 raise VerificationException(error + '\n' + output.decode('utf-8'))
2589 except subprocess.CalledProcessError as e:
2590 if e.returncode == 4:
2591 logging.debug(_('JAR signature verified: {path}').format(path=jar))
2593 raise VerificationException(error + '\n' + e.output.decode('utf-8'))
2596 def verify_apk_signature(apk, min_sdk_version=None):
2597 """verify the signature on an APK
2599 Try to use apksigner whenever possible since jarsigner is very
2600 shitty: unsigned APKs pass as "verified"! Warning, this does
2601 not work on JARs with apksigner >= 0.7 (build-tools 26.0.1)
2603 :returns: boolean whether the APK was verified
2605 if set_command_in_config('apksigner'):
2606 args = [config['apksigner'], 'verify']
2608 args += ['--min-sdk-version=' + min_sdk_version]
2610 args += ['--verbose']
2612 output = subprocess.check_output(args + [apk])
2614 logging.debug(apk + ': ' + output.decode('utf-8'))
2616 except subprocess.CalledProcessError as e:
2617 logging.error('\n' + apk + ': ' + e.output.decode('utf-8'))
2619 if not config.get('jarsigner_warning_displayed'):
2620 config['jarsigner_warning_displayed'] = True
2621 logging.warning(_("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner"))
2623 verify_jar_signature(apk)
2625 except Exception as e:
2630 def verify_old_apk_signature(apk):
2631 """verify the signature on an archived APK, supporting deprecated algorithms
2633 F-Droid aims to keep every single binary that it ever published. Therefore,
2634 it needs to be able to verify APK signatures that include deprecated/removed
2635 algorithms. For example, jarsigner treats an MD5 signature as unsigned.
2637 jarsigner passes unsigned APKs as "verified"! So this has to turn
2638 on -strict then check for result 4.
2640 :returns: boolean whether the APK was verified
2643 _java_security = os.path.join(os.getcwd(), '.java.security')
2644 with open(_java_security, 'w') as fp:
2645 fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
2649 config['jarsigner'],
2650 '-J-Djava.security.properties=' + _java_security,
2651 '-strict', '-verify', apk
2653 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
2654 except subprocess.CalledProcessError as e:
2655 if e.returncode != 4:
2658 logging.debug(_('JAR signature verified: {path}').format(path=apk))
2661 logging.error(_('Old APK signature failed to verify: {path}').format(path=apk)
2662 + '\n' + output.decode('utf-8'))
2666 apk_badchars = re.compile('''[/ :;'"]''')
2669 def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
2672 Returns None if the apk content is the same (apart from the signing key),
2673 otherwise a string describing what's different, or what went wrong when
2674 trying to do the comparison.
2680 absapk1 = os.path.abspath(apk1)
2681 absapk2 = os.path.abspath(apk2)
2683 if set_command_in_config('diffoscope'):
2684 logfilename = os.path.join(log_dir, os.path.basename(absapk1))
2685 htmlfile = logfilename + '.diffoscope.html'
2686 textfile = logfilename + '.diffoscope.txt'
2687 if subprocess.call([config['diffoscope'],
2688 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2689 '--html', htmlfile, '--text', textfile,
2690 absapk1, absapk2]) != 0:
2691 return("Failed to unpack " + apk1)
2693 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2694 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2695 for d in [apk1dir, apk2dir]:
2696 if os.path.exists(d):
2699 os.mkdir(os.path.join(d, 'jar-xf'))
2701 if subprocess.call(['jar', 'xf',
2702 os.path.abspath(apk1)],
2703 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2704 return("Failed to unpack " + apk1)
2705 if subprocess.call(['jar', 'xf',
2706 os.path.abspath(apk2)],
2707 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2708 return("Failed to unpack " + apk2)
2710 if set_command_in_config('apktool'):
2711 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2713 return("Failed to unpack " + apk1)
2714 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2716 return("Failed to unpack " + apk2)
2718 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2719 lines = p.output.splitlines()
2720 if len(lines) != 1 or 'META-INF' not in lines[0]:
2721 if set_command_in_config('meld'):
2722 p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
2723 return("Unexpected diff output - " + p.output)
2725 # since everything verifies, delete the comparison to keep cruft down
2726 shutil.rmtree(apk1dir)
2727 shutil.rmtree(apk2dir)
2729 # If we get here, it seems like they're the same!
2733 def set_command_in_config(command):
2734 '''Try to find specified command in the path, if it hasn't been
2735 manually set in config.py. If found, it is added to the config
2736 dict. The return value says whether the command is available.
2739 if command in config:
2742 tmp = find_command(command)
2744 config[command] = tmp
2749 def find_command(command):
2750 '''find the full path of a command, or None if it can't be found in the PATH'''
2753 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2755 fpath, fname = os.path.split(command)
2760 for path in os.environ["PATH"].split(os.pathsep):
2761 path = path.strip('"')
2762 exe_file = os.path.join(path, command)
2763 if is_exe(exe_file):
2770 '''generate a random password for when generating keys'''
2771 h = hashlib.sha256()
2772 h.update(os.urandom(16)) # salt
2773 h.update(socket.getfqdn().encode('utf-8'))
2774 passwd = base64.b64encode(h.digest()).strip()
2775 return passwd.decode('utf-8')
2778 def genkeystore(localconfig):
2780 Generate a new key with password provided in :param localconfig and add it to new keystore
2781 :return: hexed public key, public key fingerprint
2783 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2784 keystoredir = os.path.dirname(localconfig['keystore'])
2785 if keystoredir is None or keystoredir == '':
2786 keystoredir = os.path.join(os.getcwd(), keystoredir)
2787 if not os.path.exists(keystoredir):
2788 os.makedirs(keystoredir, mode=0o700)
2791 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
2792 'FDROID_KEY_PASS': localconfig['keypass'],
2794 p = FDroidPopen([config['keytool'], '-genkey',
2795 '-keystore', localconfig['keystore'],
2796 '-alias', localconfig['repo_keyalias'],
2797 '-keyalg', 'RSA', '-keysize', '4096',
2798 '-sigalg', 'SHA256withRSA',
2799 '-validity', '10000',
2800 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2801 '-keypass:env', 'FDROID_KEY_PASS',
2802 '-dname', localconfig['keydname']], envs=env_vars)
2803 if p.returncode != 0:
2804 raise BuildException("Failed to generate key", p.output)
2805 os.chmod(localconfig['keystore'], 0o0600)
2806 if not options.quiet:
2807 # now show the lovely key that was just generated
2808 p = FDroidPopen([config['keytool'], '-list', '-v',
2809 '-keystore', localconfig['keystore'],
2810 '-alias', localconfig['repo_keyalias'],
2811 '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
2812 logging.info(p.output.strip() + '\n\n')
2813 # get the public key
2814 p = FDroidPopenBytes([config['keytool'], '-exportcert',
2815 '-keystore', localconfig['keystore'],
2816 '-alias', localconfig['repo_keyalias'],
2817 '-storepass:env', 'FDROID_KEY_STORE_PASS']
2818 + config['smartcardoptions'],
2819 envs=env_vars, output=False, stderr_to_stdout=False)
2820 if p.returncode != 0 or len(p.output) < 20:
2821 raise BuildException("Failed to get public key", p.output)
2823 fingerprint = get_cert_fingerprint(pubkey)
2824 return hexlify(pubkey), fingerprint
2827 def get_cert_fingerprint(pubkey):
2829 Generate a certificate fingerprint the same way keytool does it
2830 (but with slightly different formatting)
2832 digest = hashlib.sha256(pubkey).digest()
2833 ret = [' '.join("%02X" % b for b in bytearray(digest))]
2834 return " ".join(ret)
2837 def get_certificate(certificate_file):
2839 Extracts a certificate from the given file.
2840 :param certificate_file: file bytes (as string) representing the certificate
2841 :return: A binary representation of the certificate's public key, or None in case of error
2843 content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
2844 if content.getComponentByName('contentType') != rfc2315.signedData:
2846 content = decoder.decode(content.getComponentByName('content'),
2847 asn1Spec=rfc2315.SignedData())[0]
2849 certificates = content.getComponentByName('certificates')
2850 cert = certificates[0].getComponentByName('certificate')
2852 logging.error("Certificates not found.")
2854 return encoder.encode(cert)
2857 def load_stats_fdroid_signing_key_fingerprints():
2858 """Load list of signing-key fingerprints stored by fdroid publish from file.
2860 :returns: list of dictionanryies containing the singing-key fingerprints.
2862 jar_file = os.path.join('stats', 'publishsigkeys.jar')
2863 if not os.path.isfile(jar_file):
2865 cmd = [config['jarsigner'], '-strict', '-verify', jar_file]
2866 p = FDroidPopen(cmd, output=False)
2867 if p.returncode != 4:
2868 raise FDroidException("Signature validation of '{}' failed! "
2869 "Please run publish again to rebuild this file.".format(jar_file))
2871 jar_sigkey = apk_signer_fingerprint(jar_file)
2872 repo_key_sig = config.get('repo_key_sha256')
2874 if jar_sigkey != repo_key_sig:
2875 raise FDroidException("Signature key fingerprint of file '{}' does not match repo_key_sha256 in config.py (found fingerprint: '{}')".format(jar_file, jar_sigkey))
2877 logging.warning("repo_key_sha256 not in config.py, setting it to the signature key fingerprint of '{}'".format(jar_file))
2878 config['repo_key_sha256'] = jar_sigkey
2879 write_to_config(config, 'repo_key_sha256')
2881 with zipfile.ZipFile(jar_file, 'r') as f:
2882 return json.loads(str(f.read('publishsigkeys.json'), 'utf-8'))
2885 def write_to_config(thisconfig, key, value=None, config_file=None):
2886 '''write a key/value to the local config.py
2888 NOTE: only supports writing string variables.
2890 :param thisconfig: config dictionary
2891 :param key: variable name in config.py to be overwritten/added
2892 :param value: optional value to be written, instead of fetched
2893 from 'thisconfig' dictionary.
2896 origkey = key + '_orig'
2897 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2898 cfg = config_file if config_file else 'config.py'
2900 # load config file, create one if it doesn't exist
2901 if not os.path.exists(cfg):
2902 open(cfg, 'a').close()
2903 logging.info("Creating empty " + cfg)
2904 with open(cfg, 'r', encoding="utf-8") as f:
2905 lines = f.readlines()
2907 # make sure the file ends with a carraige return
2909 if not lines[-1].endswith('\n'):
2912 # regex for finding and replacing python string variable
2913 # definitions/initializations
2914 pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
2915 repl = key + ' = "' + value + '"'
2916 pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
2917 repl2 = key + " = '" + value + "'"
2919 # If we replaced this line once, we make sure won't be a
2920 # second instance of this line for this key in the document.
2923 with open(cfg, 'w', encoding="utf-8") as f:
2925 if pattern.match(line) or pattern2.match(line):
2927 line = pattern.sub(repl, line)
2928 line = pattern2.sub(repl2, line)
2939 def parse_xml(path):
2940 return XMLElementTree.parse(path).getroot()
2943 def string_is_integer(string):
2951 def local_rsync(options, fromdir, todir):
2952 '''Rsync method for local to local copying of things
2954 This is an rsync wrapper with all the settings for safe use within
2955 the various fdroidserver use cases. This uses stricter rsync
2956 checking on all files since people using offline mode are already
2957 prioritizing security above ease and speed.
2960 rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
2961 '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
2962 if not options.no_checksum:
2963 rsyncargs.append('--checksum')
2965 rsyncargs += ['--verbose']
2967 rsyncargs += ['--quiet']
2968 logging.debug(' '.join(rsyncargs + [fromdir, todir]))
2969 if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
2970 raise FDroidException()
2973 def get_per_app_repos():
2974 '''per-app repos are dirs named with the packageName of a single app'''
2976 # Android packageNames are Java packages, they may contain uppercase or
2977 # lowercase letters ('A' through 'Z'), numbers, and underscores
2978 # ('_'). However, individual package name parts may only start with
2979 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2980 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2983 for root, dirs, files in os.walk(os.getcwd()):
2985 print('checking', root, 'for', d)
2986 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2987 # standard parts of an fdroid repo, so never packageNames
2990 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
2996 def is_repo_file(filename):
2997 '''Whether the file in a repo is a build product to be delivered to users'''
2998 if isinstance(filename, str):
2999 filename = filename.encode('utf-8', errors="surrogateescape")
3000 return os.path.isfile(filename) \
3001 and not filename.endswith(b'.asc') \
3002 and not filename.endswith(b'.sig') \
3003 and os.path.basename(filename) not in [
3005 b'index_unsigned.jar',
3014 def get_examples_dir():
3015 '''Return the dir where the fdroidserver example files are available'''
3017 tmp = os.path.dirname(sys.argv[0])
3018 if os.path.basename(tmp) == 'bin':
3019 egg_links = glob.glob(os.path.join(tmp, '..',
3020 'local/lib/python3.*/site-packages/fdroidserver.egg-link'))
3022 # installed from local git repo
3023 examplesdir = os.path.join(open(egg_links[0]).readline().rstrip(), 'examples')
3026 examplesdir = os.path.dirname(os.path.dirname(__file__)) + '/share/doc/fdroidserver/examples'
3027 if not os.path.exists(examplesdir): # use UNIX layout
3028 examplesdir = os.path.dirname(tmp) + '/share/doc/fdroidserver/examples'
3030 # we're running straight out of the git repo
3031 prefix = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
3032 examplesdir = prefix + '/examples'
3037 def get_wiki_timestamp(timestamp=None):
3038 """Return current time in the standard format for posting to the wiki"""
3040 if timestamp is None:
3041 timestamp = time.gmtime()
3042 return time.strftime("%Y-%m-%d %H:%M:%SZ", timestamp)
3045 def get_android_tools_versions(ndk_path=None):
3046 '''get a list of the versions of all installed Android SDK/NDK components'''
3049 sdk_path = config['sdk_path']
3050 if sdk_path[-1] != '/':
3054 ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT')
3055 if os.path.isfile(ndk_release_txt):
3056 with open(ndk_release_txt, 'r') as fp:
3057 components.append((os.path.basename(ndk_path), fp.read()[:-1]))
3059 pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE)
3060 for root, dirs, files in os.walk(sdk_path):
3061 if 'source.properties' in files:
3062 source_properties = os.path.join(root, 'source.properties')
3063 with open(source_properties, 'r') as fp:
3064 m = pattern.search(fp.read())
3066 components.append((root[len(sdk_path):], m.group(1)))
3071 def get_android_tools_version_log(ndk_path=None):
3072 '''get a list of the versions of all installed Android SDK/NDK components'''
3073 log = '== Installed Android Tools ==\n\n'
3074 components = get_android_tools_versions(ndk_path)
3075 for name, version in sorted(components):
3076 log += '* ' + name + ' (' + version + ')\n'