3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
40 import xml.etree.ElementTree as XMLElementTree
42 from binascii import hexlify
43 from datetime import datetime, timedelta
44 from distutils.version import LooseVersion
45 from queue import Queue
46 from zipfile import ZipFile
48 from pyasn1.codec.der import decoder, encoder
49 from pyasn1_modules import rfc2315
50 from pyasn1.error import PyAsn1Error
52 from distutils.util import strtobool
54 import fdroidserver.metadata
55 from fdroidserver import _
56 from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException,\
57 BuildException, VerificationException
58 from .asynchronousfilereader import AsynchronousFileReader
60 # this is the build-tools version, aapt has a separate version that
61 # has to be manually set in test_aapt_version()
62 MINIMUM_AAPT_VERSION = '26.0.0'
64 VERCODE_OPERATION_RE = re.compile(r'^([ 0-9/*+-]|%c)+$')
66 # A signature block file with a .DSA, .RSA, or .EC extension
67 CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
68 APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
69 STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
71 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
80 'sdk_path': "$ANDROID_HOME",
84 'r12b': "$ANDROID_NDK",
90 'build_tools': MINIMUM_AAPT_VERSION,
91 'force_build_tools': False,
96 'accepted_formats': ['txt', 'yml'],
97 'sync_from_local_copy_dir': False,
98 'allow_disabled_algorithms': False,
99 'per_app_repos': False,
100 'make_current_version_link': True,
101 'current_version_name_source': 'Name',
102 'update_stats': False,
104 'stats_server': None,
106 'stats_to_carbon': False,
108 'build_server_always': False,
109 'keystore': 'keystore.jks',
110 'smartcardoptions': [],
120 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
121 'repo_name': "My First FDroid Repo Demo",
122 'repo_icon': "fdroid-icon.png",
123 'repo_description': '''
124 This is a repository of apps to be used with FDroid. Applications in this
125 repository are either official binaries built by the original application
126 developers, or are binaries built from source by the admin of f-droid.org
127 using the tools on https://gitlab.com/u/fdroid.
133 def setup_global_opts(parser):
134 try: # the buildserver VM might not have PIL installed
135 from PIL import PngImagePlugin
136 logger = logging.getLogger(PngImagePlugin.__name__)
137 logger.setLevel(logging.INFO) # tame the "STREAM" debug messages
141 parser.add_argument("-v", "--verbose", action="store_true", default=False,
142 help=_("Spew out even more information than normal"))
143 parser.add_argument("-q", "--quiet", action="store_true", default=False,
144 help=_("Restrict output to warnings and errors"))
147 def _add_java_paths_to_config(pathlist, thisconfig):
148 def path_version_key(s):
150 for u in re.split('[^0-9]+', s):
152 versionlist.append(int(u))
157 for d in sorted(pathlist, key=path_version_key):
158 if os.path.islink(d):
160 j = os.path.basename(d)
161 # the last one found will be the canonical one, so order appropriately
163 r'^1\.([6-9])\.0\.jdk$', # OSX
164 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
165 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
166 r'^jdk([6-9])-openjdk$', # Arch
167 r'^java-([6-9])-openjdk$', # Arch
168 r'^java-([6-9])-jdk$', # Arch (oracle)
169 r'^java-1\.([6-9])\.0-.*$', # RedHat
170 r'^java-([6-9])-oracle$', # Debian WebUpd8
171 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
172 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
173 r'^oracle-jdk-bin-1\.([7-9]).*$', # Gentoo (oracle)
174 r'^icedtea-bin-([7-9]).*$', # Gentoo (openjdk)
176 m = re.match(regex, j)
179 for p in [d, os.path.join(d, 'Contents', 'Home')]:
180 if os.path.exists(os.path.join(p, 'bin', 'javac')):
181 thisconfig['java_paths'][m.group(1)] = p
184 def fill_config_defaults(thisconfig):
185 for k, v in default_config.items():
186 if k not in thisconfig:
189 # Expand paths (~users and $vars)
190 def expand_path(path):
194 path = os.path.expanduser(path)
195 path = os.path.expandvars(path)
200 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
205 thisconfig[k + '_orig'] = v
207 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
208 if thisconfig['java_paths'] is None:
209 thisconfig['java_paths'] = dict()
211 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
212 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
213 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
214 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
215 pathlist += glob.glob('/opt/oracle-jdk-*1.[7-9]*')
216 pathlist += glob.glob('/opt/icedtea-*[7-9]*')
217 if os.getenv('JAVA_HOME') is not None:
218 pathlist.append(os.getenv('JAVA_HOME'))
219 if os.getenv('PROGRAMFILES') is not None:
220 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
221 _add_java_paths_to_config(pathlist, thisconfig)
223 for java_version in ('7', '8', '9'):
224 if java_version not in thisconfig['java_paths']:
226 java_home = thisconfig['java_paths'][java_version]
227 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
228 if os.path.exists(jarsigner):
229 thisconfig['jarsigner'] = jarsigner
230 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
231 break # Java7 is preferred, so quit if found
233 for k in ['ndk_paths', 'java_paths']:
239 thisconfig[k][k2] = exp
240 thisconfig[k][k2 + '_orig'] = v
243 def regsub_file(pattern, repl, path):
244 with open(path, 'rb') as f:
246 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
247 with open(path, 'wb') as f:
251 def read_config(opts, config_file='config.py'):
252 """Read the repository config
254 The config is read from config_file, which is in the current
255 directory when any of the repo management commands are used. If
256 there is a local metadata file in the git repo, then config.py is
257 not required, just use defaults.
260 global config, options
262 if config is not None:
269 if os.path.isfile(config_file):
270 logging.debug(_("Reading '{config_file}'").format(config_file=config_file))
271 with io.open(config_file, "rb") as f:
272 code = compile(f.read(), config_file, 'exec')
273 exec(code, None, config)
275 logging.warning(_("No 'config.py' found, using defaults."))
277 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
279 if not type(config[k]) in (str, list, tuple):
281 _("'{field}' will be in random order! Use () or [] brackets if order is important!")
284 # smartcardoptions must be a list since its command line args for Popen
285 if 'smartcardoptions' in config:
286 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
287 elif 'keystore' in config and config['keystore'] == 'NONE':
288 # keystore='NONE' means use smartcard, these are required defaults
289 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
290 'SunPKCS11-OpenSC', '-providerClass',
291 'sun.security.pkcs11.SunPKCS11',
292 '-providerArg', 'opensc-fdroid.cfg']
294 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
295 st = os.stat(config_file)
296 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
297 logging.warning(_("unsafe permissions on '{config_file}' (should be 0600)!")
298 .format(config_file=config_file))
300 fill_config_defaults(config)
302 for k in ["repo_description", "archive_description"]:
304 config[k] = clean_description(config[k])
306 if 'serverwebroot' in config:
307 if isinstance(config['serverwebroot'], str):
308 roots = [config['serverwebroot']]
309 elif all(isinstance(item, str) for item in config['serverwebroot']):
310 roots = config['serverwebroot']
312 raise TypeError(_('only accepts strings, lists, and tuples'))
314 for rootstr in roots:
315 # since this is used with rsync, where trailing slashes have
316 # meaning, ensure there is always a trailing slash
317 if rootstr[-1] != '/':
319 rootlist.append(rootstr.replace('//', '/'))
320 config['serverwebroot'] = rootlist
322 if 'servergitmirrors' in config:
323 if isinstance(config['servergitmirrors'], str):
324 roots = [config['servergitmirrors']]
325 elif all(isinstance(item, str) for item in config['servergitmirrors']):
326 roots = config['servergitmirrors']
328 raise TypeError(_('only accepts strings, lists, and tuples'))
329 config['servergitmirrors'] = roots
334 def assert_config_keystore(config):
335 """Check weather keystore is configured correctly and raise exception if not."""
338 if 'repo_keyalias' not in config:
340 logging.critical(_("'repo_keyalias' not found in config.py!"))
341 if 'keystore' not in config:
343 logging.critical(_("'keystore' not found in config.py!"))
344 elif not os.path.exists(config['keystore']):
346 logging.critical("'" + config['keystore'] + "' does not exist!")
347 if 'keystorepass' not in config:
349 logging.critical(_("'keystorepass' not found in config.py!"))
350 if 'keypass' not in config:
352 logging.critical(_("'keypass' not found in config.py!"))
354 raise FDroidException("This command requires a signing key, " +
355 "you can create one using: fdroid update --create-key")
358 def find_sdk_tools_cmd(cmd):
359 '''find a working path to a tool from the Android SDK'''
362 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
363 # try to find a working path to this command, in all the recent possible paths
364 if 'build_tools' in config:
365 build_tools = os.path.join(config['sdk_path'], 'build-tools')
366 # if 'build_tools' was manually set and exists, check only that one
367 configed_build_tools = os.path.join(build_tools, config['build_tools'])
368 if os.path.exists(configed_build_tools):
369 tooldirs.append(configed_build_tools)
371 # no configed version, so hunt known paths for it
372 for f in sorted(os.listdir(build_tools), reverse=True):
373 if os.path.isdir(os.path.join(build_tools, f)):
374 tooldirs.append(os.path.join(build_tools, f))
375 tooldirs.append(build_tools)
376 sdk_tools = os.path.join(config['sdk_path'], 'tools')
377 if os.path.exists(sdk_tools):
378 tooldirs.append(sdk_tools)
379 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
380 if os.path.exists(sdk_platform_tools):
381 tooldirs.append(sdk_platform_tools)
382 tooldirs.append('/usr/bin')
384 path = os.path.join(d, cmd)
385 if os.path.isfile(path):
387 test_aapt_version(path)
389 # did not find the command, exit with error message
390 ensure_build_tools_exists(config)
393 def test_aapt_version(aapt):
394 '''Check whether the version of aapt is new enough'''
395 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
396 if output is None or output == '':
397 logging.error(_("'{path}' failed to execute!").format(path=aapt))
399 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
404 # the Debian package has the version string like "v0.2-23.0.2"
407 if LooseVersion(bugfix) < LooseVersion(MINIMUM_AAPT_VERSION):
409 elif LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.4062713'):
412 logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-{version} or newer!")
413 .format(aapt=aapt, version=MINIMUM_AAPT_VERSION))
415 logging.warning(_('Unknown version of aapt, might cause problems: ') + output)
418 def test_sdk_exists(thisconfig):
419 if 'sdk_path' not in thisconfig:
420 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
421 test_aapt_version(thisconfig['aapt'])
424 logging.error(_("'sdk_path' not set in 'config.py'!"))
426 if thisconfig['sdk_path'] == default_config['sdk_path']:
427 logging.error(_('No Android SDK found!'))
428 logging.error(_('You can use ANDROID_HOME to set the path to your SDK, i.e.:'))
429 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
431 if not os.path.exists(thisconfig['sdk_path']):
432 logging.critical(_("Android SDK path '{path}' does not exist!")
433 .format(path=thisconfig['sdk_path']))
435 if not os.path.isdir(thisconfig['sdk_path']):
436 logging.critical(_("Android SDK path '{path}' is not a directory!")
437 .format(path=thisconfig['sdk_path']))
439 for d in ['build-tools', 'platform-tools', 'tools']:
440 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
441 logging.critical(_("Android SDK '{path}' does not have '{dirname}' installed!")
442 .format(path=thisconfig['sdk_path'], dirname=d))
447 def ensure_build_tools_exists(thisconfig):
448 if not test_sdk_exists(thisconfig):
449 raise FDroidException(_("Android SDK not found!"))
450 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
451 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
452 if not os.path.isdir(versioned_build_tools):
453 raise FDroidException(
454 _("Android build-tools path '{path}' does not exist!")
455 .format(path=versioned_build_tools))
458 def get_local_metadata_files():
459 '''get any metadata files local to an app's source repo
461 This tries to ignore anything that does not count as app metdata,
462 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
465 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
468 def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False):
470 :param appids: arguments in the form of multiple appid:[vc] strings
471 :returns: a dictionary with the set of vercodes specified for each package
474 if not appid_versionCode_pairs:
477 for p in appid_versionCode_pairs:
478 if allow_vercodes and ':' in p:
479 package, vercode = p.split(':')
481 package, vercode = p, None
482 if package not in vercodes:
483 vercodes[package] = [vercode] if vercode else []
485 elif vercode and vercode not in vercodes[package]:
486 vercodes[package] += [vercode] if vercode else []
491 def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
492 """Build a list of App instances for processing
494 On top of what read_pkg_args does, this returns the whole app
495 metadata, but limiting the builds list to the builds matching the
496 appid_versionCode_pairs and vercodes specified. If no appid_versionCode_pairs are specified, then
497 all App and Build instances are returned.
501 vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes)
507 for appid, app in allapps.items():
508 if appid in vercodes:
511 if len(apps) != len(vercodes):
514 logging.critical(_("No such package: %s") % p)
515 raise FDroidException(_("Found invalid appids in arguments"))
517 raise FDroidException(_("No packages specified"))
520 for appid, app in apps.items():
524 app.builds = [b for b in app.builds if b.versionCode in vc]
525 if len(app.builds) != len(vercodes[appid]):
527 allvcs = [b.versionCode for b in app.builds]
528 for v in vercodes[appid]:
530 logging.critical(_("No such versionCode {versionCode} for app {appid}")
531 .format(versionCode=v, appid=appid))
534 raise FDroidException(_("Found invalid versionCodes for some apps"))
539 def get_extension(filename):
540 base, ext = os.path.splitext(filename)
543 return base, ext.lower()[1:]
546 def has_extension(filename, ext):
547 _ignored, f_ext = get_extension(filename)
551 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
554 def clean_description(description):
555 'Remove unneeded newlines and spaces from a block of description text'
557 # this is split up by paragraph to make removing the newlines easier
558 for paragraph in re.split(r'\n\n', description):
559 paragraph = re.sub('\r', '', paragraph)
560 paragraph = re.sub('\n', ' ', paragraph)
561 paragraph = re.sub(' {2,}', ' ', paragraph)
562 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
563 returnstring += paragraph + '\n\n'
564 return returnstring.rstrip('\n')
567 def publishednameinfo(filename):
568 filename = os.path.basename(filename)
569 m = publish_name_regex.match(filename)
571 result = (m.group(1), m.group(2))
572 except AttributeError:
573 raise FDroidException(_("Invalid name for published file: %s") % filename)
577 apk_release_filename = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)\.apk')
578 apk_release_filename_with_sigfp = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)_(?P<sigfp>[0-9a-f]{7})\.apk')
581 def apk_parse_release_filename(apkname):
582 """Parses the name of an APK file according the F-Droids APK naming
583 scheme and returns the tokens.
585 WARNING: Returned values don't necessarily represent the APKs actual
586 properties, the are just paresed from the file name.
588 :returns: A triplet containing (appid, versionCode, signer), where appid
589 should be the package name, versionCode should be the integer
590 represion of the APKs version and signer should be the first 7 hex
591 digists of the sha256 signing key fingerprint which was used to sign
594 m = apk_release_filename_with_sigfp.match(apkname)
596 return m.group('appid'), m.group('vercode'), m.group('sigfp')
597 m = apk_release_filename.match(apkname)
599 return m.group('appid'), m.group('vercode'), None
600 return None, None, None
603 def get_release_filename(app, build):
605 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
607 return "%s_%s.apk" % (app.id, build.versionCode)
610 def get_toolsversion_logname(app, build):
611 return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
614 def getsrcname(app, build):
615 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
627 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
630 def get_build_dir(app):
631 '''get the dir that this app will be built in'''
633 if app.RepoType == 'srclib':
634 return os.path.join('build', 'srclib', app.Repo)
636 return os.path.join('build', app.id)
640 '''checkout code from VCS and return instance of vcs and the build dir'''
641 build_dir = get_build_dir(app)
643 # Set up vcs interface and make sure we have the latest code...
644 logging.debug("Getting {0} vcs interface for {1}"
645 .format(app.RepoType, app.Repo))
646 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
650 vcs = getvcs(app.RepoType, remote, build_dir)
652 return vcs, build_dir
655 def getvcs(vcstype, remote, local):
657 return vcs_git(remote, local)
658 if vcstype == 'git-svn':
659 return vcs_gitsvn(remote, local)
661 return vcs_hg(remote, local)
663 return vcs_bzr(remote, local)
664 if vcstype == 'srclib':
665 if local != os.path.join('build', 'srclib', remote):
666 raise VCSException("Error: srclib paths are hard-coded!")
667 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
669 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
670 raise VCSException("Invalid vcs type " + vcstype)
673 def getsrclibvcs(name):
674 if name not in fdroidserver.metadata.srclibs:
675 raise VCSException("Missing srclib " + name)
676 return fdroidserver.metadata.srclibs[name]['Repo Type']
681 def __init__(self, remote, local):
683 # svn, git-svn and bzr may require auth
685 if self.repotype() in ('git-svn', 'bzr'):
687 if self.repotype == 'git-svn':
688 raise VCSException("Authentication is not supported for git-svn")
689 self.username, remote = remote.split('@')
690 if ':' not in self.username:
691 raise VCSException(_("Password required with username"))
692 self.username, self.password = self.username.split(':')
696 self.clone_failed = False
697 self.refreshed = False
703 def clientversion(self):
704 versionstr = FDroidPopen(self.clientversioncmd()).output
705 return versionstr[0:versionstr.find('\n')]
707 def clientversioncmd(self):
710 def gotorevision(self, rev, refresh=True):
711 """Take the local repository to a clean version of the given
712 revision, which is specificed in the VCS's native
713 format. Beforehand, the repository can be dirty, or even
714 non-existent. If the repository does already exist locally, it
715 will be updated from the origin, but only once in the lifetime
716 of the vcs object. None is acceptable for 'rev' if you know
717 you are cloning a clean copy of the repo - otherwise it must
718 specify a valid revision.
721 if self.clone_failed:
722 raise VCSException(_("Downloading the repository already failed once, not trying again."))
724 # The .fdroidvcs-id file for a repo tells us what VCS type
725 # and remote that directory was created from, allowing us to drop it
726 # automatically if either of those things changes.
727 fdpath = os.path.join(self.local, '..',
728 '.fdroidvcs-' + os.path.basename(self.local))
729 fdpath = os.path.normpath(fdpath)
730 cdata = self.repotype() + ' ' + self.remote
733 if os.path.exists(self.local):
734 if os.path.exists(fdpath):
735 with open(fdpath, 'r') as f:
736 fsdata = f.read().strip()
741 logging.info("Repository details for %s changed - deleting" % (
745 logging.info("Repository details for %s missing - deleting" % (
748 shutil.rmtree(self.local)
752 self.refreshed = True
755 self.gotorevisionx(rev)
756 except FDroidException as e:
759 # If necessary, write the .fdroidvcs file.
760 if writeback and not self.clone_failed:
761 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
762 with open(fdpath, 'w+') as f:
768 def gotorevisionx(self, rev): # pylint: disable=unused-argument
769 """Derived classes need to implement this.
771 It's called once basic checking has been performed.
773 raise VCSException("This VCS type doesn't define gotorevisionx")
775 # Initialise and update submodules
776 def initsubmodules(self):
777 raise VCSException('Submodules not supported for this vcs type')
779 # Get a list of all known tags
781 if not self._gettags:
782 raise VCSException('gettags not supported for this vcs type')
784 for tag in self._gettags():
785 if re.match('[-A-Za-z0-9_. /]+$', tag):
789 def latesttags(self):
790 """Get a list of all the known tags, sorted from newest to oldest"""
791 raise VCSException('latesttags not supported for this vcs type')
794 """Get current commit reference (hash, revision, etc)"""
795 raise VCSException('getref not supported for this vcs type')
798 """Returns the srclib (name, path) used in setting up the current revision, or None."""
807 def clientversioncmd(self):
808 return ['git', '--version']
810 def git(self, args, envs=dict(), cwd=None, output=True):
811 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
813 While fetch/pull/clone respect the command line option flags,
814 it seems that submodule commands do not. They do seem to
815 follow whatever is in env vars, if the version of git is new
816 enough. So we just throw the kitchen sink at it to see what
819 Also, because of CVE-2017-1000117, block all SSH URLs.
822 # supported in git >= 2.3
824 '-c', 'core.askpass=/bin/true',
825 '-c', 'core.sshCommand=/bin/false',
826 '-c', 'url.https://.insteadOf=ssh://',
828 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
829 git_config.append('-c')
830 git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':')
831 git_config.append('-c')
832 git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain)
833 git_config.append('-c')
834 git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain)
836 'GIT_TERMINAL_PROMPT': '0',
837 'GIT_ASKPASS': '/bin/true',
838 'SSH_ASKPASS': '/bin/true',
839 'GIT_SSH': '/bin/false', # for git < 2.3
841 return FDroidPopen(['git', ] + git_config + args,
842 envs=envs, cwd=cwd, output=output)
845 """If the local directory exists, but is somehow not a git repository,
846 git will traverse up the directory tree until it finds one
847 that is (i.e. fdroidserver) and then we'll proceed to destroy
848 it! This is called as a safety check.
852 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
853 result = p.output.rstrip()
854 if not result.endswith(self.local):
855 raise VCSException('Repository mismatch')
857 def gotorevisionx(self, rev):
858 if not os.path.exists(self.local):
860 p = self.git(['clone', '--', self.remote, self.local])
861 if p.returncode != 0:
862 self.clone_failed = True
863 raise VCSException("Git clone failed", p.output)
867 # Discard any working tree changes
868 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
869 'git', 'reset', '--hard'], cwd=self.local, output=False)
870 if p.returncode != 0:
871 raise VCSException(_("Git reset failed"), p.output)
872 # Remove untracked files now, in case they're tracked in the target
873 # revision (it happens!)
874 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
875 'git', 'clean', '-dffx'], cwd=self.local, output=False)
876 if p.returncode != 0:
877 raise VCSException(_("Git clean failed"), p.output)
878 if not self.refreshed:
879 # Get latest commits and tags from remote
880 p = self.git(['fetch', 'origin'], cwd=self.local)
881 if p.returncode != 0:
882 raise VCSException(_("Git fetch failed"), p.output)
883 p = self.git(['fetch', '--prune', '--tags', 'origin'], output=False, cwd=self.local)
884 if p.returncode != 0:
885 raise VCSException(_("Git fetch failed"), p.output)
886 # Recreate origin/HEAD as git clone would do it, in case it disappeared
887 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
888 if p.returncode != 0:
889 lines = p.output.splitlines()
890 if 'Multiple remote HEAD branches' not in lines[0]:
891 raise VCSException(_("Git remote set-head failed"), p.output)
892 branch = lines[1].split(' ')[-1]
893 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--', branch],
894 cwd=self.local, output=False)
895 if p2.returncode != 0:
896 raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output)
897 self.refreshed = True
898 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
899 # a github repo. Most of the time this is the same as origin/master.
900 rev = rev or 'origin/HEAD'
901 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
902 if p.returncode != 0:
903 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
904 # Get rid of any uncontrolled files left behind
905 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
906 if p.returncode != 0:
907 raise VCSException(_("Git clean failed"), p.output)
909 def initsubmodules(self):
911 submfile = os.path.join(self.local, '.gitmodules')
912 if not os.path.isfile(submfile):
913 raise NoSubmodulesException(_("No git submodules available"))
915 # fix submodules not accessible without an account and public key auth
916 with open(submfile, 'r') as f:
917 lines = f.readlines()
918 with open(submfile, 'w') as f:
920 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
921 line = re.sub('git@' + domain + ':', 'https://u:p@' + domain + '/', line)
924 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
925 if p.returncode != 0:
926 raise VCSException(_("Git submodule sync failed"), p.output)
927 p = self.git(['submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
928 if p.returncode != 0:
929 raise VCSException(_("Git submodule update failed"), p.output)
933 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
934 return p.output.splitlines()
936 tag_format = re.compile(r'tag: ([^),]*)')
938 def latesttags(self):
940 p = FDroidPopen(['git', 'log', '--tags',
941 '--simplify-by-decoration', '--pretty=format:%d'],
942 cwd=self.local, output=False)
944 for line in p.output.splitlines():
945 for tag in self.tag_format.findall(line):
950 class vcs_gitsvn(vcs):
955 def clientversioncmd(self):
956 return ['git', 'svn', '--version']
959 """If the local directory exists, but is somehow not a git repository,
960 git will traverse up the directory tree until it finds one that
961 is (i.e. fdroidserver) and then we'll proceed to destory it!
962 This is called as a safety check.
965 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
966 result = p.output.rstrip()
967 if not result.endswith(self.local):
968 raise VCSException('Repository mismatch')
970 def git(self, args, envs=dict(), cwd=None, output=True):
971 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
973 AskPass is set to /bin/true to let the process try to connect
974 without a username/password.
976 The SSH command is set to /bin/false to block all SSH URLs
977 (supported in git >= 2.3). This protects against
982 '-c', 'core.askpass=/bin/true',
983 '-c', 'core.sshCommand=/bin/false',
986 'GIT_TERMINAL_PROMPT': '0',
987 'GIT_ASKPASS': '/bin/true',
988 'SSH_ASKPASS': '/bin/true',
989 'GIT_SSH': '/bin/false', # for git < 2.3
990 'SVN_SSH': '/bin/false',
992 return FDroidPopen(['git', ] + git_config + args,
993 envs=envs, cwd=cwd, output=output)
995 def gotorevisionx(self, rev):
996 if not os.path.exists(self.local):
998 gitsvn_args = ['svn', 'clone']
1000 if ';' in self.remote:
1001 remote_split = self.remote.split(';')
1002 for i in remote_split[1:]:
1003 if i.startswith('trunk='):
1004 gitsvn_args.extend(['-T', i[6:]])
1005 elif i.startswith('tags='):
1006 gitsvn_args.extend(['-t', i[5:]])
1007 elif i.startswith('branches='):
1008 gitsvn_args.extend(['-b', i[9:]])
1009 remote = remote_split[0]
1011 remote = self.remote
1013 if not remote.startswith('https://'):
1014 raise VCSException(_('HTTPS must be used with Subversion URLs!'))
1016 # git-svn sucks at certificate validation, this throws useful errors:
1018 r = requests.head(remote)
1019 r.raise_for_status()
1020 location = r.headers.get('location')
1021 if location and not location.startswith('https://'):
1022 raise VCSException(_('Invalid redirect to non-HTTPS: {before} -> {after} ')
1023 .format(before=remote, after=location))
1025 gitsvn_args.extend(['--', remote, self.local])
1026 p = self.git(gitsvn_args)
1027 if p.returncode != 0:
1028 self.clone_failed = True
1029 raise VCSException(_('git svn clone failed'), p.output)
1033 # Discard any working tree changes
1034 p = self.git(['reset', '--hard'], cwd=self.local, output=False)
1035 if p.returncode != 0:
1036 raise VCSException("Git reset failed", p.output)
1037 # Remove untracked files now, in case they're tracked in the target
1038 # revision (it happens!)
1039 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1040 if p.returncode != 0:
1041 raise VCSException("Git clean failed", p.output)
1042 if not self.refreshed:
1043 # Get new commits, branches and tags from repo
1044 p = self.git(['svn', 'fetch'], cwd=self.local, output=False)
1045 if p.returncode != 0:
1046 raise VCSException("Git svn fetch failed")
1047 p = self.git(['svn', 'rebase'], cwd=self.local, output=False)
1048 if p.returncode != 0:
1049 raise VCSException("Git svn rebase failed", p.output)
1050 self.refreshed = True
1052 rev = rev or 'master'
1054 nospaces_rev = rev.replace(' ', '%20')
1055 # Try finding a svn tag
1056 for treeish in ['origin/', '']:
1057 p = self.git(['checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
1058 if p.returncode == 0:
1060 if p.returncode != 0:
1061 # No tag found, normal svn rev translation
1062 # Translate svn rev into git format
1063 rev_split = rev.split('/')
1066 for treeish in ['origin/', '']:
1067 if len(rev_split) > 1:
1068 treeish += rev_split[0]
1069 svn_rev = rev_split[1]
1072 # if no branch is specified, then assume trunk (i.e. 'master' branch):
1076 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
1078 p = self.git(['svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
1079 git_rev = p.output.rstrip()
1081 if p.returncode == 0 and git_rev:
1084 if p.returncode != 0 or not git_rev:
1085 # Try a plain git checkout as a last resort
1086 p = self.git(['checkout', rev], cwd=self.local, output=False)
1087 if p.returncode != 0:
1088 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
1090 # Check out the git rev equivalent to the svn rev
1091 p = self.git(['checkout', git_rev], cwd=self.local, output=False)
1092 if p.returncode != 0:
1093 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
1095 # Get rid of any uncontrolled files left behind
1096 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1097 if p.returncode != 0:
1098 raise VCSException(_("Git clean failed"), p.output)
1102 for treeish in ['origin/', '']:
1103 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
1104 if os.path.isdir(d):
1105 return os.listdir(d)
1109 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
1110 if p.returncode != 0:
1112 return p.output.strip()
1120 def clientversioncmd(self):
1121 return ['hg', '--version']
1123 def gotorevisionx(self, rev):
1124 if not os.path.exists(self.local):
1125 p = FDroidPopen(['hg', 'clone', '--ssh', '/bin/false', '--', self.remote, self.local],
1127 if p.returncode != 0:
1128 self.clone_failed = True
1129 raise VCSException("Hg clone failed", p.output)
1131 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
1132 if p.returncode != 0:
1133 raise VCSException("Hg status failed", p.output)
1134 for line in p.output.splitlines():
1135 if not line.startswith('? '):
1136 raise VCSException("Unexpected output from hg status -uS: " + line)
1137 FDroidPopen(['rm', '-rf', '--', line[2:]], cwd=self.local, output=False)
1138 if not self.refreshed:
1139 p = FDroidPopen(['hg', 'pull', '--ssh', '/bin/false'], cwd=self.local, output=False)
1140 if p.returncode != 0:
1141 raise VCSException("Hg pull failed", p.output)
1142 self.refreshed = True
1144 rev = rev or 'default'
1147 p = FDroidPopen(['hg', 'update', '-C', '--', rev], cwd=self.local, output=False)
1148 if p.returncode != 0:
1149 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
1150 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1151 # Also delete untracked files, we have to enable purge extension for that:
1152 if "'purge' is provided by the following extension" in p.output:
1153 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
1154 myfile.write("\n[extensions]\nhgext.purge=\n")
1155 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1156 if p.returncode != 0:
1157 raise VCSException("HG purge failed", p.output)
1158 elif p.returncode != 0:
1159 raise VCSException("HG purge failed", p.output)
1162 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
1163 return p.output.splitlines()[1:]
1171 def clientversioncmd(self):
1172 return ['bzr', '--version']
1174 def bzr(self, args, envs=dict(), cwd=None, output=True):
1175 '''Prevent bzr from ever using SSH to avoid security vulns'''
1179 return FDroidPopen(['bzr', ] + args, envs=envs, cwd=cwd, output=output)
1181 def gotorevisionx(self, rev):
1182 if not os.path.exists(self.local):
1183 p = self.bzr(['branch', self.remote, self.local], output=False)
1184 if p.returncode != 0:
1185 self.clone_failed = True
1186 raise VCSException("Bzr branch failed", p.output)
1188 p = self.bzr(['clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
1189 if p.returncode != 0:
1190 raise VCSException("Bzr revert failed", p.output)
1191 if not self.refreshed:
1192 p = self.bzr(['pull'], cwd=self.local, output=False)
1193 if p.returncode != 0:
1194 raise VCSException("Bzr update failed", p.output)
1195 self.refreshed = True
1197 revargs = list(['-r', rev] if rev else [])
1198 p = self.bzr(['revert'] + revargs, cwd=self.local, output=False)
1199 if p.returncode != 0:
1200 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1203 p = self.bzr(['tags'], cwd=self.local, output=False)
1204 return [tag.split(' ')[0].strip() for tag in
1205 p.output.splitlines()]
1208 def unescape_string(string):
1211 if string[0] == '"' and string[-1] == '"':
1214 return string.replace("\\'", "'")
1217 def retrieve_string(app_dir, string, xmlfiles=None):
1219 if not string.startswith('@string/'):
1220 return unescape_string(string)
1222 if xmlfiles is None:
1225 os.path.join(app_dir, 'res'),
1226 os.path.join(app_dir, 'src', 'main', 'res'),
1228 for root, dirs, files in os.walk(res_dir):
1229 if os.path.basename(root) == 'values':
1230 xmlfiles += [os.path.join(root, x) for x in files if x.endswith('.xml')]
1232 name = string[len('@string/'):]
1234 def element_content(element):
1235 if element.text is None:
1237 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1238 return s.decode('utf-8').strip()
1240 for path in xmlfiles:
1241 if not os.path.isfile(path):
1243 xml = parse_xml(path)
1244 element = xml.find('string[@name="' + name + '"]')
1245 if element is not None:
1246 content = element_content(element)
1247 return retrieve_string(app_dir, content, xmlfiles)
1252 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1253 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1256 def manifest_paths(app_dir, flavours):
1257 '''Return list of existing files that will be used to find the highest vercode'''
1259 possible_manifests = \
1260 [os.path.join(app_dir, 'AndroidManifest.xml'),
1261 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1262 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1263 os.path.join(app_dir, 'build.gradle')]
1265 for flavour in flavours:
1266 if flavour == 'yes':
1268 possible_manifests.append(
1269 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1271 return [path for path in possible_manifests if os.path.isfile(path)]
1274 def fetch_real_name(app_dir, flavours):
1275 '''Retrieve the package name. Returns the name, or None if not found.'''
1276 for path in manifest_paths(app_dir, flavours):
1277 if not has_extension(path, 'xml') or not os.path.isfile(path):
1279 logging.debug("fetch_real_name: Checking manifest at " + path)
1280 xml = parse_xml(path)
1281 app = xml.find('application')
1284 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1286 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1287 result = retrieve_string_singleline(app_dir, label)
1289 result = result.strip()
1294 def get_library_references(root_dir):
1296 proppath = os.path.join(root_dir, 'project.properties')
1297 if not os.path.isfile(proppath):
1299 with open(proppath, 'r', encoding='iso-8859-1') as f:
1301 if not line.startswith('android.library.reference.'):
1303 path = line.split('=')[1].strip()
1304 relpath = os.path.join(root_dir, path)
1305 if not os.path.isdir(relpath):
1307 logging.debug("Found subproject at %s" % path)
1308 libraries.append(path)
1312 def ant_subprojects(root_dir):
1313 subprojects = get_library_references(root_dir)
1314 for subpath in subprojects:
1315 subrelpath = os.path.join(root_dir, subpath)
1316 for p in get_library_references(subrelpath):
1317 relp = os.path.normpath(os.path.join(subpath, p))
1318 if relp not in subprojects:
1319 subprojects.insert(0, relp)
1323 def remove_debuggable_flags(root_dir):
1324 # Remove forced debuggable flags
1325 logging.debug("Removing debuggable flags from %s" % root_dir)
1326 for root, dirs, files in os.walk(root_dir):
1327 if 'AndroidManifest.xml' in files and os.path.isfile(os.path.join(root, 'AndroidManifest.xml')):
1328 regsub_file(r'android:debuggable="[^"]*"',
1330 os.path.join(root, 'AndroidManifest.xml'))
1333 vcsearch_g = re.compile(r'''.*[Vv]ersionCode\s*=?\s*["']*([0-9]+)["']*''').search
1334 vnsearch_g = re.compile(r'''.*[Vv]ersionName\s*=?\s*(["'])((?:(?=(\\?))\3.)*?)\1.*''').search
1335 psearch_g = re.compile(r'''.*(packageName|applicationId)\s*=*\s*["']([^"']+)["'].*''').search
1338 def app_matches_packagename(app, package):
1341 appid = app.UpdateCheckName or app.id
1342 if appid is None or appid == "Ignore":
1344 return appid == package
1347 def parse_androidmanifests(paths, app):
1349 Extract some information from the AndroidManifest.xml at the given path.
1350 Returns (version, vercode, package), any or all of which might be None.
1351 All values returned are strings.
1354 ignoreversions = app.UpdateCheckIgnore
1355 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1358 return (None, None, None)
1366 if not os.path.isfile(path):
1369 logging.debug(_("Parsing manifest at '{path}'").format(path=path))
1375 if app.builds and 'gradle' in app.builds[-1] and app.builds[-1].gradle:
1376 flavour = app.builds[-1].gradle[-1]
1378 if has_extension(path, 'gradle'):
1379 with open(path, 'r') as f:
1380 inside_flavour_group = 0
1381 inside_required_flavour = 0
1383 if gradle_comment.match(line):
1386 if inside_flavour_group > 0:
1387 if inside_required_flavour > 0:
1388 matches = psearch_g(line)
1390 s = matches.group(2)
1391 if app_matches_packagename(app, s):
1394 matches = vnsearch_g(line)
1396 version = matches.group(2)
1398 matches = vcsearch_g(line)
1400 vercode = matches.group(1)
1403 inside_required_flavour += 1
1405 inside_required_flavour -= 1
1407 if flavour and (flavour in line):
1408 inside_required_flavour = 1
1411 inside_flavour_group += 1
1413 inside_flavour_group -= 1
1415 if "productFlavors" in line:
1416 inside_flavour_group = 1
1418 matches = psearch_g(line)
1420 s = matches.group(2)
1421 if app_matches_packagename(app, s):
1424 matches = vnsearch_g(line)
1426 version = matches.group(2)
1428 matches = vcsearch_g(line)
1430 vercode = matches.group(1)
1433 xml = parse_xml(path)
1434 if "package" in xml.attrib:
1435 s = xml.attrib["package"]
1436 if app_matches_packagename(app, s):
1438 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1439 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1440 base_dir = os.path.dirname(path)
1441 version = retrieve_string_singleline(base_dir, version)
1442 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1443 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1444 if string_is_integer(a):
1447 logging.warning(_("Problem with xml at '{path}'").format(path=path))
1449 # Remember package name, may be defined separately from version+vercode
1451 package = max_package
1453 logging.debug("..got package={0}, version={1}, vercode={2}"
1454 .format(package, version, vercode))
1456 # Always grab the package name and version name in case they are not
1457 # together with the highest version code
1458 if max_package is None and package is not None:
1459 max_package = package
1460 if max_version is None and version is not None:
1461 max_version = version
1463 if vercode is not None \
1464 and (max_vercode is None or vercode > max_vercode):
1465 if not ignoresearch or not ignoresearch(version):
1466 if version is not None:
1467 max_version = version
1468 if vercode is not None:
1469 max_vercode = vercode
1470 if package is not None:
1471 max_package = package
1473 max_version = "Ignore"
1475 if max_version is None:
1476 max_version = "Unknown"
1478 if max_package and not is_valid_package_name(max_package):
1479 raise FDroidException(_("Invalid package name {0}").format(max_package))
1481 return (max_version, max_vercode, max_package)
1484 def is_valid_package_name(name):
1485 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1488 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1489 raw=False, prepare=True, preponly=False, refresh=True,
1491 """Get the specified source library.
1493 Returns the path to it. Normally this is the path to be used when
1494 referencing it, which may be a subdirectory of the actual project. If
1495 you want the base directory of the project, pass 'basepath=True'.
1504 name, ref = spec.split('@')
1506 number, name = name.split(':', 1)
1508 name, subdir = name.split('/', 1)
1510 if name not in fdroidserver.metadata.srclibs:
1511 raise VCSException('srclib ' + name + ' not found.')
1513 srclib = fdroidserver.metadata.srclibs[name]
1515 sdir = os.path.join(srclib_dir, name)
1518 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1519 vcs.srclib = (name, number, sdir)
1521 vcs.gotorevision(ref, refresh)
1528 libdir = os.path.join(sdir, subdir)
1529 elif srclib["Subdir"]:
1530 for subdir in srclib["Subdir"]:
1531 libdir_candidate = os.path.join(sdir, subdir)
1532 if os.path.exists(libdir_candidate):
1533 libdir = libdir_candidate
1539 remove_signing_keys(sdir)
1540 remove_debuggable_flags(sdir)
1544 if srclib["Prepare"]:
1545 cmd = replace_config_vars(srclib["Prepare"], build)
1547 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=libdir)
1548 if p.returncode != 0:
1549 raise BuildException("Error running prepare command for srclib %s"
1555 return (name, number, libdir)
1558 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1561 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1562 """ Prepare the source code for a particular build
1564 :param vcs: the appropriate vcs object for the application
1565 :param app: the application details from the metadata
1566 :param build: the build details from the metadata
1567 :param build_dir: the path to the build directory, usually 'build/app.id'
1568 :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
1569 :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
1571 Returns the (root, srclibpaths) where:
1572 :param root: is the root directory, which may be the same as 'build_dir' or may
1573 be a subdirectory of it.
1574 :param srclibpaths: is information on the srclibs being used
1577 # Optionally, the actual app source can be in a subdirectory
1579 root_dir = os.path.join(build_dir, build.subdir)
1581 root_dir = build_dir
1583 # Get a working copy of the right revision
1584 logging.info("Getting source for revision " + build.commit)
1585 vcs.gotorevision(build.commit, refresh)
1587 # Initialise submodules if required
1588 if build.submodules:
1589 logging.info(_("Initialising submodules"))
1590 vcs.initsubmodules()
1592 # Check that a subdir (if we're using one) exists. This has to happen
1593 # after the checkout, since it might not exist elsewhere
1594 if not os.path.exists(root_dir):
1595 raise BuildException('Missing subdir ' + root_dir)
1597 # Run an init command if one is required
1599 cmd = replace_config_vars(build.init, build)
1600 logging.info("Running 'init' commands in %s" % root_dir)
1602 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir)
1603 if p.returncode != 0:
1604 raise BuildException("Error running init command for %s:%s" %
1605 (app.id, build.versionName), p.output)
1607 # Apply patches if any
1609 logging.info("Applying patches")
1610 for patch in build.patch:
1611 patch = patch.strip()
1612 logging.info("Applying " + patch)
1613 patch_path = os.path.join('metadata', app.id, patch)
1614 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1615 if p.returncode != 0:
1616 raise BuildException("Failed to apply patch %s" % patch_path)
1618 # Get required source libraries
1621 logging.info("Collecting source libraries")
1622 for lib in build.srclibs:
1623 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1624 refresh=refresh, build=build))
1626 for name, number, libpath in srclibpaths:
1627 place_srclib(root_dir, int(number) if number else None, libpath)
1629 basesrclib = vcs.getsrclib()
1630 # If one was used for the main source, add that too.
1632 srclibpaths.append(basesrclib)
1634 # Update the local.properties file
1635 localprops = [os.path.join(build_dir, 'local.properties')]
1637 parts = build.subdir.split(os.sep)
1640 cur = os.path.join(cur, d)
1641 localprops += [os.path.join(cur, 'local.properties')]
1642 for path in localprops:
1644 if os.path.isfile(path):
1645 logging.info("Updating local.properties file at %s" % path)
1646 with open(path, 'r', encoding='iso-8859-1') as f:
1650 logging.info("Creating local.properties file at %s" % path)
1651 # Fix old-fashioned 'sdk-location' by copying
1652 # from sdk.dir, if necessary
1654 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1655 re.S | re.M).group(1)
1656 props += "sdk-location=%s\n" % sdkloc
1658 props += "sdk.dir=%s\n" % config['sdk_path']
1659 props += "sdk-location=%s\n" % config['sdk_path']
1660 ndk_path = build.ndk_path()
1661 # if for any reason the path isn't valid or the directory
1662 # doesn't exist, some versions of Gradle will error with a
1663 # cryptic message (even if the NDK is not even necessary).
1664 # https://gitlab.com/fdroid/fdroidserver/issues/171
1665 if ndk_path and os.path.exists(ndk_path):
1667 props += "ndk.dir=%s\n" % ndk_path
1668 props += "ndk-location=%s\n" % ndk_path
1669 # Add java.encoding if necessary
1671 props += "java.encoding=%s\n" % build.encoding
1672 with open(path, 'w', encoding='iso-8859-1') as f:
1676 if build.build_method() == 'gradle':
1677 flavours = build.gradle
1680 n = build.target.split('-')[1]
1681 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1682 r'compileSdkVersion %s' % n,
1683 os.path.join(root_dir, 'build.gradle'))
1685 # Remove forced debuggable flags
1686 remove_debuggable_flags(root_dir)
1688 # Insert version code and number into the manifest if necessary
1689 if build.forceversion:
1690 logging.info("Changing the version name")
1691 for path in manifest_paths(root_dir, flavours):
1692 if not os.path.isfile(path):
1694 if has_extension(path, 'xml'):
1695 regsub_file(r'android:versionName="[^"]*"',
1696 r'android:versionName="%s"' % build.versionName,
1698 elif has_extension(path, 'gradle'):
1699 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1700 r"""\1versionName '%s'""" % build.versionName,
1703 if build.forcevercode:
1704 logging.info("Changing the version code")
1705 for path in manifest_paths(root_dir, flavours):
1706 if not os.path.isfile(path):
1708 if has_extension(path, 'xml'):
1709 regsub_file(r'android:versionCode="[^"]*"',
1710 r'android:versionCode="%s"' % build.versionCode,
1712 elif has_extension(path, 'gradle'):
1713 regsub_file(r'versionCode[ =]+[0-9]+',
1714 r'versionCode %s' % build.versionCode,
1717 # Delete unwanted files
1719 logging.info(_("Removing specified files"))
1720 for part in getpaths(build_dir, build.rm):
1721 dest = os.path.join(build_dir, part)
1722 logging.info("Removing {0}".format(part))
1723 if os.path.lexists(dest):
1724 # rmtree can only handle directories that are not symlinks, so catch anything else
1725 if not os.path.isdir(dest) or os.path.islink(dest):
1730 logging.info("...but it didn't exist")
1732 remove_signing_keys(build_dir)
1734 # Add required external libraries
1736 logging.info("Collecting prebuilt libraries")
1737 libsdir = os.path.join(root_dir, 'libs')
1738 if not os.path.exists(libsdir):
1740 for lib in build.extlibs:
1742 logging.info("...installing extlib {0}".format(lib))
1743 libf = os.path.basename(lib)
1744 libsrc = os.path.join(extlib_dir, lib)
1745 if not os.path.exists(libsrc):
1746 raise BuildException("Missing extlib file {0}".format(libsrc))
1747 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1749 # Run a pre-build command if one is required
1751 logging.info("Running 'prebuild' commands in %s" % root_dir)
1753 cmd = replace_config_vars(build.prebuild, build)
1755 # Substitute source library paths into prebuild commands
1756 for name, number, libpath in srclibpaths:
1757 libpath = os.path.relpath(libpath, root_dir)
1758 cmd = cmd.replace('$$' + name + '$$', libpath)
1760 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir)
1761 if p.returncode != 0:
1762 raise BuildException("Error running prebuild command for %s:%s" %
1763 (app.id, build.versionName), p.output)
1765 # Generate (or update) the ant build file, build.xml...
1766 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1767 parms = ['android', 'update', 'lib-project']
1768 lparms = ['android', 'update', 'project']
1771 parms += ['-t', build.target]
1772 lparms += ['-t', build.target]
1773 if build.androidupdate:
1774 update_dirs = build.androidupdate
1776 update_dirs = ant_subprojects(root_dir) + ['.']
1778 for d in update_dirs:
1779 subdir = os.path.join(root_dir, d)
1781 logging.debug("Updating main project")
1782 cmd = parms + ['-p', d]
1784 logging.debug("Updating subproject %s" % d)
1785 cmd = lparms + ['-p', d]
1786 p = SdkToolsPopen(cmd, cwd=root_dir)
1787 # Check to see whether an error was returned without a proper exit
1788 # code (this is the case for the 'no target set or target invalid'
1790 if p.returncode != 0 or p.output.startswith("Error: "):
1791 raise BuildException("Failed to update project at %s" % d, p.output)
1792 # Clean update dirs via ant
1794 logging.info("Cleaning subproject %s" % d)
1795 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1797 return (root_dir, srclibpaths)
1800 def getpaths_map(build_dir, globpaths):
1801 """Extend via globbing the paths from a field and return them as a map from original path to resulting paths"""
1805 full_path = os.path.join(build_dir, p)
1806 full_path = os.path.normpath(full_path)
1807 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1809 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1813 def getpaths(build_dir, globpaths):
1814 """Extend via globbing the paths from a field and return them as a set"""
1815 paths_map = getpaths_map(build_dir, globpaths)
1817 for k, v in paths_map.items():
1824 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1827 def check_system_clock(dt_obj, path):
1828 """Check if system clock is updated based on provided date
1830 If an APK has files newer than the system time, suggest updating
1831 the system clock. This is useful for offline systems, used for
1832 signing, which do not have another source of clock sync info. It
1833 has to be more than 24 hours newer because ZIP/APK files do not
1837 checkdt = dt_obj - timedelta(1)
1838 if datetime.today() < checkdt:
1839 logging.warning(_('System clock is older than date in {path}!').format(path=path)
1840 + '\n' + _('Set clock to that time using:') + '\n'
1841 + 'sudo date -s "' + str(dt_obj) + '"')
1845 """permanent store of existing APKs with the date they were added
1847 This is currently the only way to permanently store the "updated"
1852 '''Load filename/date info about previously seen APKs
1854 Since the appid and date strings both will never have spaces,
1855 this is parsed as a list from the end to allow the filename to
1856 have any combo of spaces.
1859 self.path = os.path.join('stats', 'known_apks.txt')
1861 if os.path.isfile(self.path):
1862 with open(self.path, 'r', encoding='utf8') as f:
1864 t = line.rstrip().split(' ')
1866 self.apks[t[0]] = (t[1], None)
1869 date = datetime.strptime(t[-1], '%Y-%m-%d')
1870 filename = line[0:line.rfind(appid) - 1]
1871 self.apks[filename] = (appid, date)
1872 check_system_clock(date, self.path)
1873 self.changed = False
1875 def writeifchanged(self):
1876 if not self.changed:
1879 if not os.path.exists('stats'):
1883 for apk, app in self.apks.items():
1885 line = apk + ' ' + appid
1887 line += ' ' + added.strftime('%Y-%m-%d')
1890 with open(self.path, 'w', encoding='utf8') as f:
1891 for line in sorted(lst, key=natural_key):
1892 f.write(line + '\n')
1894 def recordapk(self, apkName, app, default_date=None):
1896 Record an apk (if it's new, otherwise does nothing)
1897 Returns the date it was added as a datetime instance
1899 if apkName not in self.apks:
1900 if default_date is None:
1901 default_date = datetime.utcnow()
1902 self.apks[apkName] = (app, default_date)
1904 _ignored, added = self.apks[apkName]
1907 def getapp(self, apkname):
1908 """Look up information - given the 'apkname', returns (app id, date added/None).
1910 Or returns None for an unknown apk.
1912 if apkname in self.apks:
1913 return self.apks[apkname]
1916 def getlatest(self, num):
1917 """Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first"""
1919 for apk, app in self.apks.items():
1923 if apps[appid] > added:
1927 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1928 lst = [app for app, _ignored in sortedapps]
1933 def get_file_extension(filename):
1934 """get the normalized file extension, can be blank string but never None"""
1935 if isinstance(filename, bytes):
1936 filename = filename.decode('utf-8')
1937 return os.path.splitext(filename)[1].lower()[1:]
1940 def use_androguard():
1941 """Report if androguard is available, and config its debug logging"""
1945 if use_androguard.show_path:
1946 logging.debug(_('Using androguard from "{path}"').format(path=androguard.__file__))
1947 use_androguard.show_path = False
1948 if options and options.verbose:
1949 logging.getLogger("androguard.axml").setLevel(logging.INFO)
1955 use_androguard.show_path = True
1958 def is_apk_and_debuggable_aapt(apkfile):
1959 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1961 if p.returncode != 0:
1962 raise FDroidException(_("Failed to get APK manifest information"))
1963 for line in p.output.splitlines():
1964 if 'android:debuggable' in line and not line.endswith('0x0'):
1969 def is_apk_and_debuggable_androguard(apkfile):
1971 from androguard.core.bytecodes.apk import APK
1973 raise FDroidException("androguard library is not installed and aapt not present")
1975 apkobject = APK(apkfile)
1976 if apkobject.is_valid_APK():
1977 debuggable = apkobject.get_element("application", "debuggable")
1978 if debuggable is not None:
1979 return bool(strtobool(debuggable))
1983 def is_apk_and_debuggable(apkfile):
1984 """Returns True if the given file is an APK and is debuggable
1986 :param apkfile: full path to the apk to check"""
1988 if get_file_extension(apkfile) != 'apk':
1991 if use_androguard():
1992 return is_apk_and_debuggable_androguard(apkfile)
1994 return is_apk_and_debuggable_aapt(apkfile)
1997 def get_apk_id_aapt(apkfile):
1998 """Extrat identification information from APK using aapt.
2000 :param apkfile: path to an APK file.
2001 :returns: triplet (appid, version code, version name)
2003 r = re.compile("^package: name='(?P<appid>.*)' versionCode='(?P<vercode>.*)' versionName='(?P<vername>.*)'.*")
2004 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
2005 for line in p.output.splitlines():
2008 return m.group('appid'), m.group('vercode'), m.group('vername')
2009 raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'")
2010 .format(apkfilename=apkfile))
2013 def get_minSdkVersion_aapt(apkfile):
2014 """Extract the minimum supported Android SDK from an APK using aapt
2016 :param apkfile: path to an APK file.
2017 :returns: the integer representing the SDK version
2019 r = re.compile(r"^sdkVersion:'([0-9]+)'")
2020 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
2021 for line in p.output.splitlines():
2024 return int(m.group(1))
2025 raise FDroidException(_('Reading minSdkVersion failed: "{apkfilename}"')
2026 .format(apkfilename=apkfile))
2031 self.returncode = None
2035 def SdkToolsPopen(commands, cwd=None, output=True):
2037 if cmd not in config:
2038 config[cmd] = find_sdk_tools_cmd(commands[0])
2039 abscmd = config[cmd]
2041 raise FDroidException(_("Could not find '{command}' on your system").format(command=cmd))
2043 test_aapt_version(config['aapt'])
2044 return FDroidPopen([abscmd] + commands[1:],
2045 cwd=cwd, output=output)
2048 def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2050 Run a command and capture the possibly huge output as bytes.
2052 :param commands: command and argument list like in subprocess.Popen
2053 :param cwd: optionally specifies a working directory
2054 :param envs: a optional dictionary of environment variables and their values
2055 :returns: A PopenResult.
2060 set_FDroidPopen_env()
2062 process_env = env.copy()
2063 if envs is not None and len(envs) > 0:
2064 process_env.update(envs)
2067 cwd = os.path.normpath(cwd)
2068 logging.debug("Directory: %s" % cwd)
2069 logging.debug("> %s" % ' '.join(commands))
2071 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
2072 result = PopenResult()
2075 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
2076 stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
2077 stderr=stderr_param)
2078 except OSError as e:
2079 raise BuildException("OSError while trying to execute " +
2080 ' '.join(commands) + ': ' + str(e))
2082 # TODO are these AsynchronousFileReader threads always exiting?
2083 if not stderr_to_stdout and options.verbose:
2084 stderr_queue = Queue()
2085 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
2087 while not stderr_reader.eof():
2088 while not stderr_queue.empty():
2089 line = stderr_queue.get()
2090 sys.stderr.buffer.write(line)
2095 stdout_queue = Queue()
2096 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
2099 # Check the queue for output (until there is no more to get)
2100 while not stdout_reader.eof():
2101 while not stdout_queue.empty():
2102 line = stdout_queue.get()
2103 if output and options.verbose:
2104 # Output directly to console
2105 sys.stderr.buffer.write(line)
2111 result.returncode = p.wait()
2112 result.output = buf.getvalue()
2114 # make sure all filestreams of the subprocess are closed
2115 for streamvar in ['stdin', 'stdout', 'stderr']:
2116 if hasattr(p, streamvar):
2117 stream = getattr(p, streamvar)
2123 def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2125 Run a command and capture the possibly huge output as a str.
2127 :param commands: command and argument list like in subprocess.Popen
2128 :param cwd: optionally specifies a working directory
2129 :param envs: a optional dictionary of environment variables and their values
2130 :returns: A PopenResult.
2132 result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
2133 result.output = result.output.decode('utf-8', 'ignore')
2137 gradle_comment = re.compile(r'[ ]*//')
2138 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
2139 gradle_line_matches = [
2140 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
2141 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
2142 re.compile(r'.*\.readLine\(.*'),
2146 def remove_signing_keys(build_dir):
2147 for root, dirs, files in os.walk(build_dir):
2148 if 'build.gradle' in files:
2149 path = os.path.join(root, 'build.gradle')
2151 with open(path, "r", encoding='utf8') as o:
2152 lines = o.readlines()
2158 with open(path, "w", encoding='utf8') as o:
2159 while i < len(lines):
2162 while line.endswith('\\\n'):
2163 line = line.rstrip('\\\n') + lines[i]
2166 if gradle_comment.match(line):
2171 opened += line.count('{')
2172 opened -= line.count('}')
2175 if gradle_signing_configs.match(line):
2180 if any(s.match(line) for s in gradle_line_matches):
2188 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
2191 'project.properties',
2193 'default.properties',
2194 'ant.properties', ]:
2195 if propfile in files:
2196 path = os.path.join(root, propfile)
2198 with open(path, "r", encoding='iso-8859-1') as o:
2199 lines = o.readlines()
2203 with open(path, "w", encoding='iso-8859-1') as o:
2205 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
2212 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
2215 def set_FDroidPopen_env(build=None):
2217 set up the environment variables for the build environment
2219 There is only a weak standard, the variables used by gradle, so also set
2220 up the most commonly used environment variables for SDK and NDK. Also, if
2221 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
2223 global env, orig_path
2227 orig_path = env['PATH']
2228 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
2229 env[n] = config['sdk_path']
2230 for k, v in config['java_paths'].items():
2231 env['JAVA%s_HOME' % k] = v
2233 missinglocale = True
2234 for k, v in env.items():
2235 if k == 'LANG' and v != 'C':
2236 missinglocale = False
2238 missinglocale = False
2240 env['LANG'] = 'en_US.UTF-8'
2242 if build is not None:
2243 path = build.ndk_path()
2244 paths = orig_path.split(os.pathsep)
2245 if path not in paths:
2246 paths = [path] + paths
2247 env['PATH'] = os.pathsep.join(paths)
2248 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
2249 env[n] = build.ndk_path()
2252 def replace_build_vars(cmd, build):
2253 cmd = cmd.replace('$$COMMIT$$', build.commit)
2254 cmd = cmd.replace('$$VERSION$$', build.versionName)
2255 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
2259 def replace_config_vars(cmd, build):
2260 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
2261 cmd = cmd.replace('$$NDK$$', build.ndk_path())
2262 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
2263 if build is not None:
2264 cmd = replace_build_vars(cmd, build)
2268 def place_srclib(root_dir, number, libpath):
2271 relpath = os.path.relpath(libpath, root_dir)
2272 proppath = os.path.join(root_dir, 'project.properties')
2275 if os.path.isfile(proppath):
2276 with open(proppath, "r", encoding='iso-8859-1') as o:
2277 lines = o.readlines()
2279 with open(proppath, "w", encoding='iso-8859-1') as o:
2282 if line.startswith('android.library.reference.%d=' % number):
2283 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2288 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2291 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z_\-]+\.(SF|RSA|DSA|EC)')
2294 def signer_fingerprint_short(sig):
2295 """Obtain shortened sha256 signing-key fingerprint for pkcs7 signature.
2297 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2298 for a given pkcs7 signature.
2300 :param sig: Contents of an APK signing certificate.
2301 :returns: shortened signing-key fingerprint.
2303 return signer_fingerprint(sig)[:7]
2306 def signer_fingerprint(sig):
2307 """Obtain sha256 signing-key fingerprint for pkcs7 signature.
2309 Extracts hexadecimal sha256 signing-key fingerprint string
2310 for a given pkcs7 signature.
2312 :param: Contents of an APK signature.
2313 :returns: shortened signature fingerprint.
2315 cert_encoded = get_certificate(sig)
2316 return hashlib.sha256(cert_encoded).hexdigest()
2319 def apk_signer_fingerprint(apk_path):
2320 """Obtain sha256 signing-key fingerprint for APK.
2322 Extracts hexadecimal sha256 signing-key fingerprint string
2325 :param apkpath: path to APK
2326 :returns: signature fingerprint
2329 with zipfile.ZipFile(apk_path, 'r') as apk:
2330 certs = [n for n in apk.namelist() if CERT_PATH_REGEX.match(n)]
2333 logging.error("Found no signing certificates on %s" % apk_path)
2336 logging.error("Found multiple signing certificates on %s" % apk_path)
2339 cert = apk.read(certs[0])
2340 return signer_fingerprint(cert)
2343 def apk_signer_fingerprint_short(apk_path):
2344 """Obtain shortened sha256 signing-key fingerprint for APK.
2346 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2347 for a given pkcs7 APK.
2349 :param apk_path: path to APK
2350 :returns: shortened signing-key fingerprint
2352 return apk_signer_fingerprint(apk_path)[:7]
2355 def metadata_get_sigdir(appid, vercode=None):
2356 """Get signature directory for app"""
2358 return os.path.join('metadata', appid, 'signatures', vercode)
2360 return os.path.join('metadata', appid, 'signatures')
2363 def metadata_find_developer_signature(appid, vercode=None):
2364 """Tires to find the developer signature for given appid.
2366 This picks the first signature file found in metadata an returns its
2369 :returns: sha256 signing key fingerprint of the developer signing key.
2370 None in case no signature can not be found."""
2372 # fetch list of dirs for all versions of signatures
2375 appversigdirs.append(metadata_get_sigdir(appid, vercode))
2377 appsigdir = metadata_get_sigdir(appid)
2378 if os.path.isdir(appsigdir):
2379 numre = re.compile('[0-9]+')
2380 for ver in os.listdir(appsigdir):
2381 if numre.match(ver):
2382 appversigdir = os.path.join(appsigdir, ver)
2383 appversigdirs.append(appversigdir)
2385 for sigdir in appversigdirs:
2386 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2387 glob.glob(os.path.join(sigdir, '*.EC')) + \
2388 glob.glob(os.path.join(sigdir, '*.RSA'))
2390 raise FDroidException('ambiguous signatures, please make sure there is only one signature in \'{}\'. (The signature has to be the App maintainers signature for version of the APK.)'.format(sigdir))
2392 with open(sig, 'rb') as f:
2393 return signer_fingerprint(f.read())
2397 def metadata_find_signing_files(appid, vercode):
2398 """Gets a list of singed manifests and signatures.
2400 :param appid: app id string
2401 :param vercode: app version code
2402 :returns: a list of triplets for each signing key with following paths:
2403 (signature_file, singed_file, manifest_file)
2406 sigdir = metadata_get_sigdir(appid, vercode)
2407 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2408 glob.glob(os.path.join(sigdir, '*.EC')) + \
2409 glob.glob(os.path.join(sigdir, '*.RSA'))
2410 extre = re.compile('(\.DSA|\.EC|\.RSA)$')
2412 sf = extre.sub('.SF', sig)
2413 if os.path.isfile(sf):
2414 mf = os.path.join(sigdir, 'MANIFEST.MF')
2415 if os.path.isfile(mf):
2416 ret.append((sig, sf, mf))
2420 def metadata_find_developer_signing_files(appid, vercode):
2421 """Get developer signature files for specified app from metadata.
2423 :returns: A triplet of paths for signing files from metadata:
2424 (signature_file, singed_file, manifest_file)
2426 allsigningfiles = metadata_find_signing_files(appid, vercode)
2427 if allsigningfiles and len(allsigningfiles) == 1:
2428 return allsigningfiles[0]
2433 def apk_strip_signatures(signed_apk, strip_manifest=False):
2434 """Removes signatures from APK.
2436 :param signed_apk: path to apk file.
2437 :param strip_manifest: when set to True also the manifest file will
2438 be removed from the APK.
2440 with tempfile.TemporaryDirectory() as tmpdir:
2441 tmp_apk = os.path.join(tmpdir, 'tmp.apk')
2442 shutil.move(signed_apk, tmp_apk)
2443 with ZipFile(tmp_apk, 'r') as in_apk:
2444 with ZipFile(signed_apk, 'w') as out_apk:
2445 for info in in_apk.infolist():
2446 if not apk_sigfile.match(info.filename):
2448 if info.filename != 'META-INF/MANIFEST.MF':
2449 buf = in_apk.read(info.filename)
2450 out_apk.writestr(info, buf)
2452 buf = in_apk.read(info.filename)
2453 out_apk.writestr(info, buf)
2456 def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest):
2457 """Implats a signature from metadata into an APK.
2459 Note: this changes there supplied APK in place. So copy it if you
2460 need the original to be preserved.
2462 :param apkpath: location of the apk
2464 # get list of available signature files in metadata
2465 with tempfile.TemporaryDirectory() as tmpdir:
2466 apkwithnewsig = os.path.join(tmpdir, 'newsig.apk')
2467 with ZipFile(apkpath, 'r') as in_apk:
2468 with ZipFile(apkwithnewsig, 'w') as out_apk:
2469 for sig_file in [signaturefile, signedfile, manifest]:
2470 with open(sig_file, 'rb') as fp:
2472 info = zipfile.ZipInfo('META-INF/' + os.path.basename(sig_file))
2473 info.compress_type = zipfile.ZIP_DEFLATED
2474 info.create_system = 0 # "Windows" aka "FAT", what Android SDK uses
2475 out_apk.writestr(info, buf)
2476 for info in in_apk.infolist():
2477 if not apk_sigfile.match(info.filename):
2478 if info.filename != 'META-INF/MANIFEST.MF':
2479 buf = in_apk.read(info.filename)
2480 out_apk.writestr(info, buf)
2482 p = SdkToolsPopen(['zipalign', '-v', '4', apkwithnewsig, apkpath])
2483 if p.returncode != 0:
2484 raise BuildException("Failed to align application")
2487 def apk_extract_signatures(apkpath, outdir, manifest=True):
2488 """Extracts a signature files from APK and puts them into target directory.
2490 :param apkpath: location of the apk
2491 :param outdir: folder where the extracted signature files will be stored
2492 :param manifest: (optionally) disable extracting manifest file
2494 with ZipFile(apkpath, 'r') as in_apk:
2495 for f in in_apk.infolist():
2496 if apk_sigfile.match(f.filename) or \
2497 (manifest and f.filename == 'META-INF/MANIFEST.MF'):
2498 newpath = os.path.join(outdir, os.path.basename(f.filename))
2499 with open(newpath, 'wb') as out_file:
2500 out_file.write(in_apk.read(f.filename))
2503 def sign_apk(unsigned_path, signed_path, keyalias):
2504 """Sign and zipalign an unsigned APK, then save to a new file, deleting the unsigned
2506 android-18 (4.3) finally added support for reasonable hash
2507 algorithms, like SHA-256, before then, the only options were MD5
2508 and SHA1 :-/ This aims to use SHA-256 when the APK does not target
2509 older Android versions, and is therefore safe to do so.
2511 https://issuetracker.google.com/issues/36956587
2512 https://android-review.googlesource.com/c/platform/libcore/+/44491
2516 if get_minSdkVersion_aapt(unsigned_path) < 18:
2517 signature_algorithm = ['-sigalg', 'SHA1withRSA', '-digestalg', 'SHA1']
2519 signature_algorithm = ['-sigalg', 'SHA256withRSA', '-digestalg', 'SHA-256']
2521 p = FDroidPopen([config['jarsigner'], '-keystore', config['keystore'],
2522 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2523 '-keypass:env', 'FDROID_KEY_PASS']
2524 + signature_algorithm + [unsigned_path, keyalias],
2526 'FDROID_KEY_STORE_PASS': config['keystorepass'],
2527 'FDROID_KEY_PASS': config['keypass'], })
2528 if p.returncode != 0:
2529 raise BuildException(_("Failed to sign application"), p.output)
2531 p = SdkToolsPopen(['zipalign', '-v', '4', unsigned_path, signed_path])
2532 if p.returncode != 0:
2533 raise BuildException(_("Failed to zipalign application"))
2534 os.remove(unsigned_path)
2537 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
2538 """Verify that two apks are the same
2540 One of the inputs is signed, the other is unsigned. The signature metadata
2541 is transferred from the signed to the unsigned apk, and then jarsigner is
2542 used to verify that the signature from the signed apk is also varlid for
2543 the unsigned one. If the APK given as unsigned actually does have a
2544 signature, it will be stripped out and ignored.
2546 There are two SHA1 git commit IDs that fdroidserver includes in the builds
2547 it makes: fdroidserverid and buildserverid. Originally, these were inserted
2548 into AndroidManifest.xml, but that makes the build not reproducible. So
2549 instead they are included as separate files in the APK's META-INF/ folder.
2550 If those files exist in the signed APK, they will be part of the signature
2551 and need to also be included in the unsigned APK for it to validate.
2553 :param signed_apk: Path to a signed apk file
2554 :param unsigned_apk: Path to an unsigned apk file expected to match it
2555 :param tmp_dir: Path to directory for temporary files
2556 :returns: None if the verification is successful, otherwise a string
2557 describing what went wrong.
2560 if not os.path.isfile(signed_apk):
2561 return 'can not verify: file does not exists: {}'.format(signed_apk)
2563 if not os.path.isfile(unsigned_apk):
2564 return 'can not verify: file does not exists: {}'.format(unsigned_apk)
2566 with ZipFile(signed_apk, 'r') as signed:
2567 meta_inf_files = ['META-INF/MANIFEST.MF']
2568 for f in signed.namelist():
2569 if apk_sigfile.match(f) \
2570 or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
2571 meta_inf_files.append(f)
2572 if len(meta_inf_files) < 3:
2573 return "Signature files missing from {0}".format(signed_apk)
2575 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2576 with ZipFile(unsigned_apk, 'r') as unsigned:
2577 # only read the signature from the signed APK, everything else from unsigned
2578 with ZipFile(tmp_apk, 'w') as tmp:
2579 for filename in meta_inf_files:
2580 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2581 for info in unsigned.infolist():
2582 if info.filename in meta_inf_files:
2583 logging.warning('Ignoring %s from %s',
2584 info.filename, unsigned_apk)
2586 if info.filename in tmp.namelist():
2587 return "duplicate filename found: " + info.filename
2588 tmp.writestr(info, unsigned.read(info.filename))
2590 verified = verify_apk_signature(tmp_apk)
2593 logging.info("...NOT verified - {0}".format(tmp_apk))
2594 return compare_apks(signed_apk, tmp_apk, tmp_dir,
2595 os.path.dirname(unsigned_apk))
2597 logging.info("...successfully verified")
2601 def verify_jar_signature(jar):
2602 """Verifies the signature of a given JAR file.
2604 jarsigner is very shitty: unsigned JARs pass as "verified"! So
2605 this has to turn on -strict then check for result 4, since this
2606 does not expect the signature to be from a CA-signed certificate.
2608 :raises: VerificationException() if the JAR's signature could not be verified
2612 error = _('JAR signature failed to verify: {path}').format(path=jar)
2614 output = subprocess.check_output([config['jarsigner'], '-strict', '-verify', jar],
2615 stderr=subprocess.STDOUT)
2616 raise VerificationException(error + '\n' + output.decode('utf-8'))
2617 except subprocess.CalledProcessError as e:
2618 if e.returncode == 4:
2619 logging.debug(_('JAR signature verified: {path}').format(path=jar))
2621 raise VerificationException(error + '\n' + e.output.decode('utf-8'))
2624 def verify_apk_signature(apk, min_sdk_version=None):
2625 """verify the signature on an APK
2627 Try to use apksigner whenever possible since jarsigner is very
2628 shitty: unsigned APKs pass as "verified"! Warning, this does
2629 not work on JARs with apksigner >= 0.7 (build-tools 26.0.1)
2631 :returns: boolean whether the APK was verified
2633 if set_command_in_config('apksigner'):
2634 args = [config['apksigner'], 'verify']
2636 args += ['--min-sdk-version=' + min_sdk_version]
2638 args += ['--verbose']
2640 output = subprocess.check_output(args + [apk])
2642 logging.debug(apk + ': ' + output.decode('utf-8'))
2644 except subprocess.CalledProcessError as e:
2645 logging.error('\n' + apk + ': ' + e.output.decode('utf-8'))
2647 if not config.get('jarsigner_warning_displayed'):
2648 config['jarsigner_warning_displayed'] = True
2649 logging.warning(_("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner"))
2651 verify_jar_signature(apk)
2653 except Exception as e:
2658 def verify_old_apk_signature(apk):
2659 """verify the signature on an archived APK, supporting deprecated algorithms
2661 F-Droid aims to keep every single binary that it ever published. Therefore,
2662 it needs to be able to verify APK signatures that include deprecated/removed
2663 algorithms. For example, jarsigner treats an MD5 signature as unsigned.
2665 jarsigner passes unsigned APKs as "verified"! So this has to turn
2666 on -strict then check for result 4.
2668 :returns: boolean whether the APK was verified
2671 _java_security = os.path.join(os.getcwd(), '.java.security')
2672 with open(_java_security, 'w') as fp:
2673 fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
2677 config['jarsigner'],
2678 '-J-Djava.security.properties=' + _java_security,
2679 '-strict', '-verify', apk
2681 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
2682 except subprocess.CalledProcessError as e:
2683 if e.returncode != 4:
2686 logging.debug(_('JAR signature verified: {path}').format(path=apk))
2689 logging.error(_('Old APK signature failed to verify: {path}').format(path=apk)
2690 + '\n' + output.decode('utf-8'))
2694 apk_badchars = re.compile('''[/ :;'"]''')
2697 def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
2700 Returns None if the apk content is the same (apart from the signing key),
2701 otherwise a string describing what's different, or what went wrong when
2702 trying to do the comparison.
2708 absapk1 = os.path.abspath(apk1)
2709 absapk2 = os.path.abspath(apk2)
2711 if set_command_in_config('diffoscope'):
2712 logfilename = os.path.join(log_dir, os.path.basename(absapk1))
2713 htmlfile = logfilename + '.diffoscope.html'
2714 textfile = logfilename + '.diffoscope.txt'
2715 if subprocess.call([config['diffoscope'],
2716 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2717 '--html', htmlfile, '--text', textfile,
2718 absapk1, absapk2]) != 0:
2719 return("Failed to unpack " + apk1)
2721 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2722 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2723 for d in [apk1dir, apk2dir]:
2724 if os.path.exists(d):
2727 os.mkdir(os.path.join(d, 'jar-xf'))
2729 if subprocess.call(['jar', 'xf',
2730 os.path.abspath(apk1)],
2731 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2732 return("Failed to unpack " + apk1)
2733 if subprocess.call(['jar', 'xf',
2734 os.path.abspath(apk2)],
2735 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2736 return("Failed to unpack " + apk2)
2738 if set_command_in_config('apktool'):
2739 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2741 return("Failed to unpack " + apk1)
2742 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2744 return("Failed to unpack " + apk2)
2746 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2747 lines = p.output.splitlines()
2748 if len(lines) != 1 or 'META-INF' not in lines[0]:
2749 if set_command_in_config('meld'):
2750 p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
2751 return("Unexpected diff output - " + p.output)
2753 # since everything verifies, delete the comparison to keep cruft down
2754 shutil.rmtree(apk1dir)
2755 shutil.rmtree(apk2dir)
2757 # If we get here, it seems like they're the same!
2761 def set_command_in_config(command):
2762 '''Try to find specified command in the path, if it hasn't been
2763 manually set in config.py. If found, it is added to the config
2764 dict. The return value says whether the command is available.
2767 if command in config:
2770 tmp = find_command(command)
2772 config[command] = tmp
2777 def find_command(command):
2778 '''find the full path of a command, or None if it can't be found in the PATH'''
2781 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2783 fpath, fname = os.path.split(command)
2788 for path in os.environ["PATH"].split(os.pathsep):
2789 path = path.strip('"')
2790 exe_file = os.path.join(path, command)
2791 if is_exe(exe_file):
2798 '''generate a random password for when generating keys'''
2799 h = hashlib.sha256()
2800 h.update(os.urandom(16)) # salt
2801 h.update(socket.getfqdn().encode('utf-8'))
2802 passwd = base64.b64encode(h.digest()).strip()
2803 return passwd.decode('utf-8')
2806 def genkeystore(localconfig):
2808 Generate a new key with password provided in :param localconfig and add it to new keystore
2809 :return: hexed public key, public key fingerprint
2811 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2812 keystoredir = os.path.dirname(localconfig['keystore'])
2813 if keystoredir is None or keystoredir == '':
2814 keystoredir = os.path.join(os.getcwd(), keystoredir)
2815 if not os.path.exists(keystoredir):
2816 os.makedirs(keystoredir, mode=0o700)
2819 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
2820 'FDROID_KEY_PASS': localconfig['keypass'],
2822 p = FDroidPopen([config['keytool'], '-genkey',
2823 '-keystore', localconfig['keystore'],
2824 '-alias', localconfig['repo_keyalias'],
2825 '-keyalg', 'RSA', '-keysize', '4096',
2826 '-sigalg', 'SHA256withRSA',
2827 '-validity', '10000',
2828 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2829 '-keypass:env', 'FDROID_KEY_PASS',
2830 '-dname', localconfig['keydname']], envs=env_vars)
2831 if p.returncode != 0:
2832 raise BuildException("Failed to generate key", p.output)
2833 os.chmod(localconfig['keystore'], 0o0600)
2834 if not options.quiet:
2835 # now show the lovely key that was just generated
2836 p = FDroidPopen([config['keytool'], '-list', '-v',
2837 '-keystore', localconfig['keystore'],
2838 '-alias', localconfig['repo_keyalias'],
2839 '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
2840 logging.info(p.output.strip() + '\n\n')
2841 # get the public key
2842 p = FDroidPopenBytes([config['keytool'], '-exportcert',
2843 '-keystore', localconfig['keystore'],
2844 '-alias', localconfig['repo_keyalias'],
2845 '-storepass:env', 'FDROID_KEY_STORE_PASS']
2846 + config['smartcardoptions'],
2847 envs=env_vars, output=False, stderr_to_stdout=False)
2848 if p.returncode != 0 or len(p.output) < 20:
2849 raise BuildException("Failed to get public key", p.output)
2851 fingerprint = get_cert_fingerprint(pubkey)
2852 return hexlify(pubkey), fingerprint
2855 def get_cert_fingerprint(pubkey):
2857 Generate a certificate fingerprint the same way keytool does it
2858 (but with slightly different formatting)
2860 digest = hashlib.sha256(pubkey).digest()
2861 ret = [' '.join("%02X" % b for b in bytearray(digest))]
2862 return " ".join(ret)
2865 def get_certificate(certificate_file):
2867 Extracts a certificate from the given file.
2868 :param certificate_file: file bytes (as string) representing the certificate
2869 :return: A binary representation of the certificate's public key, or None in case of error
2871 content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
2872 if content.getComponentByName('contentType') != rfc2315.signedData:
2874 content = decoder.decode(content.getComponentByName('content'),
2875 asn1Spec=rfc2315.SignedData())[0]
2877 certificates = content.getComponentByName('certificates')
2878 cert = certificates[0].getComponentByName('certificate')
2880 logging.error("Certificates not found.")
2882 return encoder.encode(cert)
2885 def load_stats_fdroid_signing_key_fingerprints():
2886 """Load list of signing-key fingerprints stored by fdroid publish from file.
2888 :returns: list of dictionanryies containing the singing-key fingerprints.
2890 jar_file = os.path.join('stats', 'publishsigkeys.jar')
2891 if not os.path.isfile(jar_file):
2893 cmd = [config['jarsigner'], '-strict', '-verify', jar_file]
2894 p = FDroidPopen(cmd, output=False)
2895 if p.returncode != 4:
2896 raise FDroidException("Signature validation of '{}' failed! "
2897 "Please run publish again to rebuild this file.".format(jar_file))
2899 jar_sigkey = apk_signer_fingerprint(jar_file)
2900 repo_key_sig = config.get('repo_key_sha256')
2902 if jar_sigkey != repo_key_sig:
2903 raise FDroidException("Signature key fingerprint of file '{}' does not match repo_key_sha256 in config.py (found fingerprint: '{}')".format(jar_file, jar_sigkey))
2905 logging.warning("repo_key_sha256 not in config.py, setting it to the signature key fingerprint of '{}'".format(jar_file))
2906 config['repo_key_sha256'] = jar_sigkey
2907 write_to_config(config, 'repo_key_sha256')
2909 with zipfile.ZipFile(jar_file, 'r') as f:
2910 return json.loads(str(f.read('publishsigkeys.json'), 'utf-8'))
2913 def write_to_config(thisconfig, key, value=None, config_file=None):
2914 '''write a key/value to the local config.py
2916 NOTE: only supports writing string variables.
2918 :param thisconfig: config dictionary
2919 :param key: variable name in config.py to be overwritten/added
2920 :param value: optional value to be written, instead of fetched
2921 from 'thisconfig' dictionary.
2924 origkey = key + '_orig'
2925 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2926 cfg = config_file if config_file else 'config.py'
2928 # load config file, create one if it doesn't exist
2929 if not os.path.exists(cfg):
2930 open(cfg, 'a').close()
2931 logging.info("Creating empty " + cfg)
2932 with open(cfg, 'r', encoding="utf-8") as f:
2933 lines = f.readlines()
2935 # make sure the file ends with a carraige return
2937 if not lines[-1].endswith('\n'):
2940 # regex for finding and replacing python string variable
2941 # definitions/initializations
2942 pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
2943 repl = key + ' = "' + value + '"'
2944 pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
2945 repl2 = key + " = '" + value + "'"
2947 # If we replaced this line once, we make sure won't be a
2948 # second instance of this line for this key in the document.
2951 with open(cfg, 'w', encoding="utf-8") as f:
2953 if pattern.match(line) or pattern2.match(line):
2955 line = pattern.sub(repl, line)
2956 line = pattern2.sub(repl2, line)
2967 def parse_xml(path):
2968 return XMLElementTree.parse(path).getroot()
2971 def string_is_integer(string):
2979 def local_rsync(options, fromdir, todir):
2980 '''Rsync method for local to local copying of things
2982 This is an rsync wrapper with all the settings for safe use within
2983 the various fdroidserver use cases. This uses stricter rsync
2984 checking on all files since people using offline mode are already
2985 prioritizing security above ease and speed.
2988 rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
2989 '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
2990 if not options.no_checksum:
2991 rsyncargs.append('--checksum')
2993 rsyncargs += ['--verbose']
2995 rsyncargs += ['--quiet']
2996 logging.debug(' '.join(rsyncargs + [fromdir, todir]))
2997 if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
2998 raise FDroidException()
3001 def get_per_app_repos():
3002 '''per-app repos are dirs named with the packageName of a single app'''
3004 # Android packageNames are Java packages, they may contain uppercase or
3005 # lowercase letters ('A' through 'Z'), numbers, and underscores
3006 # ('_'). However, individual package name parts may only start with
3007 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
3008 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
3011 for root, dirs, files in os.walk(os.getcwd()):
3013 print('checking', root, 'for', d)
3014 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
3015 # standard parts of an fdroid repo, so never packageNames
3018 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
3024 def is_repo_file(filename):
3025 '''Whether the file in a repo is a build product to be delivered to users'''
3026 if isinstance(filename, str):
3027 filename = filename.encode('utf-8', errors="surrogateescape")
3028 return os.path.isfile(filename) \
3029 and not filename.endswith(b'.asc') \
3030 and not filename.endswith(b'.sig') \
3031 and os.path.basename(filename) not in [
3033 b'index_unsigned.jar',
3042 def get_examples_dir():
3043 '''Return the dir where the fdroidserver example files are available'''
3045 tmp = os.path.dirname(sys.argv[0])
3046 if os.path.basename(tmp) == 'bin':
3047 egg_links = glob.glob(os.path.join(tmp, '..',
3048 'local/lib/python3.*/site-packages/fdroidserver.egg-link'))
3050 # installed from local git repo
3051 examplesdir = os.path.join(open(egg_links[0]).readline().rstrip(), 'examples')
3054 examplesdir = os.path.dirname(os.path.dirname(__file__)) + '/share/doc/fdroidserver/examples'
3055 if not os.path.exists(examplesdir): # use UNIX layout
3056 examplesdir = os.path.dirname(tmp) + '/share/doc/fdroidserver/examples'
3058 # we're running straight out of the git repo
3059 prefix = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
3060 examplesdir = prefix + '/examples'
3065 def get_wiki_timestamp(timestamp=None):
3066 """Return current time in the standard format for posting to the wiki"""
3068 if timestamp is None:
3069 timestamp = time.gmtime()
3070 return time.strftime("%Y-%m-%d %H:%M:%SZ", timestamp)
3073 def get_android_tools_versions(ndk_path=None):
3074 '''get a list of the versions of all installed Android SDK/NDK components'''
3077 sdk_path = config['sdk_path']
3078 if sdk_path[-1] != '/':
3082 ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT')
3083 if os.path.isfile(ndk_release_txt):
3084 with open(ndk_release_txt, 'r') as fp:
3085 components.append((os.path.basename(ndk_path), fp.read()[:-1]))
3087 pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE)
3088 for root, dirs, files in os.walk(sdk_path):
3089 if 'source.properties' in files:
3090 source_properties = os.path.join(root, 'source.properties')
3091 with open(source_properties, 'r') as fp:
3092 m = pattern.search(fp.read())
3094 components.append((root[len(sdk_path):], m.group(1)))
3099 def get_android_tools_version_log(ndk_path=None):
3100 '''get a list of the versions of all installed Android SDK/NDK components'''
3101 log = '== Installed Android Tools ==\n\n'
3102 components = get_android_tools_versions(ndk_path)
3103 for name, version in sorted(components):
3104 log += '* ' + name + ' (' + version + ')\n'
3109 def get_git_describe_link():
3110 """Get a link to the current fdroiddata commit, to post to the wiki
3114 output = subprocess.check_output(['git', 'describe', '--always', '--dirty', '--abbrev=0'],
3115 universal_newlines=True).strip()
3116 except subprocess.CalledProcessError:
3119 commit = output.replace('-dirty', '')
3120 return ('* fdroiddata: [https://gitlab.com/fdroid/fdroiddata/commit/{commit} {id}]\n'
3121 .format(commit=commit, id=output))
3123 logging.error(_("'{path}' failed to execute!").format(path='git describe'))