3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
40 import xml.etree.ElementTree as XMLElementTree
42 from binascii import hexlify
43 from datetime import datetime, timedelta
44 from distutils.version import LooseVersion
45 from queue import Queue
46 from zipfile import ZipFile
48 from pyasn1.codec.der import decoder, encoder
49 from pyasn1_modules import rfc2315
50 from pyasn1.error import PyAsn1Error
52 from distutils.util import strtobool
54 import fdroidserver.metadata
55 from fdroidserver import _
56 from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException,\
57 BuildException, VerificationException
58 from .asynchronousfilereader import AsynchronousFileReader
60 # this is the build-tools version, aapt has a separate version that
61 # has to be manually set in test_aapt_version()
62 MINIMUM_AAPT_VERSION = '26.0.0'
64 VERCODE_OPERATION_RE = re.compile(r'^([ 0-9/*+-]|%c)+$')
66 # A signature block file with a .DSA, .RSA, or .EC extension
67 CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
68 APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
69 STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
71 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
80 'sdk_path': "$ANDROID_HOME",
84 'r12b': "$ANDROID_NDK",
90 'build_tools': MINIMUM_AAPT_VERSION,
91 'force_build_tools': False,
96 'accepted_formats': ['txt', 'yml'],
97 'sync_from_local_copy_dir': False,
98 'allow_disabled_algorithms': False,
99 'per_app_repos': False,
100 'make_current_version_link': True,
101 'current_version_name_source': 'Name',
102 'update_stats': False,
104 'stats_server': None,
106 'stats_to_carbon': False,
108 'build_server_always': False,
109 'keystore': 'keystore.jks',
110 'smartcardoptions': [],
120 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
121 'repo_name': "My First FDroid Repo Demo",
122 'repo_icon': "fdroid-icon.png",
123 'repo_description': '''
124 This is a repository of apps to be used with FDroid. Applications in this
125 repository are either official binaries built by the original application
126 developers, or are binaries built from source by the admin of f-droid.org
127 using the tools on https://gitlab.com/u/fdroid.
133 def setup_global_opts(parser):
134 try: # the buildserver VM might not have PIL installed
135 from PIL import PngImagePlugin
136 logger = logging.getLogger(PngImagePlugin.__name__)
137 logger.setLevel(logging.INFO) # tame the "STREAM" debug messages
141 parser.add_argument("-v", "--verbose", action="store_true", default=False,
142 help=_("Spew out even more information than normal"))
143 parser.add_argument("-q", "--quiet", action="store_true", default=False,
144 help=_("Restrict output to warnings and errors"))
147 def _add_java_paths_to_config(pathlist, thisconfig):
148 def path_version_key(s):
150 for u in re.split('[^0-9]+', s):
152 versionlist.append(int(u))
157 for d in sorted(pathlist, key=path_version_key):
158 if os.path.islink(d):
160 j = os.path.basename(d)
161 # the last one found will be the canonical one, so order appropriately
163 r'^1\.([6-9])\.0\.jdk$', # OSX
164 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
165 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
166 r'^jdk([6-9])-openjdk$', # Arch
167 r'^java-([6-9])-openjdk$', # Arch
168 r'^java-([6-9])-jdk$', # Arch (oracle)
169 r'^java-1\.([6-9])\.0-.*$', # RedHat
170 r'^java-([6-9])-oracle$', # Debian WebUpd8
171 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
172 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
173 r'^oracle-jdk-bin-1\.([7-9]).*$', # Gentoo (oracle)
174 r'^icedtea-bin-([7-9]).*$', # Gentoo (openjdk)
176 m = re.match(regex, j)
179 for p in [d, os.path.join(d, 'Contents', 'Home')]:
180 if os.path.exists(os.path.join(p, 'bin', 'javac')):
181 thisconfig['java_paths'][m.group(1)] = p
184 def fill_config_defaults(thisconfig):
185 for k, v in default_config.items():
186 if k not in thisconfig:
189 # Expand paths (~users and $vars)
190 def expand_path(path):
194 path = os.path.expanduser(path)
195 path = os.path.expandvars(path)
200 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
205 thisconfig[k + '_orig'] = v
207 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
208 if thisconfig['java_paths'] is None:
209 thisconfig['java_paths'] = dict()
211 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
212 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
213 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
214 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
215 pathlist += glob.glob('/opt/oracle-jdk-*1.[7-9]*')
216 pathlist += glob.glob('/opt/icedtea-*[7-9]*')
217 if os.getenv('JAVA_HOME') is not None:
218 pathlist.append(os.getenv('JAVA_HOME'))
219 if os.getenv('PROGRAMFILES') is not None:
220 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
221 _add_java_paths_to_config(pathlist, thisconfig)
223 for java_version in ('7', '8', '9'):
224 if java_version not in thisconfig['java_paths']:
226 java_home = thisconfig['java_paths'][java_version]
227 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
228 if os.path.exists(jarsigner):
229 thisconfig['jarsigner'] = jarsigner
230 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
231 break # Java7 is preferred, so quit if found
233 for k in ['ndk_paths', 'java_paths']:
239 thisconfig[k][k2] = exp
240 thisconfig[k][k2 + '_orig'] = v
243 def regsub_file(pattern, repl, path):
244 with open(path, 'rb') as f:
246 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
247 with open(path, 'wb') as f:
251 def read_config(opts, config_file='config.py'):
252 """Read the repository config
254 The config is read from config_file, which is in the current
255 directory when any of the repo management commands are used. If
256 there is a local metadata file in the git repo, then config.py is
257 not required, just use defaults.
260 global config, options
262 if config is not None:
269 if os.path.isfile(config_file):
270 logging.debug(_("Reading '{config_file}'").format(config_file=config_file))
271 with io.open(config_file, "rb") as f:
272 code = compile(f.read(), config_file, 'exec')
273 exec(code, None, config)
275 logging.warning(_("No 'config.py' found, using defaults."))
277 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
279 if not type(config[k]) in (str, list, tuple):
281 _("'{field}' will be in random order! Use () or [] brackets if order is important!")
284 # smartcardoptions must be a list since its command line args for Popen
285 if 'smartcardoptions' in config:
286 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
287 elif 'keystore' in config and config['keystore'] == 'NONE':
288 # keystore='NONE' means use smartcard, these are required defaults
289 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
290 'SunPKCS11-OpenSC', '-providerClass',
291 'sun.security.pkcs11.SunPKCS11',
292 '-providerArg', 'opensc-fdroid.cfg']
294 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
295 st = os.stat(config_file)
296 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
297 logging.warning(_("unsafe permissions on '{config_file}' (should be 0600)!")
298 .format(config_file=config_file))
300 fill_config_defaults(config)
302 for k in ["repo_description", "archive_description"]:
304 config[k] = clean_description(config[k])
306 if 'serverwebroot' in config:
307 if isinstance(config['serverwebroot'], str):
308 roots = [config['serverwebroot']]
309 elif all(isinstance(item, str) for item in config['serverwebroot']):
310 roots = config['serverwebroot']
312 raise TypeError(_('only accepts strings, lists, and tuples'))
314 for rootstr in roots:
315 # since this is used with rsync, where trailing slashes have
316 # meaning, ensure there is always a trailing slash
317 if rootstr[-1] != '/':
319 rootlist.append(rootstr.replace('//', '/'))
320 config['serverwebroot'] = rootlist
322 if 'servergitmirrors' in config:
323 if isinstance(config['servergitmirrors'], str):
324 roots = [config['servergitmirrors']]
325 elif all(isinstance(item, str) for item in config['servergitmirrors']):
326 roots = config['servergitmirrors']
328 raise TypeError(_('only accepts strings, lists, and tuples'))
329 config['servergitmirrors'] = roots
334 def assert_config_keystore(config):
335 """Check weather keystore is configured correctly and raise exception if not."""
338 if 'repo_keyalias' not in config:
340 logging.critical(_("'repo_keyalias' not found in config.py!"))
341 if 'keystore' not in config:
343 logging.critical(_("'keystore' not found in config.py!"))
344 elif not os.path.exists(config['keystore']):
346 logging.critical("'" + config['keystore'] + "' does not exist!")
347 if 'keystorepass' not in config:
349 logging.critical(_("'keystorepass' not found in config.py!"))
350 if 'keypass' not in config:
352 logging.critical(_("'keypass' not found in config.py!"))
354 raise FDroidException("This command requires a signing key, " +
355 "you can create one using: fdroid update --create-key")
358 def find_sdk_tools_cmd(cmd):
359 '''find a working path to a tool from the Android SDK'''
362 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
363 # try to find a working path to this command, in all the recent possible paths
364 if 'build_tools' in config:
365 build_tools = os.path.join(config['sdk_path'], 'build-tools')
366 # if 'build_tools' was manually set and exists, check only that one
367 configed_build_tools = os.path.join(build_tools, config['build_tools'])
368 if os.path.exists(configed_build_tools):
369 tooldirs.append(configed_build_tools)
371 # no configed version, so hunt known paths for it
372 for f in sorted(os.listdir(build_tools), reverse=True):
373 if os.path.isdir(os.path.join(build_tools, f)):
374 tooldirs.append(os.path.join(build_tools, f))
375 tooldirs.append(build_tools)
376 sdk_tools = os.path.join(config['sdk_path'], 'tools')
377 if os.path.exists(sdk_tools):
378 tooldirs.append(sdk_tools)
379 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
380 if os.path.exists(sdk_platform_tools):
381 tooldirs.append(sdk_platform_tools)
382 tooldirs.append('/usr/bin')
384 path = os.path.join(d, cmd)
385 if os.path.isfile(path):
387 test_aapt_version(path)
389 # did not find the command, exit with error message
390 ensure_build_tools_exists(config)
393 def test_aapt_version(aapt):
394 '''Check whether the version of aapt is new enough'''
395 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
396 if output is None or output == '':
397 logging.error(_("'{path}' failed to execute!").format(path=aapt))
399 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
404 # the Debian package has the version string like "v0.2-23.0.2"
407 if LooseVersion(bugfix) < LooseVersion(MINIMUM_AAPT_VERSION):
409 elif LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.4062713'):
412 logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-{version} or newer!")
413 .format(aapt=aapt, version=MINIMUM_AAPT_VERSION))
415 logging.warning(_('Unknown version of aapt, might cause problems: ') + output)
418 def test_sdk_exists(thisconfig):
419 if 'sdk_path' not in thisconfig:
420 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
421 test_aapt_version(thisconfig['aapt'])
424 logging.error(_("'sdk_path' not set in 'config.py'!"))
426 if thisconfig['sdk_path'] == default_config['sdk_path']:
427 logging.error(_('No Android SDK found!'))
428 logging.error(_('You can use ANDROID_HOME to set the path to your SDK, i.e.:'))
429 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
431 if not os.path.exists(thisconfig['sdk_path']):
432 logging.critical(_("Android SDK path '{path}' does not exist!")
433 .format(path=thisconfig['sdk_path']))
435 if not os.path.isdir(thisconfig['sdk_path']):
436 logging.critical(_("Android SDK path '{path}' is not a directory!")
437 .format(path=thisconfig['sdk_path']))
439 for d in ['build-tools', 'platform-tools', 'tools']:
440 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
441 logging.critical(_("Android SDK '{path}' does not have '{dirname}' installed!")
442 .format(path=thisconfig['sdk_path'], dirname=d))
447 def ensure_build_tools_exists(thisconfig):
448 if not test_sdk_exists(thisconfig):
449 raise FDroidException(_("Android SDK not found!"))
450 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
451 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
452 if not os.path.isdir(versioned_build_tools):
453 raise FDroidException(
454 _("Android build-tools path '{path}' does not exist!")
455 .format(path=versioned_build_tools))
458 def get_local_metadata_files():
459 '''get any metadata files local to an app's source repo
461 This tries to ignore anything that does not count as app metdata,
462 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
465 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
468 def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False):
470 :param appids: arguments in the form of multiple appid:[vc] strings
471 :returns: a dictionary with the set of vercodes specified for each package
474 if not appid_versionCode_pairs:
477 for p in appid_versionCode_pairs:
478 if allow_vercodes and ':' in p:
479 package, vercode = p.split(':')
481 package, vercode = p, None
482 if package not in vercodes:
483 vercodes[package] = [vercode] if vercode else []
485 elif vercode and vercode not in vercodes[package]:
486 vercodes[package] += [vercode] if vercode else []
491 def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
492 """Build a list of App instances for processing
494 On top of what read_pkg_args does, this returns the whole app
495 metadata, but limiting the builds list to the builds matching the
496 appid_versionCode_pairs and vercodes specified. If no appid_versionCode_pairs are specified, then
497 all App and Build instances are returned.
501 vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes)
507 for appid, app in allapps.items():
508 if appid in vercodes:
511 if len(apps) != len(vercodes):
514 logging.critical(_("No such package: %s") % p)
515 raise FDroidException(_("Found invalid appids in arguments"))
517 raise FDroidException(_("No packages specified"))
520 for appid, app in apps.items():
524 app.builds = [b for b in app.builds if b.versionCode in vc]
525 if len(app.builds) != len(vercodes[appid]):
527 allvcs = [b.versionCode for b in app.builds]
528 for v in vercodes[appid]:
530 logging.critical(_("No such versionCode {versionCode} for app {appid}")
531 .format(versionCode=v, appid=appid))
534 raise FDroidException(_("Found invalid versionCodes for some apps"))
539 def get_extension(filename):
540 base, ext = os.path.splitext(filename)
543 return base, ext.lower()[1:]
546 def has_extension(filename, ext):
547 _ignored, f_ext = get_extension(filename)
551 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
554 def clean_description(description):
555 'Remove unneeded newlines and spaces from a block of description text'
557 # this is split up by paragraph to make removing the newlines easier
558 for paragraph in re.split(r'\n\n', description):
559 paragraph = re.sub('\r', '', paragraph)
560 paragraph = re.sub('\n', ' ', paragraph)
561 paragraph = re.sub(' {2,}', ' ', paragraph)
562 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
563 returnstring += paragraph + '\n\n'
564 return returnstring.rstrip('\n')
567 def publishednameinfo(filename):
568 filename = os.path.basename(filename)
569 m = publish_name_regex.match(filename)
571 result = (m.group(1), m.group(2))
572 except AttributeError:
573 raise FDroidException(_("Invalid name for published file: %s") % filename)
577 apk_release_filename = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)\.apk')
578 apk_release_filename_with_sigfp = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)_(?P<sigfp>[0-9a-f]{7})\.apk')
581 def apk_parse_release_filename(apkname):
582 """Parses the name of an APK file according the F-Droids APK naming
583 scheme and returns the tokens.
585 WARNING: Returned values don't necessarily represent the APKs actual
586 properties, the are just paresed from the file name.
588 :returns: A triplet containing (appid, versionCode, signer), where appid
589 should be the package name, versionCode should be the integer
590 represion of the APKs version and signer should be the first 7 hex
591 digists of the sha256 signing key fingerprint which was used to sign
594 m = apk_release_filename_with_sigfp.match(apkname)
596 return m.group('appid'), m.group('vercode'), m.group('sigfp')
597 m = apk_release_filename.match(apkname)
599 return m.group('appid'), m.group('vercode'), None
600 return None, None, None
603 def get_release_filename(app, build):
605 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
607 return "%s_%s.apk" % (app.id, build.versionCode)
610 def get_toolsversion_logname(app, build):
611 return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
614 def getsrcname(app, build):
615 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
627 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
630 def get_build_dir(app):
631 '''get the dir that this app will be built in'''
633 if app.RepoType == 'srclib':
634 return os.path.join('build', 'srclib', app.Repo)
636 return os.path.join('build', app.id)
640 '''checkout code from VCS and return instance of vcs and the build dir'''
641 build_dir = get_build_dir(app)
643 # Set up vcs interface and make sure we have the latest code...
644 logging.debug("Getting {0} vcs interface for {1}"
645 .format(app.RepoType, app.Repo))
646 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
650 vcs = getvcs(app.RepoType, remote, build_dir)
652 return vcs, build_dir
655 def getvcs(vcstype, remote, local):
657 return vcs_git(remote, local)
658 if vcstype == 'git-svn':
659 return vcs_gitsvn(remote, local)
661 return vcs_hg(remote, local)
663 return vcs_bzr(remote, local)
664 if vcstype == 'srclib':
665 if local != os.path.join('build', 'srclib', remote):
666 raise VCSException("Error: srclib paths are hard-coded!")
667 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
669 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
670 raise VCSException("Invalid vcs type " + vcstype)
673 def getsrclibvcs(name):
674 if name not in fdroidserver.metadata.srclibs:
675 raise VCSException("Missing srclib " + name)
676 return fdroidserver.metadata.srclibs[name]['Repo Type']
681 def __init__(self, remote, local):
683 # svn, git-svn and bzr may require auth
685 if self.repotype() in ('git-svn', 'bzr'):
687 if self.repotype == 'git-svn':
688 raise VCSException("Authentication is not supported for git-svn")
689 self.username, remote = remote.split('@')
690 if ':' not in self.username:
691 raise VCSException(_("Password required with username"))
692 self.username, self.password = self.username.split(':')
696 self.clone_failed = False
697 self.refreshed = False
703 def clientversion(self):
704 versionstr = FDroidPopen(self.clientversioncmd()).output
705 return versionstr[0:versionstr.find('\n')]
707 def clientversioncmd(self):
710 def gotorevision(self, rev, refresh=True):
711 """Take the local repository to a clean version of the given
712 revision, which is specificed in the VCS's native
713 format. Beforehand, the repository can be dirty, or even
714 non-existent. If the repository does already exist locally, it
715 will be updated from the origin, but only once in the lifetime
716 of the vcs object. None is acceptable for 'rev' if you know
717 you are cloning a clean copy of the repo - otherwise it must
718 specify a valid revision.
721 if self.clone_failed:
722 raise VCSException(_("Downloading the repository already failed once, not trying again."))
724 # The .fdroidvcs-id file for a repo tells us what VCS type
725 # and remote that directory was created from, allowing us to drop it
726 # automatically if either of those things changes.
727 fdpath = os.path.join(self.local, '..',
728 '.fdroidvcs-' + os.path.basename(self.local))
729 fdpath = os.path.normpath(fdpath)
730 cdata = self.repotype() + ' ' + self.remote
733 if os.path.exists(self.local):
734 if os.path.exists(fdpath):
735 with open(fdpath, 'r') as f:
736 fsdata = f.read().strip()
741 logging.info("Repository details for %s changed - deleting" % (
745 logging.info("Repository details for %s missing - deleting" % (
748 shutil.rmtree(self.local)
752 self.refreshed = True
755 self.gotorevisionx(rev)
756 except FDroidException as e:
759 # If necessary, write the .fdroidvcs file.
760 if writeback and not self.clone_failed:
761 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
762 with open(fdpath, 'w+') as f:
768 def gotorevisionx(self, rev): # pylint: disable=unused-argument
769 """Derived classes need to implement this.
771 It's called once basic checking has been performed.
773 raise VCSException("This VCS type doesn't define gotorevisionx")
775 # Initialise and update submodules
776 def initsubmodules(self):
777 raise VCSException('Submodules not supported for this vcs type')
779 # Get a list of all known tags
781 if not self._gettags:
782 raise VCSException('gettags not supported for this vcs type')
784 for tag in self._gettags():
785 if re.match('[-A-Za-z0-9_. /]+$', tag):
789 def latesttags(self):
790 """Get a list of all the known tags, sorted from newest to oldest"""
791 raise VCSException('latesttags not supported for this vcs type')
794 """Get current commit reference (hash, revision, etc)"""
795 raise VCSException('getref not supported for this vcs type')
798 """Returns the srclib (name, path) used in setting up the current revision, or None."""
807 def clientversioncmd(self):
808 return ['git', '--version']
810 def git(self, args, envs=dict(), cwd=None, output=True):
811 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
813 While fetch/pull/clone respect the command line option flags,
814 it seems that submodule commands do not. They do seem to
815 follow whatever is in env vars, if the version of git is new
816 enough. So we just throw the kitchen sink at it to see what
819 Also, because of CVE-2017-1000117, block all SSH URLs.
822 # supported in git >= 2.3
824 '-c', 'core.askpass=/bin/true',
825 '-c', 'core.sshCommand=/bin/false',
826 '-c', 'url.https://.insteadOf=ssh://',
828 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
829 git_config.append('-c')
830 git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':')
831 git_config.append('-c')
832 git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain)
833 git_config.append('-c')
834 git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain)
836 'GIT_TERMINAL_PROMPT': '0',
837 'GIT_ASKPASS': '/bin/true',
838 'SSH_ASKPASS': '/bin/true',
839 'GIT_SSH': '/bin/false', # for git < 2.3
841 return FDroidPopen(['git', ] + git_config + args,
842 envs=envs, cwd=cwd, output=output)
845 """If the local directory exists, but is somehow not a git repository,
846 git will traverse up the directory tree until it finds one
847 that is (i.e. fdroidserver) and then we'll proceed to destroy
848 it! This is called as a safety check.
852 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
853 result = p.output.rstrip()
854 if not result.endswith(self.local):
855 raise VCSException('Repository mismatch')
857 def gotorevisionx(self, rev):
858 if not os.path.exists(self.local):
860 p = self.git(['clone', '--', self.remote, self.local])
861 if p.returncode != 0:
862 self.clone_failed = True
863 raise VCSException("Git clone failed", p.output)
867 # Discard any working tree changes
868 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
869 'git', 'reset', '--hard'], cwd=self.local, output=False)
870 if p.returncode != 0:
871 raise VCSException(_("Git reset failed"), p.output)
872 # Remove untracked files now, in case they're tracked in the target
873 # revision (it happens!)
874 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
875 'git', 'clean', '-dffx'], cwd=self.local, output=False)
876 if p.returncode != 0:
877 raise VCSException(_("Git clean failed"), p.output)
878 if not self.refreshed:
879 # Get latest commits and tags from remote
880 p = self.git(['fetch', 'origin'], cwd=self.local)
881 if p.returncode != 0:
882 raise VCSException(_("Git fetch failed"), p.output)
883 p = self.git(['fetch', '--prune', '--tags', 'origin'], output=False, cwd=self.local)
884 if p.returncode != 0:
885 raise VCSException(_("Git fetch failed"), p.output)
886 # Recreate origin/HEAD as git clone would do it, in case it disappeared
887 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
888 if p.returncode != 0:
889 lines = p.output.splitlines()
890 if 'Multiple remote HEAD branches' not in lines[0]:
891 raise VCSException(_("Git remote set-head failed"), p.output)
892 branch = lines[1].split(' ')[-1]
893 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--', branch],
894 cwd=self.local, output=False)
895 if p2.returncode != 0:
896 raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output)
897 self.refreshed = True
898 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
899 # a github repo. Most of the time this is the same as origin/master.
900 rev = rev or 'origin/HEAD'
901 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
902 if p.returncode != 0:
903 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
904 # Get rid of any uncontrolled files left behind
905 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
906 if p.returncode != 0:
907 raise VCSException(_("Git clean failed"), p.output)
909 def initsubmodules(self):
911 submfile = os.path.join(self.local, '.gitmodules')
912 if not os.path.isfile(submfile):
913 raise NoSubmodulesException(_("No git submodules available"))
915 # fix submodules not accessible without an account and public key auth
916 with open(submfile, 'r') as f:
917 lines = f.readlines()
918 with open(submfile, 'w') as f:
920 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
921 line = re.sub('git@' + domain + ':', 'https://u:p@' + domain + '/', line)
924 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
925 if p.returncode != 0:
926 raise VCSException(_("Git submodule sync failed"), p.output)
927 p = self.git(['submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
928 if p.returncode != 0:
929 raise VCSException(_("Git submodule update failed"), p.output)
933 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
934 return p.output.splitlines()
936 tag_format = re.compile(r'tag: ([^),]*)')
938 def latesttags(self):
940 p = FDroidPopen(['git', 'log', '--tags',
941 '--simplify-by-decoration', '--pretty=format:%d'],
942 cwd=self.local, output=False)
944 for line in p.output.splitlines():
945 for tag in self.tag_format.findall(line):
950 class vcs_gitsvn(vcs):
955 def clientversioncmd(self):
956 return ['git', 'svn', '--version']
959 """If the local directory exists, but is somehow not a git repository,
960 git will traverse up the directory tree until it finds one that
961 is (i.e. fdroidserver) and then we'll proceed to destory it!
962 This is called as a safety check.
965 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
966 result = p.output.rstrip()
967 if not result.endswith(self.local):
968 raise VCSException('Repository mismatch')
970 def git(self, args, envs=dict(), cwd=None, output=True):
971 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
973 AskPass is set to /bin/true to let the process try to connect
974 without a username/password.
976 The SSH command is set to /bin/false to block all SSH URLs
977 (supported in git >= 2.3). This protects against
982 '-c', 'core.askpass=/bin/true',
983 '-c', 'core.sshCommand=/bin/false',
986 'GIT_TERMINAL_PROMPT': '0',
987 'GIT_ASKPASS': '/bin/true',
988 'SSH_ASKPASS': '/bin/true',
989 'GIT_SSH': '/bin/false', # for git < 2.3
990 'SVN_SSH': '/bin/false',
992 return FDroidPopen(['git', ] + git_config + args,
993 envs=envs, cwd=cwd, output=output)
995 def gotorevisionx(self, rev):
996 if not os.path.exists(self.local):
998 gitsvn_args = ['svn', 'clone']
1000 if ';' in self.remote:
1001 remote_split = self.remote.split(';')
1002 for i in remote_split[1:]:
1003 if i.startswith('trunk='):
1004 gitsvn_args.extend(['-T', i[6:]])
1005 elif i.startswith('tags='):
1006 gitsvn_args.extend(['-t', i[5:]])
1007 elif i.startswith('branches='):
1008 gitsvn_args.extend(['-b', i[9:]])
1009 remote = remote_split[0]
1011 remote = self.remote
1013 if not remote.startswith('https://'):
1014 raise VCSException(_('HTTPS must be used with Subversion URLs!'))
1016 # git-svn sucks at certificate validation, this throws useful errors:
1018 r = requests.head(remote)
1019 r.raise_for_status()
1020 location = r.headers.get('location')
1021 if location and not location.startswith('https://'):
1022 raise VCSException(_('Invalid redirect to non-HTTPS: {before} -> {after} ')
1023 .format(before=remote, after=location))
1025 gitsvn_args.extend(['--', remote, self.local])
1026 p = self.git(gitsvn_args)
1027 if p.returncode != 0:
1028 self.clone_failed = True
1029 raise VCSException(_('git svn clone failed'), p.output)
1033 # Discard any working tree changes
1034 p = self.git(['reset', '--hard'], cwd=self.local, output=False)
1035 if p.returncode != 0:
1036 raise VCSException("Git reset failed", p.output)
1037 # Remove untracked files now, in case they're tracked in the target
1038 # revision (it happens!)
1039 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1040 if p.returncode != 0:
1041 raise VCSException("Git clean failed", p.output)
1042 if not self.refreshed:
1043 # Get new commits, branches and tags from repo
1044 p = self.git(['svn', 'fetch'], cwd=self.local, output=False)
1045 if p.returncode != 0:
1046 raise VCSException("Git svn fetch failed")
1047 p = self.git(['svn', 'rebase'], cwd=self.local, output=False)
1048 if p.returncode != 0:
1049 raise VCSException("Git svn rebase failed", p.output)
1050 self.refreshed = True
1052 rev = rev or 'master'
1054 nospaces_rev = rev.replace(' ', '%20')
1055 # Try finding a svn tag
1056 for treeish in ['origin/', '']:
1057 p = self.git(['checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
1058 if p.returncode == 0:
1060 if p.returncode != 0:
1061 # No tag found, normal svn rev translation
1062 # Translate svn rev into git format
1063 rev_split = rev.split('/')
1066 for treeish in ['origin/', '']:
1067 if len(rev_split) > 1:
1068 treeish += rev_split[0]
1069 svn_rev = rev_split[1]
1072 # if no branch is specified, then assume trunk (i.e. 'master' branch):
1076 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
1078 p = self.git(['svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
1079 git_rev = p.output.rstrip()
1081 if p.returncode == 0 and git_rev:
1084 if p.returncode != 0 or not git_rev:
1085 # Try a plain git checkout as a last resort
1086 p = self.git(['checkout', rev], cwd=self.local, output=False)
1087 if p.returncode != 0:
1088 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
1090 # Check out the git rev equivalent to the svn rev
1091 p = self.git(['checkout', git_rev], cwd=self.local, output=False)
1092 if p.returncode != 0:
1093 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
1095 # Get rid of any uncontrolled files left behind
1096 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1097 if p.returncode != 0:
1098 raise VCSException(_("Git clean failed"), p.output)
1102 for treeish in ['origin/', '']:
1103 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
1104 if os.path.isdir(d):
1105 return os.listdir(d)
1109 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
1110 if p.returncode != 0:
1112 return p.output.strip()
1120 def clientversioncmd(self):
1121 return ['hg', '--version']
1123 def gotorevisionx(self, rev):
1124 if not os.path.exists(self.local):
1125 p = FDroidPopen(['hg', 'clone', '--ssh', '/bin/false', '--', self.remote, self.local],
1127 if p.returncode != 0:
1128 self.clone_failed = True
1129 raise VCSException("Hg clone failed", p.output)
1131 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
1132 if p.returncode != 0:
1133 raise VCSException("Hg status failed", p.output)
1134 for line in p.output.splitlines():
1135 if not line.startswith('? '):
1136 raise VCSException("Unexpected output from hg status -uS: " + line)
1137 FDroidPopen(['rm', '-rf', '--', line[2:]], cwd=self.local, output=False)
1138 if not self.refreshed:
1139 p = FDroidPopen(['hg', 'pull', '--ssh', '/bin/false'], cwd=self.local, output=False)
1140 if p.returncode != 0:
1141 raise VCSException("Hg pull failed", p.output)
1142 self.refreshed = True
1144 rev = rev or 'default'
1147 p = FDroidPopen(['hg', 'update', '-C', '--', rev], cwd=self.local, output=False)
1148 if p.returncode != 0:
1149 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
1150 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1151 # Also delete untracked files, we have to enable purge extension for that:
1152 if "'purge' is provided by the following extension" in p.output:
1153 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
1154 myfile.write("\n[extensions]\nhgext.purge=\n")
1155 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1156 if p.returncode != 0:
1157 raise VCSException("HG purge failed", p.output)
1158 elif p.returncode != 0:
1159 raise VCSException("HG purge failed", p.output)
1162 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
1163 return p.output.splitlines()[1:]
1171 def clientversioncmd(self):
1172 return ['bzr', '--version']
1174 def bzr(self, args, envs=dict(), cwd=None, output=True):
1175 '''Prevent bzr from ever using SSH to avoid security vulns'''
1179 return FDroidPopen(['bzr', ] + args, envs=envs, cwd=cwd, output=output)
1181 def gotorevisionx(self, rev):
1182 if not os.path.exists(self.local):
1183 p = self.bzr(['branch', self.remote, self.local], output=False)
1184 if p.returncode != 0:
1185 self.clone_failed = True
1186 raise VCSException("Bzr branch failed", p.output)
1188 p = self.bzr(['clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
1189 if p.returncode != 0:
1190 raise VCSException("Bzr revert failed", p.output)
1191 if not self.refreshed:
1192 p = self.bzr(['pull'], cwd=self.local, output=False)
1193 if p.returncode != 0:
1194 raise VCSException("Bzr update failed", p.output)
1195 self.refreshed = True
1197 revargs = list(['-r', rev] if rev else [])
1198 p = self.bzr(['revert'] + revargs, cwd=self.local, output=False)
1199 if p.returncode != 0:
1200 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1203 p = self.bzr(['tags'], cwd=self.local, output=False)
1204 return [tag.split(' ')[0].strip() for tag in
1205 p.output.splitlines()]
1208 def unescape_string(string):
1211 if string[0] == '"' and string[-1] == '"':
1214 return string.replace("\\'", "'")
1217 def retrieve_string(app_dir, string, xmlfiles=None):
1219 if not string.startswith('@string/'):
1220 return unescape_string(string)
1222 if xmlfiles is None:
1225 os.path.join(app_dir, 'res'),
1226 os.path.join(app_dir, 'src', 'main', 'res'),
1228 for root, dirs, files in os.walk(res_dir):
1229 if os.path.basename(root) == 'values':
1230 xmlfiles += [os.path.join(root, x) for x in files if x.endswith('.xml')]
1232 name = string[len('@string/'):]
1234 def element_content(element):
1235 if element.text is None:
1237 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1238 return s.decode('utf-8').strip()
1240 for path in xmlfiles:
1241 if not os.path.isfile(path):
1243 xml = parse_xml(path)
1244 element = xml.find('string[@name="' + name + '"]')
1245 if element is not None:
1246 content = element_content(element)
1247 return retrieve_string(app_dir, content, xmlfiles)
1252 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1253 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1256 def manifest_paths(app_dir, flavours):
1257 '''Return list of existing files that will be used to find the highest vercode'''
1259 possible_manifests = \
1260 [os.path.join(app_dir, 'AndroidManifest.xml'),
1261 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1262 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1263 os.path.join(app_dir, 'build.gradle')]
1265 for flavour in flavours:
1266 if flavour == 'yes':
1268 possible_manifests.append(
1269 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1271 return [path for path in possible_manifests if os.path.isfile(path)]
1274 def fetch_real_name(app_dir, flavours):
1275 '''Retrieve the package name. Returns the name, or None if not found.'''
1276 for path in manifest_paths(app_dir, flavours):
1277 if not has_extension(path, 'xml') or not os.path.isfile(path):
1279 logging.debug("fetch_real_name: Checking manifest at " + path)
1280 xml = parse_xml(path)
1281 app = xml.find('application')
1284 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1286 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1287 result = retrieve_string_singleline(app_dir, label)
1289 result = result.strip()
1294 def get_library_references(root_dir):
1296 proppath = os.path.join(root_dir, 'project.properties')
1297 if not os.path.isfile(proppath):
1299 with open(proppath, 'r', encoding='iso-8859-1') as f:
1301 if not line.startswith('android.library.reference.'):
1303 path = line.split('=')[1].strip()
1304 relpath = os.path.join(root_dir, path)
1305 if not os.path.isdir(relpath):
1307 logging.debug("Found subproject at %s" % path)
1308 libraries.append(path)
1312 def ant_subprojects(root_dir):
1313 subprojects = get_library_references(root_dir)
1314 for subpath in subprojects:
1315 subrelpath = os.path.join(root_dir, subpath)
1316 for p in get_library_references(subrelpath):
1317 relp = os.path.normpath(os.path.join(subpath, p))
1318 if relp not in subprojects:
1319 subprojects.insert(0, relp)
1323 def remove_debuggable_flags(root_dir):
1324 # Remove forced debuggable flags
1325 logging.debug("Removing debuggable flags from %s" % root_dir)
1326 for root, dirs, files in os.walk(root_dir):
1327 if 'AndroidManifest.xml' in files and os.path.isfile(os.path.join(root, 'AndroidManifest.xml')):
1328 regsub_file(r'android:debuggable="[^"]*"',
1330 os.path.join(root, 'AndroidManifest.xml'))
1333 vcsearch_g = re.compile(r'''.*[Vv]ersionCode\s*=?\s*["']*([0-9]+)["']*''').search
1334 vnsearch_g = re.compile(r'''.*[Vv]ersionName\s*=?\s*(["'])((?:(?=(\\?))\3.)*?)\1.*''').search
1335 psearch_g = re.compile(r'''.*(packageName|applicationId)\s*=*\s*["']([^"']+)["'].*''').search
1338 def app_matches_packagename(app, package):
1341 appid = app.UpdateCheckName or app.id
1342 if appid is None or appid == "Ignore":
1344 return appid == package
1347 def parse_androidmanifests(paths, app):
1349 Extract some information from the AndroidManifest.xml at the given path.
1350 Returns (version, vercode, package), any or all of which might be None.
1351 All values returned are strings.
1354 ignoreversions = app.UpdateCheckIgnore
1355 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1358 return (None, None, None)
1366 if not os.path.isfile(path):
1369 logging.debug(_("Parsing manifest at '{path}'").format(path=path))
1375 if app.builds and 'gradle' in app.builds[-1] and app.builds[-1].gradle:
1376 flavour = app.builds[-1].gradle[-1]
1378 if has_extension(path, 'gradle'):
1379 with open(path, 'r') as f:
1380 inside_flavour_group = 0
1381 inside_required_flavour = 0
1383 if gradle_comment.match(line):
1386 if inside_flavour_group > 0:
1387 if inside_required_flavour > 0:
1388 matches = psearch_g(line)
1390 s = matches.group(2)
1391 if app_matches_packagename(app, s):
1394 matches = vnsearch_g(line)
1396 version = matches.group(2)
1398 matches = vcsearch_g(line)
1400 vercode = matches.group(1)
1403 inside_required_flavour += 1
1405 inside_required_flavour -= 1
1407 if flavour and (flavour in line):
1408 inside_required_flavour = 1
1411 inside_flavour_group += 1
1413 inside_flavour_group -= 1
1415 if "productFlavors" in line:
1416 inside_flavour_group = 1
1418 matches = psearch_g(line)
1420 s = matches.group(2)
1421 if app_matches_packagename(app, s):
1424 matches = vnsearch_g(line)
1426 version = matches.group(2)
1428 matches = vcsearch_g(line)
1430 vercode = matches.group(1)
1433 xml = parse_xml(path)
1434 if "package" in xml.attrib:
1435 s = xml.attrib["package"]
1436 if app_matches_packagename(app, s):
1438 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1439 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1440 base_dir = os.path.dirname(path)
1441 version = retrieve_string_singleline(base_dir, version)
1442 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1443 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1444 if string_is_integer(a):
1447 logging.warning(_("Problem with xml at '{path}'").format(path=path))
1449 # Remember package name, may be defined separately from version+vercode
1451 package = max_package
1453 logging.debug("..got package={0}, version={1}, vercode={2}"
1454 .format(package, version, vercode))
1456 # Always grab the package name and version name in case they are not
1457 # together with the highest version code
1458 if max_package is None and package is not None:
1459 max_package = package
1460 if max_version is None and version is not None:
1461 max_version = version
1463 if vercode is not None \
1464 and (max_vercode is None or vercode > max_vercode):
1465 if not ignoresearch or not ignoresearch(version):
1466 if version is not None:
1467 max_version = version
1468 if vercode is not None:
1469 max_vercode = vercode
1470 if package is not None:
1471 max_package = package
1473 max_version = "Ignore"
1475 if max_version is None:
1476 max_version = "Unknown"
1478 if max_package and not is_valid_package_name(max_package):
1479 raise FDroidException(_("Invalid package name {0}").format(max_package))
1481 return (max_version, max_vercode, max_package)
1484 def is_valid_package_name(name):
1485 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1488 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1489 raw=False, prepare=True, preponly=False, refresh=True,
1491 """Get the specified source library.
1493 Returns the path to it. Normally this is the path to be used when
1494 referencing it, which may be a subdirectory of the actual project. If
1495 you want the base directory of the project, pass 'basepath=True'.
1504 name, ref = spec.split('@')
1506 number, name = name.split(':', 1)
1508 name, subdir = name.split('/', 1)
1510 if name not in fdroidserver.metadata.srclibs:
1511 raise VCSException('srclib ' + name + ' not found.')
1513 srclib = fdroidserver.metadata.srclibs[name]
1515 sdir = os.path.join(srclib_dir, name)
1518 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1519 vcs.srclib = (name, number, sdir)
1521 vcs.gotorevision(ref, refresh)
1528 libdir = os.path.join(sdir, subdir)
1529 elif srclib["Subdir"]:
1530 for subdir in srclib["Subdir"]:
1531 libdir_candidate = os.path.join(sdir, subdir)
1532 if os.path.exists(libdir_candidate):
1533 libdir = libdir_candidate
1539 remove_signing_keys(sdir)
1540 remove_debuggable_flags(sdir)
1544 if srclib["Prepare"]:
1545 cmd = replace_config_vars(srclib["Prepare"], build)
1547 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=libdir)
1548 if p.returncode != 0:
1549 raise BuildException("Error running prepare command for srclib %s"
1555 return (name, number, libdir)
1558 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1561 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1562 """ Prepare the source code for a particular build
1564 :param vcs: the appropriate vcs object for the application
1565 :param app: the application details from the metadata
1566 :param build: the build details from the metadata
1567 :param build_dir: the path to the build directory, usually 'build/app.id'
1568 :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
1569 :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
1571 Returns the (root, srclibpaths) where:
1572 :param root: is the root directory, which may be the same as 'build_dir' or may
1573 be a subdirectory of it.
1574 :param srclibpaths: is information on the srclibs being used
1577 # Optionally, the actual app source can be in a subdirectory
1579 root_dir = os.path.join(build_dir, build.subdir)
1581 root_dir = build_dir
1583 # Get a working copy of the right revision
1584 logging.info("Getting source for revision " + build.commit)
1585 vcs.gotorevision(build.commit, refresh)
1587 # Initialise submodules if required
1588 if build.submodules:
1589 logging.info(_("Initialising submodules"))
1590 vcs.initsubmodules()
1592 # Check that a subdir (if we're using one) exists. This has to happen
1593 # after the checkout, since it might not exist elsewhere
1594 if not os.path.exists(root_dir):
1595 raise BuildException('Missing subdir ' + root_dir)
1597 # Run an init command if one is required
1599 cmd = replace_config_vars(build.init, build)
1600 logging.info("Running 'init' commands in %s" % root_dir)
1602 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir)
1603 if p.returncode != 0:
1604 raise BuildException("Error running init command for %s:%s" %
1605 (app.id, build.versionName), p.output)
1607 # Apply patches if any
1609 logging.info("Applying patches")
1610 for patch in build.patch:
1611 patch = patch.strip()
1612 logging.info("Applying " + patch)
1613 patch_path = os.path.join('metadata', app.id, patch)
1614 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1615 if p.returncode != 0:
1616 raise BuildException("Failed to apply patch %s" % patch_path)
1618 # Get required source libraries
1621 logging.info("Collecting source libraries")
1622 for lib in build.srclibs:
1623 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1624 refresh=refresh, build=build))
1626 for name, number, libpath in srclibpaths:
1627 place_srclib(root_dir, int(number) if number else None, libpath)
1629 basesrclib = vcs.getsrclib()
1630 # If one was used for the main source, add that too.
1632 srclibpaths.append(basesrclib)
1634 # Update the local.properties file
1635 localprops = [os.path.join(build_dir, 'local.properties')]
1637 parts = build.subdir.split(os.sep)
1640 cur = os.path.join(cur, d)
1641 localprops += [os.path.join(cur, 'local.properties')]
1642 for path in localprops:
1644 if os.path.isfile(path):
1645 logging.info("Updating local.properties file at %s" % path)
1646 with open(path, 'r', encoding='iso-8859-1') as f:
1650 logging.info("Creating local.properties file at %s" % path)
1651 # Fix old-fashioned 'sdk-location' by copying
1652 # from sdk.dir, if necessary
1654 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1655 re.S | re.M).group(1)
1656 props += "sdk-location=%s\n" % sdkloc
1658 props += "sdk.dir=%s\n" % config['sdk_path']
1659 props += "sdk-location=%s\n" % config['sdk_path']
1660 ndk_path = build.ndk_path()
1661 # if for any reason the path isn't valid or the directory
1662 # doesn't exist, some versions of Gradle will error with a
1663 # cryptic message (even if the NDK is not even necessary).
1664 # https://gitlab.com/fdroid/fdroidserver/issues/171
1665 if ndk_path and os.path.exists(ndk_path):
1667 props += "ndk.dir=%s\n" % ndk_path
1668 props += "ndk-location=%s\n" % ndk_path
1669 # Add java.encoding if necessary
1671 props += "java.encoding=%s\n" % build.encoding
1672 with open(path, 'w', encoding='iso-8859-1') as f:
1676 if build.build_method() == 'gradle':
1677 flavours = build.gradle
1680 n = build.target.split('-')[1]
1681 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1682 r'compileSdkVersion %s' % n,
1683 os.path.join(root_dir, 'build.gradle'))
1685 # Remove forced debuggable flags
1686 remove_debuggable_flags(root_dir)
1688 # Insert version code and number into the manifest if necessary
1689 if build.forceversion:
1690 logging.info("Changing the version name")
1691 for path in manifest_paths(root_dir, flavours):
1692 if not os.path.isfile(path):
1694 if has_extension(path, 'xml'):
1695 regsub_file(r'android:versionName="[^"]*"',
1696 r'android:versionName="%s"' % build.versionName,
1698 elif has_extension(path, 'gradle'):
1699 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1700 r"""\1versionName '%s'""" % build.versionName,
1703 if build.forcevercode:
1704 logging.info("Changing the version code")
1705 for path in manifest_paths(root_dir, flavours):
1706 if not os.path.isfile(path):
1708 if has_extension(path, 'xml'):
1709 regsub_file(r'android:versionCode="[^"]*"',
1710 r'android:versionCode="%s"' % build.versionCode,
1712 elif has_extension(path, 'gradle'):
1713 regsub_file(r'versionCode[ =]+[0-9]+',
1714 r'versionCode %s' % build.versionCode,
1717 # Delete unwanted files
1719 logging.info(_("Removing specified files"))
1720 for part in getpaths(build_dir, build.rm):
1721 dest = os.path.join(build_dir, part)
1722 logging.info("Removing {0}".format(part))
1723 if os.path.lexists(dest):
1724 # rmtree can only handle directories that are not symlinks, so catch anything else
1725 if not os.path.isdir(dest) or os.path.islink(dest):
1730 logging.info("...but it didn't exist")
1732 remove_signing_keys(build_dir)
1734 # Add required external libraries
1736 logging.info("Collecting prebuilt libraries")
1737 libsdir = os.path.join(root_dir, 'libs')
1738 if not os.path.exists(libsdir):
1740 for lib in build.extlibs:
1742 logging.info("...installing extlib {0}".format(lib))
1743 libf = os.path.basename(lib)
1744 libsrc = os.path.join(extlib_dir, lib)
1745 if not os.path.exists(libsrc):
1746 raise BuildException("Missing extlib file {0}".format(libsrc))
1747 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1749 # Run a pre-build command if one is required
1751 logging.info("Running 'prebuild' commands in %s" % root_dir)
1753 cmd = replace_config_vars(build.prebuild, build)
1755 # Substitute source library paths into prebuild commands
1756 for name, number, libpath in srclibpaths:
1757 libpath = os.path.relpath(libpath, root_dir)
1758 cmd = cmd.replace('$$' + name + '$$', libpath)
1760 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir)
1761 if p.returncode != 0:
1762 raise BuildException("Error running prebuild command for %s:%s" %
1763 (app.id, build.versionName), p.output)
1765 # Generate (or update) the ant build file, build.xml...
1766 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1767 parms = ['android', 'update', 'lib-project']
1768 lparms = ['android', 'update', 'project']
1771 parms += ['-t', build.target]
1772 lparms += ['-t', build.target]
1773 if build.androidupdate:
1774 update_dirs = build.androidupdate
1776 update_dirs = ant_subprojects(root_dir) + ['.']
1778 for d in update_dirs:
1779 subdir = os.path.join(root_dir, d)
1781 logging.debug("Updating main project")
1782 cmd = parms + ['-p', d]
1784 logging.debug("Updating subproject %s" % d)
1785 cmd = lparms + ['-p', d]
1786 p = SdkToolsPopen(cmd, cwd=root_dir)
1787 # Check to see whether an error was returned without a proper exit
1788 # code (this is the case for the 'no target set or target invalid'
1790 if p.returncode != 0 or p.output.startswith("Error: "):
1791 raise BuildException("Failed to update project at %s" % d, p.output)
1792 # Clean update dirs via ant
1794 logging.info("Cleaning subproject %s" % d)
1795 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1797 return (root_dir, srclibpaths)
1800 def getpaths_map(build_dir, globpaths):
1801 """Extend via globbing the paths from a field and return them as a map from original path to resulting paths"""
1805 full_path = os.path.join(build_dir, p)
1806 full_path = os.path.normpath(full_path)
1807 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1809 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1813 def getpaths(build_dir, globpaths):
1814 """Extend via globbing the paths from a field and return them as a set"""
1815 paths_map = getpaths_map(build_dir, globpaths)
1817 for k, v in paths_map.items():
1824 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1827 def check_system_clock(dt_obj, path):
1828 """Check if system clock is updated based on provided date
1830 If an APK has files newer than the system time, suggest updating
1831 the system clock. This is useful for offline systems, used for
1832 signing, which do not have another source of clock sync info. It
1833 has to be more than 24 hours newer because ZIP/APK files do not
1837 checkdt = dt_obj - timedelta(1)
1838 if datetime.today() < checkdt:
1839 logging.warning(_('System clock is older than date in {path}!').format(path=path)
1840 + '\n' + _('Set clock to that time using:') + '\n'
1841 + 'sudo date -s "' + str(dt_obj) + '"')
1845 """permanent store of existing APKs with the date they were added
1847 This is currently the only way to permanently store the "updated"
1852 '''Load filename/date info about previously seen APKs
1854 Since the appid and date strings both will never have spaces,
1855 this is parsed as a list from the end to allow the filename to
1856 have any combo of spaces.
1859 self.path = os.path.join('stats', 'known_apks.txt')
1861 if os.path.isfile(self.path):
1862 with open(self.path, 'r', encoding='utf8') as f:
1864 t = line.rstrip().split(' ')
1866 self.apks[t[0]] = (t[1], None)
1869 date = datetime.strptime(t[-1], '%Y-%m-%d')
1870 filename = line[0:line.rfind(appid) - 1]
1871 self.apks[filename] = (appid, date)
1872 check_system_clock(date, self.path)
1873 self.changed = False
1875 def writeifchanged(self):
1876 if not self.changed:
1879 if not os.path.exists('stats'):
1883 for apk, app in self.apks.items():
1885 line = apk + ' ' + appid
1887 line += ' ' + added.strftime('%Y-%m-%d')
1890 with open(self.path, 'w', encoding='utf8') as f:
1891 for line in sorted(lst, key=natural_key):
1892 f.write(line + '\n')
1894 def recordapk(self, apkName, app, default_date=None):
1896 Record an apk (if it's new, otherwise does nothing)
1897 Returns the date it was added as a datetime instance
1899 if apkName not in self.apks:
1900 if default_date is None:
1901 default_date = datetime.utcnow()
1902 self.apks[apkName] = (app, default_date)
1904 _ignored, added = self.apks[apkName]
1907 def getapp(self, apkname):
1908 """Look up information - given the 'apkname', returns (app id, date added/None).
1910 Or returns None for an unknown apk.
1912 if apkname in self.apks:
1913 return self.apks[apkname]
1916 def getlatest(self, num):
1917 """Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first"""
1919 for apk, app in self.apks.items():
1923 if apps[appid] > added:
1927 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1928 lst = [app for app, _ignored in sortedapps]
1933 def get_file_extension(filename):
1934 """get the normalized file extension, can be blank string but never None"""
1935 if isinstance(filename, bytes):
1936 filename = filename.decode('utf-8')
1937 return os.path.splitext(filename)[1].lower()[1:]
1940 def use_androguard():
1941 """Report if androguard is available, and config its debug logging"""
1945 if use_androguard.show_path:
1946 logging.debug(_('Using androguard from "{path}"').format(path=androguard.__file__))
1947 use_androguard.show_path = False
1948 if options and options.verbose:
1949 logging.getLogger("androguard.axml").setLevel(logging.INFO)
1955 use_androguard.show_path = True
1958 def _get_androguard_APK(apkfile):
1960 from androguard.core.bytecodes.apk import APK
1962 raise FDroidException("androguard library is not installed and aapt not present")
1967 def ensure_final_value(packageName, arsc, value):
1968 """Ensure incoming value is always the value, not the resid
1970 androguard will sometimes return the Android "resId" aka
1971 Resource ID instead of the actual value. This checks whether
1972 the value is actually a resId, then performs the Android
1973 Resource lookup as needed.
1979 try: # can be a literal value or a resId
1980 res_id = int('0x' + value[1:], 16)
1981 res_id = arsc.get_id(packageName, res_id)[1]
1982 returnValue = arsc.get_string(packageName, res_id)[1]
1988 def is_apk_and_debuggable_aapt(apkfile):
1989 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1991 if p.returncode != 0:
1992 raise FDroidException(_("Failed to get APK manifest information"))
1993 for line in p.output.splitlines():
1994 if 'android:debuggable' in line and not line.endswith('0x0'):
1999 def is_apk_and_debuggable_androguard(apkfile):
2000 apkobject = _get_androguard_APK(apkfile)
2001 if apkobject.is_valid_APK():
2002 debuggable = apkobject.get_element("application", "debuggable")
2003 if debuggable is not None:
2004 return bool(strtobool(debuggable))
2008 def is_apk_and_debuggable(apkfile):
2009 """Returns True if the given file is an APK and is debuggable
2011 :param apkfile: full path to the apk to check"""
2013 if get_file_extension(apkfile) != 'apk':
2016 if use_androguard():
2017 return is_apk_and_debuggable_androguard(apkfile)
2019 return is_apk_and_debuggable_aapt(apkfile)
2022 def get_apk_id(apkfile):
2023 """Extract identification information from APK using aapt.
2025 :param apkfile: path to an APK file.
2026 :returns: triplet (appid, version code, version name)
2028 if use_androguard():
2029 return get_apk_id_androguard(apkfile)
2031 return get_apk_id_aapt(apkfile)
2034 def get_apk_id_androguard(apkfile):
2035 if not os.path.exists(apkfile):
2036 raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'")
2037 .format(apkfilename=apkfile))
2038 a = _get_androguard_APK(apkfile)
2039 versionName = ensure_final_value(a.package, a.get_android_resources(), a.get_androidversion_name())
2041 versionName = '' # versionName is expected to always be a str
2042 return a.package, a.get_androidversion_code(), versionName
2045 def get_apk_id_aapt(apkfile):
2046 r = re.compile("^package: name='(?P<appid>.*)' versionCode='(?P<vercode>.*)' versionName='(?P<vername>.*?)'(?: platformBuildVersionName='.*')?")
2047 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
2048 for line in p.output.splitlines():
2051 return m.group('appid'), m.group('vercode'), m.group('vername')
2052 raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'")
2053 .format(apkfilename=apkfile))
2056 def get_minSdkVersion_aapt(apkfile):
2057 """Extract the minimum supported Android SDK from an APK using aapt
2059 :param apkfile: path to an APK file.
2060 :returns: the integer representing the SDK version
2062 r = re.compile(r"^sdkVersion:'([0-9]+)'")
2063 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
2064 for line in p.output.splitlines():
2067 return int(m.group(1))
2068 raise FDroidException(_('Reading minSdkVersion failed: "{apkfilename}"')
2069 .format(apkfilename=apkfile))
2074 self.returncode = None
2078 def SdkToolsPopen(commands, cwd=None, output=True):
2080 if cmd not in config:
2081 config[cmd] = find_sdk_tools_cmd(commands[0])
2082 abscmd = config[cmd]
2084 raise FDroidException(_("Could not find '{command}' on your system").format(command=cmd))
2086 test_aapt_version(config['aapt'])
2087 return FDroidPopen([abscmd] + commands[1:],
2088 cwd=cwd, output=output)
2091 def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2093 Run a command and capture the possibly huge output as bytes.
2095 :param commands: command and argument list like in subprocess.Popen
2096 :param cwd: optionally specifies a working directory
2097 :param envs: a optional dictionary of environment variables and their values
2098 :returns: A PopenResult.
2103 set_FDroidPopen_env()
2105 process_env = env.copy()
2106 if envs is not None and len(envs) > 0:
2107 process_env.update(envs)
2110 cwd = os.path.normpath(cwd)
2111 logging.debug("Directory: %s" % cwd)
2112 logging.debug("> %s" % ' '.join(commands))
2114 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
2115 result = PopenResult()
2118 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
2119 stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
2120 stderr=stderr_param)
2121 except OSError as e:
2122 raise BuildException("OSError while trying to execute " +
2123 ' '.join(commands) + ': ' + str(e))
2125 # TODO are these AsynchronousFileReader threads always exiting?
2126 if not stderr_to_stdout and options.verbose:
2127 stderr_queue = Queue()
2128 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
2130 while not stderr_reader.eof():
2131 while not stderr_queue.empty():
2132 line = stderr_queue.get()
2133 sys.stderr.buffer.write(line)
2138 stdout_queue = Queue()
2139 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
2142 # Check the queue for output (until there is no more to get)
2143 while not stdout_reader.eof():
2144 while not stdout_queue.empty():
2145 line = stdout_queue.get()
2146 if output and options.verbose:
2147 # Output directly to console
2148 sys.stderr.buffer.write(line)
2154 result.returncode = p.wait()
2155 result.output = buf.getvalue()
2157 # make sure all filestreams of the subprocess are closed
2158 for streamvar in ['stdin', 'stdout', 'stderr']:
2159 if hasattr(p, streamvar):
2160 stream = getattr(p, streamvar)
2166 def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2168 Run a command and capture the possibly huge output as a str.
2170 :param commands: command and argument list like in subprocess.Popen
2171 :param cwd: optionally specifies a working directory
2172 :param envs: a optional dictionary of environment variables and their values
2173 :returns: A PopenResult.
2175 result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
2176 result.output = result.output.decode('utf-8', 'ignore')
2180 gradle_comment = re.compile(r'[ ]*//')
2181 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
2182 gradle_line_matches = [
2183 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
2184 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
2185 re.compile(r'.*\.readLine\(.*'),
2189 def remove_signing_keys(build_dir):
2190 for root, dirs, files in os.walk(build_dir):
2191 if 'build.gradle' in files:
2192 path = os.path.join(root, 'build.gradle')
2194 with open(path, "r", encoding='utf8') as o:
2195 lines = o.readlines()
2201 with open(path, "w", encoding='utf8') as o:
2202 while i < len(lines):
2205 while line.endswith('\\\n'):
2206 line = line.rstrip('\\\n') + lines[i]
2209 if gradle_comment.match(line):
2214 opened += line.count('{')
2215 opened -= line.count('}')
2218 if gradle_signing_configs.match(line):
2223 if any(s.match(line) for s in gradle_line_matches):
2231 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
2234 'project.properties',
2236 'default.properties',
2237 'ant.properties', ]:
2238 if propfile in files:
2239 path = os.path.join(root, propfile)
2241 with open(path, "r", encoding='iso-8859-1') as o:
2242 lines = o.readlines()
2246 with open(path, "w", encoding='iso-8859-1') as o:
2248 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
2255 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
2258 def set_FDroidPopen_env(build=None):
2260 set up the environment variables for the build environment
2262 There is only a weak standard, the variables used by gradle, so also set
2263 up the most commonly used environment variables for SDK and NDK. Also, if
2264 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
2266 global env, orig_path
2270 orig_path = env['PATH']
2271 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
2272 env[n] = config['sdk_path']
2273 for k, v in config['java_paths'].items():
2274 env['JAVA%s_HOME' % k] = v
2276 missinglocale = True
2277 for k, v in env.items():
2278 if k == 'LANG' and v != 'C':
2279 missinglocale = False
2281 missinglocale = False
2283 env['LANG'] = 'en_US.UTF-8'
2285 if build is not None:
2286 path = build.ndk_path()
2287 paths = orig_path.split(os.pathsep)
2288 if path not in paths:
2289 paths = [path] + paths
2290 env['PATH'] = os.pathsep.join(paths)
2291 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
2292 env[n] = build.ndk_path()
2295 def replace_build_vars(cmd, build):
2296 cmd = cmd.replace('$$COMMIT$$', build.commit)
2297 cmd = cmd.replace('$$VERSION$$', build.versionName)
2298 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
2302 def replace_config_vars(cmd, build):
2303 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
2304 cmd = cmd.replace('$$NDK$$', build.ndk_path())
2305 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
2306 if build is not None:
2307 cmd = replace_build_vars(cmd, build)
2311 def place_srclib(root_dir, number, libpath):
2314 relpath = os.path.relpath(libpath, root_dir)
2315 proppath = os.path.join(root_dir, 'project.properties')
2318 if os.path.isfile(proppath):
2319 with open(proppath, "r", encoding='iso-8859-1') as o:
2320 lines = o.readlines()
2322 with open(proppath, "w", encoding='iso-8859-1') as o:
2325 if line.startswith('android.library.reference.%d=' % number):
2326 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2331 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2334 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z_\-]+\.(SF|RSA|DSA|EC)')
2337 def signer_fingerprint_short(sig):
2338 """Obtain shortened sha256 signing-key fingerprint for pkcs7 signature.
2340 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2341 for a given pkcs7 signature.
2343 :param sig: Contents of an APK signing certificate.
2344 :returns: shortened signing-key fingerprint.
2346 return signer_fingerprint(sig)[:7]
2349 def signer_fingerprint(sig):
2350 """Obtain sha256 signing-key fingerprint for pkcs7 signature.
2352 Extracts hexadecimal sha256 signing-key fingerprint string
2353 for a given pkcs7 signature.
2355 :param: Contents of an APK signature.
2356 :returns: shortened signature fingerprint.
2358 cert_encoded = get_certificate(sig)
2359 return hashlib.sha256(cert_encoded).hexdigest()
2362 def apk_signer_fingerprint(apk_path):
2363 """Obtain sha256 signing-key fingerprint for APK.
2365 Extracts hexadecimal sha256 signing-key fingerprint string
2368 :param apkpath: path to APK
2369 :returns: signature fingerprint
2372 with zipfile.ZipFile(apk_path, 'r') as apk:
2373 certs = [n for n in apk.namelist() if CERT_PATH_REGEX.match(n)]
2376 logging.error("Found no signing certificates on %s" % apk_path)
2379 logging.error("Found multiple signing certificates on %s" % apk_path)
2382 cert = apk.read(certs[0])
2383 return signer_fingerprint(cert)
2386 def apk_signer_fingerprint_short(apk_path):
2387 """Obtain shortened sha256 signing-key fingerprint for APK.
2389 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2390 for a given pkcs7 APK.
2392 :param apk_path: path to APK
2393 :returns: shortened signing-key fingerprint
2395 return apk_signer_fingerprint(apk_path)[:7]
2398 def metadata_get_sigdir(appid, vercode=None):
2399 """Get signature directory for app"""
2401 return os.path.join('metadata', appid, 'signatures', vercode)
2403 return os.path.join('metadata', appid, 'signatures')
2406 def metadata_find_developer_signature(appid, vercode=None):
2407 """Tires to find the developer signature for given appid.
2409 This picks the first signature file found in metadata an returns its
2412 :returns: sha256 signing key fingerprint of the developer signing key.
2413 None in case no signature can not be found."""
2415 # fetch list of dirs for all versions of signatures
2418 appversigdirs.append(metadata_get_sigdir(appid, vercode))
2420 appsigdir = metadata_get_sigdir(appid)
2421 if os.path.isdir(appsigdir):
2422 numre = re.compile('[0-9]+')
2423 for ver in os.listdir(appsigdir):
2424 if numre.match(ver):
2425 appversigdir = os.path.join(appsigdir, ver)
2426 appversigdirs.append(appversigdir)
2428 for sigdir in appversigdirs:
2429 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2430 glob.glob(os.path.join(sigdir, '*.EC')) + \
2431 glob.glob(os.path.join(sigdir, '*.RSA'))
2433 raise FDroidException('ambiguous signatures, please make sure there is only one signature in \'{}\'. (The signature has to be the App maintainers signature for version of the APK.)'.format(sigdir))
2435 with open(sig, 'rb') as f:
2436 return signer_fingerprint(f.read())
2440 def metadata_find_signing_files(appid, vercode):
2441 """Gets a list of singed manifests and signatures.
2443 :param appid: app id string
2444 :param vercode: app version code
2445 :returns: a list of triplets for each signing key with following paths:
2446 (signature_file, singed_file, manifest_file)
2449 sigdir = metadata_get_sigdir(appid, vercode)
2450 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2451 glob.glob(os.path.join(sigdir, '*.EC')) + \
2452 glob.glob(os.path.join(sigdir, '*.RSA'))
2453 extre = re.compile('(\.DSA|\.EC|\.RSA)$')
2455 sf = extre.sub('.SF', sig)
2456 if os.path.isfile(sf):
2457 mf = os.path.join(sigdir, 'MANIFEST.MF')
2458 if os.path.isfile(mf):
2459 ret.append((sig, sf, mf))
2463 def metadata_find_developer_signing_files(appid, vercode):
2464 """Get developer signature files for specified app from metadata.
2466 :returns: A triplet of paths for signing files from metadata:
2467 (signature_file, singed_file, manifest_file)
2469 allsigningfiles = metadata_find_signing_files(appid, vercode)
2470 if allsigningfiles and len(allsigningfiles) == 1:
2471 return allsigningfiles[0]
2476 def apk_strip_signatures(signed_apk, strip_manifest=False):
2477 """Removes signatures from APK.
2479 :param signed_apk: path to apk file.
2480 :param strip_manifest: when set to True also the manifest file will
2481 be removed from the APK.
2483 with tempfile.TemporaryDirectory() as tmpdir:
2484 tmp_apk = os.path.join(tmpdir, 'tmp.apk')
2485 shutil.move(signed_apk, tmp_apk)
2486 with ZipFile(tmp_apk, 'r') as in_apk:
2487 with ZipFile(signed_apk, 'w') as out_apk:
2488 for info in in_apk.infolist():
2489 if not apk_sigfile.match(info.filename):
2491 if info.filename != 'META-INF/MANIFEST.MF':
2492 buf = in_apk.read(info.filename)
2493 out_apk.writestr(info, buf)
2495 buf = in_apk.read(info.filename)
2496 out_apk.writestr(info, buf)
2499 def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest):
2500 """Implats a signature from metadata into an APK.
2502 Note: this changes there supplied APK in place. So copy it if you
2503 need the original to be preserved.
2505 :param apkpath: location of the apk
2507 # get list of available signature files in metadata
2508 with tempfile.TemporaryDirectory() as tmpdir:
2509 apkwithnewsig = os.path.join(tmpdir, 'newsig.apk')
2510 with ZipFile(apkpath, 'r') as in_apk:
2511 with ZipFile(apkwithnewsig, 'w') as out_apk:
2512 for sig_file in [signaturefile, signedfile, manifest]:
2513 with open(sig_file, 'rb') as fp:
2515 info = zipfile.ZipInfo('META-INF/' + os.path.basename(sig_file))
2516 info.compress_type = zipfile.ZIP_DEFLATED
2517 info.create_system = 0 # "Windows" aka "FAT", what Android SDK uses
2518 out_apk.writestr(info, buf)
2519 for info in in_apk.infolist():
2520 if not apk_sigfile.match(info.filename):
2521 if info.filename != 'META-INF/MANIFEST.MF':
2522 buf = in_apk.read(info.filename)
2523 out_apk.writestr(info, buf)
2525 p = SdkToolsPopen(['zipalign', '-v', '4', apkwithnewsig, apkpath])
2526 if p.returncode != 0:
2527 raise BuildException("Failed to align application")
2530 def apk_extract_signatures(apkpath, outdir, manifest=True):
2531 """Extracts a signature files from APK and puts them into target directory.
2533 :param apkpath: location of the apk
2534 :param outdir: folder where the extracted signature files will be stored
2535 :param manifest: (optionally) disable extracting manifest file
2537 with ZipFile(apkpath, 'r') as in_apk:
2538 for f in in_apk.infolist():
2539 if apk_sigfile.match(f.filename) or \
2540 (manifest and f.filename == 'META-INF/MANIFEST.MF'):
2541 newpath = os.path.join(outdir, os.path.basename(f.filename))
2542 with open(newpath, 'wb') as out_file:
2543 out_file.write(in_apk.read(f.filename))
2546 def sign_apk(unsigned_path, signed_path, keyalias):
2547 """Sign and zipalign an unsigned APK, then save to a new file, deleting the unsigned
2549 android-18 (4.3) finally added support for reasonable hash
2550 algorithms, like SHA-256, before then, the only options were MD5
2551 and SHA1 :-/ This aims to use SHA-256 when the APK does not target
2552 older Android versions, and is therefore safe to do so.
2554 https://issuetracker.google.com/issues/36956587
2555 https://android-review.googlesource.com/c/platform/libcore/+/44491
2559 if get_minSdkVersion_aapt(unsigned_path) < 18:
2560 signature_algorithm = ['-sigalg', 'SHA1withRSA', '-digestalg', 'SHA1']
2562 signature_algorithm = ['-sigalg', 'SHA256withRSA', '-digestalg', 'SHA-256']
2564 p = FDroidPopen([config['jarsigner'], '-keystore', config['keystore'],
2565 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2566 '-keypass:env', 'FDROID_KEY_PASS']
2567 + signature_algorithm + [unsigned_path, keyalias],
2569 'FDROID_KEY_STORE_PASS': config['keystorepass'],
2570 'FDROID_KEY_PASS': config['keypass'], })
2571 if p.returncode != 0:
2572 raise BuildException(_("Failed to sign application"), p.output)
2574 p = SdkToolsPopen(['zipalign', '-v', '4', unsigned_path, signed_path])
2575 if p.returncode != 0:
2576 raise BuildException(_("Failed to zipalign application"))
2577 os.remove(unsigned_path)
2580 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
2581 """Verify that two apks are the same
2583 One of the inputs is signed, the other is unsigned. The signature metadata
2584 is transferred from the signed to the unsigned apk, and then jarsigner is
2585 used to verify that the signature from the signed apk is also varlid for
2586 the unsigned one. If the APK given as unsigned actually does have a
2587 signature, it will be stripped out and ignored.
2589 There are two SHA1 git commit IDs that fdroidserver includes in the builds
2590 it makes: fdroidserverid and buildserverid. Originally, these were inserted
2591 into AndroidManifest.xml, but that makes the build not reproducible. So
2592 instead they are included as separate files in the APK's META-INF/ folder.
2593 If those files exist in the signed APK, they will be part of the signature
2594 and need to also be included in the unsigned APK for it to validate.
2596 :param signed_apk: Path to a signed apk file
2597 :param unsigned_apk: Path to an unsigned apk file expected to match it
2598 :param tmp_dir: Path to directory for temporary files
2599 :returns: None if the verification is successful, otherwise a string
2600 describing what went wrong.
2603 if not os.path.isfile(signed_apk):
2604 return 'can not verify: file does not exists: {}'.format(signed_apk)
2606 if not os.path.isfile(unsigned_apk):
2607 return 'can not verify: file does not exists: {}'.format(unsigned_apk)
2609 with ZipFile(signed_apk, 'r') as signed:
2610 meta_inf_files = ['META-INF/MANIFEST.MF']
2611 for f in signed.namelist():
2612 if apk_sigfile.match(f) \
2613 or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
2614 meta_inf_files.append(f)
2615 if len(meta_inf_files) < 3:
2616 return "Signature files missing from {0}".format(signed_apk)
2618 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2619 with ZipFile(unsigned_apk, 'r') as unsigned:
2620 # only read the signature from the signed APK, everything else from unsigned
2621 with ZipFile(tmp_apk, 'w') as tmp:
2622 for filename in meta_inf_files:
2623 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2624 for info in unsigned.infolist():
2625 if info.filename in meta_inf_files:
2626 logging.warning('Ignoring %s from %s',
2627 info.filename, unsigned_apk)
2629 if info.filename in tmp.namelist():
2630 return "duplicate filename found: " + info.filename
2631 tmp.writestr(info, unsigned.read(info.filename))
2633 verified = verify_apk_signature(tmp_apk)
2636 logging.info("...NOT verified - {0}".format(tmp_apk))
2637 return compare_apks(signed_apk, tmp_apk, tmp_dir,
2638 os.path.dirname(unsigned_apk))
2640 logging.info("...successfully verified")
2644 def verify_jar_signature(jar):
2645 """Verifies the signature of a given JAR file.
2647 jarsigner is very shitty: unsigned JARs pass as "verified"! So
2648 this has to turn on -strict then check for result 4, since this
2649 does not expect the signature to be from a CA-signed certificate.
2651 :raises: VerificationException() if the JAR's signature could not be verified
2655 error = _('JAR signature failed to verify: {path}').format(path=jar)
2657 output = subprocess.check_output([config['jarsigner'], '-strict', '-verify', jar],
2658 stderr=subprocess.STDOUT)
2659 raise VerificationException(error + '\n' + output.decode('utf-8'))
2660 except subprocess.CalledProcessError as e:
2661 if e.returncode == 4:
2662 logging.debug(_('JAR signature verified: {path}').format(path=jar))
2664 raise VerificationException(error + '\n' + e.output.decode('utf-8'))
2667 def verify_apk_signature(apk, min_sdk_version=None):
2668 """verify the signature on an APK
2670 Try to use apksigner whenever possible since jarsigner is very
2671 shitty: unsigned APKs pass as "verified"! Warning, this does
2672 not work on JARs with apksigner >= 0.7 (build-tools 26.0.1)
2674 :returns: boolean whether the APK was verified
2676 if set_command_in_config('apksigner'):
2677 args = [config['apksigner'], 'verify']
2679 args += ['--min-sdk-version=' + min_sdk_version]
2681 args += ['--verbose']
2683 output = subprocess.check_output(args + [apk])
2685 logging.debug(apk + ': ' + output.decode('utf-8'))
2687 except subprocess.CalledProcessError as e:
2688 logging.error('\n' + apk + ': ' + e.output.decode('utf-8'))
2690 if not config.get('jarsigner_warning_displayed'):
2691 config['jarsigner_warning_displayed'] = True
2692 logging.warning(_("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner"))
2694 verify_jar_signature(apk)
2696 except Exception as e:
2701 def verify_old_apk_signature(apk):
2702 """verify the signature on an archived APK, supporting deprecated algorithms
2704 F-Droid aims to keep every single binary that it ever published. Therefore,
2705 it needs to be able to verify APK signatures that include deprecated/removed
2706 algorithms. For example, jarsigner treats an MD5 signature as unsigned.
2708 jarsigner passes unsigned APKs as "verified"! So this has to turn
2709 on -strict then check for result 4.
2711 Just to be safe, this never reuses the file, and locks down the
2712 file permissions while in use. That should prevent a bad actor
2713 from changing the settings during operation.
2715 :returns: boolean whether the APK was verified
2719 _java_security = os.path.join(os.getcwd(), '.java.security')
2720 if os.path.exists(_java_security):
2721 os.remove(_java_security)
2722 with open(_java_security, 'w') as fp:
2723 fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
2724 os.chmod(_java_security, 0o400)
2728 config['jarsigner'],
2729 '-J-Djava.security.properties=' + _java_security,
2730 '-strict', '-verify', apk
2732 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
2733 except subprocess.CalledProcessError as e:
2734 if e.returncode != 4:
2737 logging.debug(_('JAR signature verified: {path}').format(path=apk))
2740 if os.path.exists(_java_security):
2741 os.chmod(_java_security, 0o600)
2742 os.remove(_java_security)
2744 logging.error(_('Old APK signature failed to verify: {path}').format(path=apk)
2745 + '\n' + output.decode('utf-8'))
2749 apk_badchars = re.compile('''[/ :;'"]''')
2752 def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
2755 Returns None if the apk content is the same (apart from the signing key),
2756 otherwise a string describing what's different, or what went wrong when
2757 trying to do the comparison.
2763 absapk1 = os.path.abspath(apk1)
2764 absapk2 = os.path.abspath(apk2)
2766 if set_command_in_config('diffoscope'):
2767 logfilename = os.path.join(log_dir, os.path.basename(absapk1))
2768 htmlfile = logfilename + '.diffoscope.html'
2769 textfile = logfilename + '.diffoscope.txt'
2770 if subprocess.call([config['diffoscope'],
2771 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2772 '--html', htmlfile, '--text', textfile,
2773 absapk1, absapk2]) != 0:
2774 return("Failed to unpack " + apk1)
2776 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2777 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2778 for d in [apk1dir, apk2dir]:
2779 if os.path.exists(d):
2782 os.mkdir(os.path.join(d, 'jar-xf'))
2784 if subprocess.call(['jar', 'xf',
2785 os.path.abspath(apk1)],
2786 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2787 return("Failed to unpack " + apk1)
2788 if subprocess.call(['jar', 'xf',
2789 os.path.abspath(apk2)],
2790 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2791 return("Failed to unpack " + apk2)
2793 if set_command_in_config('apktool'):
2794 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2796 return("Failed to unpack " + apk1)
2797 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2799 return("Failed to unpack " + apk2)
2801 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2802 lines = p.output.splitlines()
2803 if len(lines) != 1 or 'META-INF' not in lines[0]:
2804 if set_command_in_config('meld'):
2805 p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
2806 return("Unexpected diff output - " + p.output)
2808 # since everything verifies, delete the comparison to keep cruft down
2809 shutil.rmtree(apk1dir)
2810 shutil.rmtree(apk2dir)
2812 # If we get here, it seems like they're the same!
2816 def set_command_in_config(command):
2817 '''Try to find specified command in the path, if it hasn't been
2818 manually set in config.py. If found, it is added to the config
2819 dict. The return value says whether the command is available.
2822 if command in config:
2825 tmp = find_command(command)
2827 config[command] = tmp
2832 def find_command(command):
2833 '''find the full path of a command, or None if it can't be found in the PATH'''
2836 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2838 fpath, fname = os.path.split(command)
2843 for path in os.environ["PATH"].split(os.pathsep):
2844 path = path.strip('"')
2845 exe_file = os.path.join(path, command)
2846 if is_exe(exe_file):
2853 '''generate a random password for when generating keys'''
2854 h = hashlib.sha256()
2855 h.update(os.urandom(16)) # salt
2856 h.update(socket.getfqdn().encode('utf-8'))
2857 passwd = base64.b64encode(h.digest()).strip()
2858 return passwd.decode('utf-8')
2861 def genkeystore(localconfig):
2863 Generate a new key with password provided in :param localconfig and add it to new keystore
2864 :return: hexed public key, public key fingerprint
2866 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2867 keystoredir = os.path.dirname(localconfig['keystore'])
2868 if keystoredir is None or keystoredir == '':
2869 keystoredir = os.path.join(os.getcwd(), keystoredir)
2870 if not os.path.exists(keystoredir):
2871 os.makedirs(keystoredir, mode=0o700)
2874 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
2875 'FDROID_KEY_PASS': localconfig['keypass'],
2877 p = FDroidPopen([config['keytool'], '-genkey',
2878 '-keystore', localconfig['keystore'],
2879 '-alias', localconfig['repo_keyalias'],
2880 '-keyalg', 'RSA', '-keysize', '4096',
2881 '-sigalg', 'SHA256withRSA',
2882 '-validity', '10000',
2883 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2884 '-keypass:env', 'FDROID_KEY_PASS',
2885 '-dname', localconfig['keydname']], envs=env_vars)
2886 if p.returncode != 0:
2887 raise BuildException("Failed to generate key", p.output)
2888 os.chmod(localconfig['keystore'], 0o0600)
2889 if not options.quiet:
2890 # now show the lovely key that was just generated
2891 p = FDroidPopen([config['keytool'], '-list', '-v',
2892 '-keystore', localconfig['keystore'],
2893 '-alias', localconfig['repo_keyalias'],
2894 '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
2895 logging.info(p.output.strip() + '\n\n')
2896 # get the public key
2897 p = FDroidPopenBytes([config['keytool'], '-exportcert',
2898 '-keystore', localconfig['keystore'],
2899 '-alias', localconfig['repo_keyalias'],
2900 '-storepass:env', 'FDROID_KEY_STORE_PASS']
2901 + config['smartcardoptions'],
2902 envs=env_vars, output=False, stderr_to_stdout=False)
2903 if p.returncode != 0 or len(p.output) < 20:
2904 raise BuildException("Failed to get public key", p.output)
2906 fingerprint = get_cert_fingerprint(pubkey)
2907 return hexlify(pubkey), fingerprint
2910 def get_cert_fingerprint(pubkey):
2912 Generate a certificate fingerprint the same way keytool does it
2913 (but with slightly different formatting)
2915 digest = hashlib.sha256(pubkey).digest()
2916 ret = [' '.join("%02X" % b for b in bytearray(digest))]
2917 return " ".join(ret)
2920 def get_certificate(certificate_file):
2922 Extracts a certificate from the given file.
2923 :param certificate_file: file bytes (as string) representing the certificate
2924 :return: A binary representation of the certificate's public key, or None in case of error
2926 content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
2927 if content.getComponentByName('contentType') != rfc2315.signedData:
2929 content = decoder.decode(content.getComponentByName('content'),
2930 asn1Spec=rfc2315.SignedData())[0]
2932 certificates = content.getComponentByName('certificates')
2933 cert = certificates[0].getComponentByName('certificate')
2935 logging.error("Certificates not found.")
2937 return encoder.encode(cert)
2940 def load_stats_fdroid_signing_key_fingerprints():
2941 """Load list of signing-key fingerprints stored by fdroid publish from file.
2943 :returns: list of dictionanryies containing the singing-key fingerprints.
2945 jar_file = os.path.join('stats', 'publishsigkeys.jar')
2946 if not os.path.isfile(jar_file):
2948 cmd = [config['jarsigner'], '-strict', '-verify', jar_file]
2949 p = FDroidPopen(cmd, output=False)
2950 if p.returncode != 4:
2951 raise FDroidException("Signature validation of '{}' failed! "
2952 "Please run publish again to rebuild this file.".format(jar_file))
2954 jar_sigkey = apk_signer_fingerprint(jar_file)
2955 repo_key_sig = config.get('repo_key_sha256')
2957 if jar_sigkey != repo_key_sig:
2958 raise FDroidException("Signature key fingerprint of file '{}' does not match repo_key_sha256 in config.py (found fingerprint: '{}')".format(jar_file, jar_sigkey))
2960 logging.warning("repo_key_sha256 not in config.py, setting it to the signature key fingerprint of '{}'".format(jar_file))
2961 config['repo_key_sha256'] = jar_sigkey
2962 write_to_config(config, 'repo_key_sha256')
2964 with zipfile.ZipFile(jar_file, 'r') as f:
2965 return json.loads(str(f.read('publishsigkeys.json'), 'utf-8'))
2968 def write_to_config(thisconfig, key, value=None, config_file=None):
2969 '''write a key/value to the local config.py
2971 NOTE: only supports writing string variables.
2973 :param thisconfig: config dictionary
2974 :param key: variable name in config.py to be overwritten/added
2975 :param value: optional value to be written, instead of fetched
2976 from 'thisconfig' dictionary.
2979 origkey = key + '_orig'
2980 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2981 cfg = config_file if config_file else 'config.py'
2983 # load config file, create one if it doesn't exist
2984 if not os.path.exists(cfg):
2985 open(cfg, 'a').close()
2986 logging.info("Creating empty " + cfg)
2987 with open(cfg, 'r', encoding="utf-8") as f:
2988 lines = f.readlines()
2990 # make sure the file ends with a carraige return
2992 if not lines[-1].endswith('\n'):
2995 # regex for finding and replacing python string variable
2996 # definitions/initializations
2997 pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
2998 repl = key + ' = "' + value + '"'
2999 pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
3000 repl2 = key + " = '" + value + "'"
3002 # If we replaced this line once, we make sure won't be a
3003 # second instance of this line for this key in the document.
3006 with open(cfg, 'w', encoding="utf-8") as f:
3008 if pattern.match(line) or pattern2.match(line):
3010 line = pattern.sub(repl, line)
3011 line = pattern2.sub(repl2, line)
3022 def parse_xml(path):
3023 return XMLElementTree.parse(path).getroot()
3026 def string_is_integer(string):
3034 def local_rsync(options, fromdir, todir):
3035 '''Rsync method for local to local copying of things
3037 This is an rsync wrapper with all the settings for safe use within
3038 the various fdroidserver use cases. This uses stricter rsync
3039 checking on all files since people using offline mode are already
3040 prioritizing security above ease and speed.
3043 rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
3044 '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
3045 if not options.no_checksum:
3046 rsyncargs.append('--checksum')
3048 rsyncargs += ['--verbose']
3050 rsyncargs += ['--quiet']
3051 logging.debug(' '.join(rsyncargs + [fromdir, todir]))
3052 if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
3053 raise FDroidException()
3056 def get_per_app_repos():
3057 '''per-app repos are dirs named with the packageName of a single app'''
3059 # Android packageNames are Java packages, they may contain uppercase or
3060 # lowercase letters ('A' through 'Z'), numbers, and underscores
3061 # ('_'). However, individual package name parts may only start with
3062 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
3063 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
3066 for root, dirs, files in os.walk(os.getcwd()):
3068 print('checking', root, 'for', d)
3069 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
3070 # standard parts of an fdroid repo, so never packageNames
3073 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
3079 def is_repo_file(filename):
3080 '''Whether the file in a repo is a build product to be delivered to users'''
3081 if isinstance(filename, str):
3082 filename = filename.encode('utf-8', errors="surrogateescape")
3083 return os.path.isfile(filename) \
3084 and not filename.endswith(b'.asc') \
3085 and not filename.endswith(b'.sig') \
3086 and os.path.basename(filename) not in [
3088 b'index_unsigned.jar',
3097 def get_examples_dir():
3098 '''Return the dir where the fdroidserver example files are available'''
3100 tmp = os.path.dirname(sys.argv[0])
3101 if os.path.basename(tmp) == 'bin':
3102 egg_links = glob.glob(os.path.join(tmp, '..',
3103 'local/lib/python3.*/site-packages/fdroidserver.egg-link'))
3105 # installed from local git repo
3106 examplesdir = os.path.join(open(egg_links[0]).readline().rstrip(), 'examples')
3109 examplesdir = os.path.dirname(os.path.dirname(__file__)) + '/share/doc/fdroidserver/examples'
3110 if not os.path.exists(examplesdir): # use UNIX layout
3111 examplesdir = os.path.dirname(tmp) + '/share/doc/fdroidserver/examples'
3113 # we're running straight out of the git repo
3114 prefix = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
3115 examplesdir = prefix + '/examples'
3120 def get_wiki_timestamp(timestamp=None):
3121 """Return current time in the standard format for posting to the wiki"""
3123 if timestamp is None:
3124 timestamp = time.gmtime()
3125 return time.strftime("%Y-%m-%d %H:%M:%SZ", timestamp)
3128 def get_android_tools_versions(ndk_path=None):
3129 '''get a list of the versions of all installed Android SDK/NDK components'''
3132 sdk_path = config['sdk_path']
3133 if sdk_path[-1] != '/':
3137 ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT')
3138 if os.path.isfile(ndk_release_txt):
3139 with open(ndk_release_txt, 'r') as fp:
3140 components.append((os.path.basename(ndk_path), fp.read()[:-1]))
3142 pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE)
3143 for root, dirs, files in os.walk(sdk_path):
3144 if 'source.properties' in files:
3145 source_properties = os.path.join(root, 'source.properties')
3146 with open(source_properties, 'r') as fp:
3147 m = pattern.search(fp.read())
3149 components.append((root[len(sdk_path):], m.group(1)))
3154 def get_android_tools_version_log(ndk_path=None):
3155 '''get a list of the versions of all installed Android SDK/NDK components'''
3156 log = '== Installed Android Tools ==\n\n'
3157 components = get_android_tools_versions(ndk_path)
3158 for name, version in sorted(components):
3159 log += '* ' + name + ' (' + version + ')\n'
3164 def get_git_describe_link():
3165 """Get a link to the current fdroiddata commit, to post to the wiki
3169 output = subprocess.check_output(['git', 'describe', '--always', '--dirty', '--abbrev=0'],
3170 universal_newlines=True).strip()
3171 except subprocess.CalledProcessError:
3174 commit = output.replace('-dirty', '')
3175 return ('* fdroiddata: [https://gitlab.com/fdroid/fdroiddata/commit/{commit} {id}]\n'
3176 .format(commit=commit, id=output))
3178 logging.error(_("'{path}' failed to execute!").format(path='git describe'))