3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
40 import xml.etree.ElementTree as XMLElementTree
42 from binascii import hexlify
43 from datetime import datetime, timedelta
44 from distutils.version import LooseVersion
45 from queue import Queue
46 from zipfile import ZipFile
48 from pyasn1.codec.der import decoder, encoder
49 from pyasn1_modules import rfc2315
50 from pyasn1.error import PyAsn1Error
52 from distutils.util import strtobool
54 import fdroidserver.metadata
55 from fdroidserver import _
56 from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException,\
57 BuildException, VerificationException
58 from .asynchronousfilereader import AsynchronousFileReader
60 # this is the build-tools version, aapt has a separate version that
61 # has to be manually set in test_aapt_version()
62 MINIMUM_AAPT_VERSION = '26.0.0'
64 VERCODE_OPERATION_RE = re.compile(r'^([ 0-9/*+-]|%c)+$')
66 # A signature block file with a .DSA, .RSA, or .EC extension
67 CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
68 APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
69 STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
71 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
80 'sdk_path': "$ANDROID_HOME",
84 'r12b': "$ANDROID_NDK",
90 'build_tools': MINIMUM_AAPT_VERSION,
91 'force_build_tools': False,
96 'accepted_formats': ['txt', 'yml'],
97 'sync_from_local_copy_dir': False,
98 'allow_disabled_algorithms': False,
99 'per_app_repos': False,
100 'make_current_version_link': True,
101 'current_version_name_source': 'Name',
102 'update_stats': False,
104 'stats_server': None,
106 'stats_to_carbon': False,
108 'build_server_always': False,
109 'keystore': 'keystore.jks',
110 'smartcardoptions': [],
120 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
121 'repo_name': "My First FDroid Repo Demo",
122 'repo_icon': "fdroid-icon.png",
123 'repo_description': '''
124 This is a repository of apps to be used with FDroid. Applications in this
125 repository are either official binaries built by the original application
126 developers, or are binaries built from source by the admin of f-droid.org
127 using the tools on https://gitlab.com/u/fdroid.
133 def setup_global_opts(parser):
134 try: # the buildserver VM might not have PIL installed
135 from PIL import PngImagePlugin
136 logger = logging.getLogger(PngImagePlugin.__name__)
137 logger.setLevel(logging.INFO) # tame the "STREAM" debug messages
141 parser.add_argument("-v", "--verbose", action="store_true", default=False,
142 help=_("Spew out even more information than normal"))
143 parser.add_argument("-q", "--quiet", action="store_true", default=False,
144 help=_("Restrict output to warnings and errors"))
147 def _add_java_paths_to_config(pathlist, thisconfig):
148 def path_version_key(s):
150 for u in re.split('[^0-9]+', s):
152 versionlist.append(int(u))
157 for d in sorted(pathlist, key=path_version_key):
158 if os.path.islink(d):
160 j = os.path.basename(d)
161 # the last one found will be the canonical one, so order appropriately
163 r'^1\.([6-9])\.0\.jdk$', # OSX
164 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
165 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
166 r'^jdk([6-9])-openjdk$', # Arch
167 r'^java-([6-9])-openjdk$', # Arch
168 r'^java-([6-9])-jdk$', # Arch (oracle)
169 r'^java-1\.([6-9])\.0-.*$', # RedHat
170 r'^java-([6-9])-oracle$', # Debian WebUpd8
171 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
172 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
173 r'^oracle-jdk-bin-1\.([7-9]).*$', # Gentoo (oracle)
174 r'^icedtea-bin-([7-9]).*$', # Gentoo (openjdk)
176 m = re.match(regex, j)
179 for p in [d, os.path.join(d, 'Contents', 'Home')]:
180 if os.path.exists(os.path.join(p, 'bin', 'javac')):
181 thisconfig['java_paths'][m.group(1)] = p
184 def fill_config_defaults(thisconfig):
185 for k, v in default_config.items():
186 if k not in thisconfig:
189 # Expand paths (~users and $vars)
190 def expand_path(path):
194 path = os.path.expanduser(path)
195 path = os.path.expandvars(path)
200 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
205 thisconfig[k + '_orig'] = v
207 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
208 if thisconfig['java_paths'] is None:
209 thisconfig['java_paths'] = dict()
211 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
212 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
213 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
214 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
215 pathlist += glob.glob('/opt/oracle-jdk-*1.[7-9]*')
216 pathlist += glob.glob('/opt/icedtea-*[7-9]*')
217 if os.getenv('JAVA_HOME') is not None:
218 pathlist.append(os.getenv('JAVA_HOME'))
219 if os.getenv('PROGRAMFILES') is not None:
220 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
221 _add_java_paths_to_config(pathlist, thisconfig)
223 for java_version in ('7', '8', '9'):
224 if java_version not in thisconfig['java_paths']:
226 java_home = thisconfig['java_paths'][java_version]
227 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
228 if os.path.exists(jarsigner):
229 thisconfig['jarsigner'] = jarsigner
230 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
231 break # Java7 is preferred, so quit if found
233 for k in ['ndk_paths', 'java_paths']:
239 thisconfig[k][k2] = exp
240 thisconfig[k][k2 + '_orig'] = v
243 def regsub_file(pattern, repl, path):
244 with open(path, 'rb') as f:
246 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
247 with open(path, 'wb') as f:
251 def read_config(opts, config_file='config.py'):
252 """Read the repository config
254 The config is read from config_file, which is in the current
255 directory when any of the repo management commands are used. If
256 there is a local metadata file in the git repo, then config.py is
257 not required, just use defaults.
260 global config, options
262 if config is not None:
269 if os.path.isfile(config_file):
270 logging.debug(_("Reading '{config_file}'").format(config_file=config_file))
271 with io.open(config_file, "rb") as f:
272 code = compile(f.read(), config_file, 'exec')
273 exec(code, None, config)
275 logging.warning(_("No 'config.py' found, using defaults."))
277 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
279 if not type(config[k]) in (str, list, tuple):
281 _("'{field}' will be in random order! Use () or [] brackets if order is important!")
284 # smartcardoptions must be a list since its command line args for Popen
285 if 'smartcardoptions' in config:
286 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
287 elif 'keystore' in config and config['keystore'] == 'NONE':
288 # keystore='NONE' means use smartcard, these are required defaults
289 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
290 'SunPKCS11-OpenSC', '-providerClass',
291 'sun.security.pkcs11.SunPKCS11',
292 '-providerArg', 'opensc-fdroid.cfg']
294 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
295 st = os.stat(config_file)
296 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
297 logging.warning(_("unsafe permissions on '{config_file}' (should be 0600)!")
298 .format(config_file=config_file))
300 fill_config_defaults(config)
302 for k in ["repo_description", "archive_description"]:
304 config[k] = clean_description(config[k])
306 if 'serverwebroot' in config:
307 if isinstance(config['serverwebroot'], str):
308 roots = [config['serverwebroot']]
309 elif all(isinstance(item, str) for item in config['serverwebroot']):
310 roots = config['serverwebroot']
312 raise TypeError(_('only accepts strings, lists, and tuples'))
314 for rootstr in roots:
315 # since this is used with rsync, where trailing slashes have
316 # meaning, ensure there is always a trailing slash
317 if rootstr[-1] != '/':
319 rootlist.append(rootstr.replace('//', '/'))
320 config['serverwebroot'] = rootlist
322 if 'servergitmirrors' in config:
323 if isinstance(config['servergitmirrors'], str):
324 roots = [config['servergitmirrors']]
325 elif all(isinstance(item, str) for item in config['servergitmirrors']):
326 roots = config['servergitmirrors']
328 raise TypeError(_('only accepts strings, lists, and tuples'))
329 config['servergitmirrors'] = roots
334 def assert_config_keystore(config):
335 """Check weather keystore is configured correctly and raise exception if not."""
338 if 'repo_keyalias' not in config:
340 logging.critical(_("'repo_keyalias' not found in config.py!"))
341 if 'keystore' not in config:
343 logging.critical(_("'keystore' not found in config.py!"))
344 elif not os.path.exists(config['keystore']):
346 logging.critical("'" + config['keystore'] + "' does not exist!")
347 if 'keystorepass' not in config:
349 logging.critical(_("'keystorepass' not found in config.py!"))
350 if 'keypass' not in config:
352 logging.critical(_("'keypass' not found in config.py!"))
354 raise FDroidException("This command requires a signing key, " +
355 "you can create one using: fdroid update --create-key")
358 def find_sdk_tools_cmd(cmd):
359 '''find a working path to a tool from the Android SDK'''
362 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
363 # try to find a working path to this command, in all the recent possible paths
364 if 'build_tools' in config:
365 build_tools = os.path.join(config['sdk_path'], 'build-tools')
366 # if 'build_tools' was manually set and exists, check only that one
367 configed_build_tools = os.path.join(build_tools, config['build_tools'])
368 if os.path.exists(configed_build_tools):
369 tooldirs.append(configed_build_tools)
371 # no configed version, so hunt known paths for it
372 for f in sorted(os.listdir(build_tools), reverse=True):
373 if os.path.isdir(os.path.join(build_tools, f)):
374 tooldirs.append(os.path.join(build_tools, f))
375 tooldirs.append(build_tools)
376 sdk_tools = os.path.join(config['sdk_path'], 'tools')
377 if os.path.exists(sdk_tools):
378 tooldirs.append(sdk_tools)
379 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
380 if os.path.exists(sdk_platform_tools):
381 tooldirs.append(sdk_platform_tools)
382 tooldirs.append('/usr/bin')
384 path = os.path.join(d, cmd)
385 if os.path.isfile(path):
387 test_aapt_version(path)
389 # did not find the command, exit with error message
390 ensure_build_tools_exists(config)
393 def test_aapt_version(aapt):
394 '''Check whether the version of aapt is new enough'''
395 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
396 if output is None or output == '':
397 logging.error(_("'{path}' failed to execute!").format(path=aapt))
399 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
404 # the Debian package has the version string like "v0.2-23.0.2"
407 if LooseVersion(bugfix) < LooseVersion(MINIMUM_AAPT_VERSION):
409 elif LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.4062713'):
412 logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-{version} or newer!")
413 .format(aapt=aapt, version=MINIMUM_AAPT_VERSION))
415 logging.warning(_('Unknown version of aapt, might cause problems: ') + output)
418 def test_sdk_exists(thisconfig):
419 if 'sdk_path' not in thisconfig:
420 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
421 test_aapt_version(thisconfig['aapt'])
424 logging.error(_("'sdk_path' not set in 'config.py'!"))
426 if thisconfig['sdk_path'] == default_config['sdk_path']:
427 logging.error(_('No Android SDK found!'))
428 logging.error(_('You can use ANDROID_HOME to set the path to your SDK, i.e.:'))
429 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
431 if not os.path.exists(thisconfig['sdk_path']):
432 logging.critical(_("Android SDK path '{path}' does not exist!")
433 .format(path=thisconfig['sdk_path']))
435 if not os.path.isdir(thisconfig['sdk_path']):
436 logging.critical(_("Android SDK path '{path}' is not a directory!")
437 .format(path=thisconfig['sdk_path']))
439 for d in ['build-tools', 'platform-tools', 'tools']:
440 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
441 logging.critical(_("Android SDK '{path}' does not have '{dirname}' installed!")
442 .format(path=thisconfig['sdk_path'], dirname=d))
447 def ensure_build_tools_exists(thisconfig):
448 if not test_sdk_exists(thisconfig):
449 raise FDroidException(_("Android SDK not found!"))
450 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
451 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
452 if not os.path.isdir(versioned_build_tools):
453 raise FDroidException(
454 _("Android build-tools path '{path}' does not exist!")
455 .format(path=versioned_build_tools))
458 def get_local_metadata_files():
459 '''get any metadata files local to an app's source repo
461 This tries to ignore anything that does not count as app metdata,
462 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
465 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
468 def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False):
470 :param appids: arguments in the form of multiple appid:[vc] strings
471 :returns: a dictionary with the set of vercodes specified for each package
474 if not appid_versionCode_pairs:
477 for p in appid_versionCode_pairs:
478 if allow_vercodes and ':' in p:
479 package, vercode = p.split(':')
481 package, vercode = p, None
482 if package not in vercodes:
483 vercodes[package] = [vercode] if vercode else []
485 elif vercode and vercode not in vercodes[package]:
486 vercodes[package] += [vercode] if vercode else []
491 def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
492 """Build a list of App instances for processing
494 On top of what read_pkg_args does, this returns the whole app
495 metadata, but limiting the builds list to the builds matching the
496 appid_versionCode_pairs and vercodes specified. If no appid_versionCode_pairs are specified, then
497 all App and Build instances are returned.
501 vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes)
507 for appid, app in allapps.items():
508 if appid in vercodes:
511 if len(apps) != len(vercodes):
514 logging.critical(_("No such package: %s") % p)
515 raise FDroidException(_("Found invalid appids in arguments"))
517 raise FDroidException(_("No packages specified"))
520 for appid, app in apps.items():
524 app.builds = [b for b in app.builds if b.versionCode in vc]
525 if len(app.builds) != len(vercodes[appid]):
527 allvcs = [b.versionCode for b in app.builds]
528 for v in vercodes[appid]:
530 logging.critical(_("No such versionCode {versionCode} for app {appid}")
531 .format(versionCode=v, appid=appid))
534 raise FDroidException(_("Found invalid versionCodes for some apps"))
539 def get_extension(filename):
540 base, ext = os.path.splitext(filename)
543 return base, ext.lower()[1:]
546 def has_extension(filename, ext):
547 _ignored, f_ext = get_extension(filename)
551 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
554 def clean_description(description):
555 'Remove unneeded newlines and spaces from a block of description text'
557 # this is split up by paragraph to make removing the newlines easier
558 for paragraph in re.split(r'\n\n', description):
559 paragraph = re.sub('\r', '', paragraph)
560 paragraph = re.sub('\n', ' ', paragraph)
561 paragraph = re.sub(' {2,}', ' ', paragraph)
562 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
563 returnstring += paragraph + '\n\n'
564 return returnstring.rstrip('\n')
567 def publishednameinfo(filename):
568 filename = os.path.basename(filename)
569 m = publish_name_regex.match(filename)
571 result = (m.group(1), m.group(2))
572 except AttributeError:
573 raise FDroidException(_("Invalid name for published file: %s") % filename)
577 apk_release_filename = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)\.apk')
578 apk_release_filename_with_sigfp = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)_(?P<sigfp>[0-9a-f]{7})\.apk')
581 def apk_parse_release_filename(apkname):
582 """Parses the name of an APK file according the F-Droids APK naming
583 scheme and returns the tokens.
585 WARNING: Returned values don't necessarily represent the APKs actual
586 properties, the are just paresed from the file name.
588 :returns: A triplet containing (appid, versionCode, signer), where appid
589 should be the package name, versionCode should be the integer
590 represion of the APKs version and signer should be the first 7 hex
591 digists of the sha256 signing key fingerprint which was used to sign
594 m = apk_release_filename_with_sigfp.match(apkname)
596 return m.group('appid'), m.group('vercode'), m.group('sigfp')
597 m = apk_release_filename.match(apkname)
599 return m.group('appid'), m.group('vercode'), None
600 return None, None, None
603 def get_release_filename(app, build):
605 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
607 return "%s_%s.apk" % (app.id, build.versionCode)
610 def get_toolsversion_logname(app, build):
611 return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
614 def getsrcname(app, build):
615 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
627 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
630 def get_build_dir(app):
631 '''get the dir that this app will be built in'''
633 if app.RepoType == 'srclib':
634 return os.path.join('build', 'srclib', app.Repo)
636 return os.path.join('build', app.id)
640 '''checkout code from VCS and return instance of vcs and the build dir'''
641 build_dir = get_build_dir(app)
643 # Set up vcs interface and make sure we have the latest code...
644 logging.debug("Getting {0} vcs interface for {1}"
645 .format(app.RepoType, app.Repo))
646 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
650 vcs = getvcs(app.RepoType, remote, build_dir)
652 return vcs, build_dir
655 def getvcs(vcstype, remote, local):
657 return vcs_git(remote, local)
658 if vcstype == 'git-svn':
659 return vcs_gitsvn(remote, local)
661 return vcs_hg(remote, local)
663 return vcs_bzr(remote, local)
664 if vcstype == 'srclib':
665 if local != os.path.join('build', 'srclib', remote):
666 raise VCSException("Error: srclib paths are hard-coded!")
667 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
669 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
670 raise VCSException("Invalid vcs type " + vcstype)
673 def getsrclibvcs(name):
674 if name not in fdroidserver.metadata.srclibs:
675 raise VCSException("Missing srclib " + name)
676 return fdroidserver.metadata.srclibs[name]['Repo Type']
681 def __init__(self, remote, local):
683 # svn, git-svn and bzr may require auth
685 if self.repotype() in ('git-svn', 'bzr'):
687 if self.repotype == 'git-svn':
688 raise VCSException("Authentication is not supported for git-svn")
689 self.username, remote = remote.split('@')
690 if ':' not in self.username:
691 raise VCSException(_("Password required with username"))
692 self.username, self.password = self.username.split(':')
696 self.clone_failed = False
697 self.refreshed = False
703 def clientversion(self):
704 versionstr = FDroidPopen(self.clientversioncmd()).output
705 return versionstr[0:versionstr.find('\n')]
707 def clientversioncmd(self):
710 def gotorevision(self, rev, refresh=True):
711 """Take the local repository to a clean version of the given
712 revision, which is specificed in the VCS's native
713 format. Beforehand, the repository can be dirty, or even
714 non-existent. If the repository does already exist locally, it
715 will be updated from the origin, but only once in the lifetime
716 of the vcs object. None is acceptable for 'rev' if you know
717 you are cloning a clean copy of the repo - otherwise it must
718 specify a valid revision.
721 if self.clone_failed:
722 raise VCSException(_("Downloading the repository already failed once, not trying again."))
724 # The .fdroidvcs-id file for a repo tells us what VCS type
725 # and remote that directory was created from, allowing us to drop it
726 # automatically if either of those things changes.
727 fdpath = os.path.join(self.local, '..',
728 '.fdroidvcs-' + os.path.basename(self.local))
729 fdpath = os.path.normpath(fdpath)
730 cdata = self.repotype() + ' ' + self.remote
733 if os.path.exists(self.local):
734 if os.path.exists(fdpath):
735 with open(fdpath, 'r') as f:
736 fsdata = f.read().strip()
741 logging.info("Repository details for %s changed - deleting" % (
745 logging.info("Repository details for %s missing - deleting" % (
748 shutil.rmtree(self.local)
752 self.refreshed = True
755 self.gotorevisionx(rev)
756 except FDroidException as e:
759 # If necessary, write the .fdroidvcs file.
760 if writeback and not self.clone_failed:
761 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
762 with open(fdpath, 'w+') as f:
768 def gotorevisionx(self, rev): # pylint: disable=unused-argument
769 """Derived classes need to implement this.
771 It's called once basic checking has been performed.
773 raise VCSException("This VCS type doesn't define gotorevisionx")
775 # Initialise and update submodules
776 def initsubmodules(self):
777 raise VCSException('Submodules not supported for this vcs type')
779 # Get a list of all known tags
781 if not self._gettags:
782 raise VCSException('gettags not supported for this vcs type')
784 for tag in self._gettags():
785 if re.match('[-A-Za-z0-9_. /]+$', tag):
789 def latesttags(self):
790 """Get a list of all the known tags, sorted from newest to oldest"""
791 raise VCSException('latesttags not supported for this vcs type')
794 """Get current commit reference (hash, revision, etc)"""
795 raise VCSException('getref not supported for this vcs type')
798 """Returns the srclib (name, path) used in setting up the current revision, or None."""
807 def clientversioncmd(self):
808 return ['git', '--version']
810 def git(self, args, envs=dict(), cwd=None, output=True):
811 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
813 While fetch/pull/clone respect the command line option flags,
814 it seems that submodule commands do not. They do seem to
815 follow whatever is in env vars, if the version of git is new
816 enough. So we just throw the kitchen sink at it to see what
819 Also, because of CVE-2017-1000117, block all SSH URLs.
822 # supported in git >= 2.3
824 '-c', 'core.askpass=/bin/true',
825 '-c', 'core.sshCommand=/bin/false',
826 '-c', 'url.https://.insteadOf=ssh://',
828 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
829 git_config.append('-c')
830 git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':')
831 git_config.append('-c')
832 git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain)
833 git_config.append('-c')
834 git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain)
836 'GIT_TERMINAL_PROMPT': '0',
837 'GIT_ASKPASS': '/bin/true',
838 'SSH_ASKPASS': '/bin/true',
839 'GIT_SSH': '/bin/false', # for git < 2.3
841 return FDroidPopen(['git', ] + git_config + args,
842 envs=envs, cwd=cwd, output=output)
845 """If the local directory exists, but is somehow not a git repository,
846 git will traverse up the directory tree until it finds one
847 that is (i.e. fdroidserver) and then we'll proceed to destroy
848 it! This is called as a safety check.
852 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
853 result = p.output.rstrip()
854 if not result.endswith(self.local):
855 raise VCSException('Repository mismatch')
857 def gotorevisionx(self, rev):
858 if not os.path.exists(self.local):
860 p = self.git(['clone', '--', self.remote, self.local])
861 if p.returncode != 0:
862 self.clone_failed = True
863 raise VCSException("Git clone failed", p.output)
867 # Discard any working tree changes
868 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
869 'git', 'reset', '--hard'], cwd=self.local, output=False)
870 if p.returncode != 0:
871 raise VCSException(_("Git reset failed"), p.output)
872 # Remove untracked files now, in case they're tracked in the target
873 # revision (it happens!)
874 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
875 'git', 'clean', '-dffx'], cwd=self.local, output=False)
876 if p.returncode != 0:
877 raise VCSException(_("Git clean failed"), p.output)
878 if not self.refreshed:
879 # Get latest commits and tags from remote
880 p = self.git(['fetch', 'origin'], cwd=self.local)
881 if p.returncode != 0:
882 raise VCSException(_("Git fetch failed"), p.output)
883 p = self.git(['fetch', '--prune', '--tags', 'origin'], output=False, cwd=self.local)
884 if p.returncode != 0:
885 raise VCSException(_("Git fetch failed"), p.output)
886 # Recreate origin/HEAD as git clone would do it, in case it disappeared
887 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
888 if p.returncode != 0:
889 lines = p.output.splitlines()
890 if 'Multiple remote HEAD branches' not in lines[0]:
891 raise VCSException(_("Git remote set-head failed"), p.output)
892 branch = lines[1].split(' ')[-1]
893 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--', branch],
894 cwd=self.local, output=False)
895 if p2.returncode != 0:
896 raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output)
897 self.refreshed = True
898 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
899 # a github repo. Most of the time this is the same as origin/master.
900 rev = rev or 'origin/HEAD'
901 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
902 if p.returncode != 0:
903 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
904 # Get rid of any uncontrolled files left behind
905 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
906 if p.returncode != 0:
907 raise VCSException(_("Git clean failed"), p.output)
909 def initsubmodules(self):
911 submfile = os.path.join(self.local, '.gitmodules')
912 if not os.path.isfile(submfile):
913 raise NoSubmodulesException(_("No git submodules available"))
915 # fix submodules not accessible without an account and public key auth
916 with open(submfile, 'r') as f:
917 lines = f.readlines()
918 with open(submfile, 'w') as f:
920 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
921 line = re.sub('git@' + domain + ':', 'https://u:p@' + domain + '/', line)
924 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
925 if p.returncode != 0:
926 raise VCSException(_("Git submodule sync failed"), p.output)
927 p = self.git(['submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
928 if p.returncode != 0:
929 raise VCSException(_("Git submodule update failed"), p.output)
933 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
934 return p.output.splitlines()
936 tag_format = re.compile(r'tag: ([^),]*)')
938 def latesttags(self):
940 p = FDroidPopen(['git', 'log', '--tags',
941 '--simplify-by-decoration', '--pretty=format:%d'],
942 cwd=self.local, output=False)
944 for line in p.output.splitlines():
945 for tag in self.tag_format.findall(line):
950 class vcs_gitsvn(vcs):
955 def clientversioncmd(self):
956 return ['git', 'svn', '--version']
959 """If the local directory exists, but is somehow not a git repository,
960 git will traverse up the directory tree until it finds one that
961 is (i.e. fdroidserver) and then we'll proceed to destory it!
962 This is called as a safety check.
965 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
966 result = p.output.rstrip()
967 if not result.endswith(self.local):
968 raise VCSException('Repository mismatch')
970 def git(self, args, envs=dict(), cwd=None, output=True):
971 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
973 AskPass is set to /bin/true to let the process try to connect
974 without a username/password.
976 The SSH command is set to /bin/false to block all SSH URLs
977 (supported in git >= 2.3). This protects against
982 '-c', 'core.askpass=/bin/true',
983 '-c', 'core.sshCommand=/bin/false',
986 'GIT_TERMINAL_PROMPT': '0',
987 'GIT_ASKPASS': '/bin/true',
988 'SSH_ASKPASS': '/bin/true',
989 'GIT_SSH': '/bin/false', # for git < 2.3
990 'SVN_SSH': '/bin/false',
992 return FDroidPopen(['git', ] + git_config + args,
993 envs=envs, cwd=cwd, output=output)
995 def gotorevisionx(self, rev):
996 if not os.path.exists(self.local):
998 gitsvn_args = ['svn', 'clone']
1000 if ';' in self.remote:
1001 remote_split = self.remote.split(';')
1002 for i in remote_split[1:]:
1003 if i.startswith('trunk='):
1004 gitsvn_args.extend(['-T', i[6:]])
1005 elif i.startswith('tags='):
1006 gitsvn_args.extend(['-t', i[5:]])
1007 elif i.startswith('branches='):
1008 gitsvn_args.extend(['-b', i[9:]])
1009 remote = remote_split[0]
1011 remote = self.remote
1013 if not remote.startswith('https://'):
1014 raise VCSException(_('HTTPS must be used with Subversion URLs!'))
1016 # git-svn sucks at certificate validation, this throws useful errors:
1018 r = requests.head(remote)
1019 r.raise_for_status()
1020 location = r.headers.get('location')
1021 if location and not location.startswith('https://'):
1022 raise VCSException(_('Invalid redirect to non-HTTPS: {before} -> {after} ')
1023 .format(before=remote, after=location))
1025 gitsvn_args.extend(['--', remote, self.local])
1026 p = self.git(gitsvn_args)
1027 if p.returncode != 0:
1028 self.clone_failed = True
1029 raise VCSException(_('git svn clone failed'), p.output)
1033 # Discard any working tree changes
1034 p = self.git(['reset', '--hard'], cwd=self.local, output=False)
1035 if p.returncode != 0:
1036 raise VCSException("Git reset failed", p.output)
1037 # Remove untracked files now, in case they're tracked in the target
1038 # revision (it happens!)
1039 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1040 if p.returncode != 0:
1041 raise VCSException("Git clean failed", p.output)
1042 if not self.refreshed:
1043 # Get new commits, branches and tags from repo
1044 p = self.git(['svn', 'fetch'], cwd=self.local, output=False)
1045 if p.returncode != 0:
1046 raise VCSException("Git svn fetch failed")
1047 p = self.git(['svn', 'rebase'], cwd=self.local, output=False)
1048 if p.returncode != 0:
1049 raise VCSException("Git svn rebase failed", p.output)
1050 self.refreshed = True
1052 rev = rev or 'master'
1054 nospaces_rev = rev.replace(' ', '%20')
1055 # Try finding a svn tag
1056 for treeish in ['origin/', '']:
1057 p = self.git(['checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
1058 if p.returncode == 0:
1060 if p.returncode != 0:
1061 # No tag found, normal svn rev translation
1062 # Translate svn rev into git format
1063 rev_split = rev.split('/')
1066 for treeish in ['origin/', '']:
1067 if len(rev_split) > 1:
1068 treeish += rev_split[0]
1069 svn_rev = rev_split[1]
1072 # if no branch is specified, then assume trunk (i.e. 'master' branch):
1076 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
1078 p = self.git(['svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
1079 git_rev = p.output.rstrip()
1081 if p.returncode == 0 and git_rev:
1084 if p.returncode != 0 or not git_rev:
1085 # Try a plain git checkout as a last resort
1086 p = self.git(['checkout', rev], cwd=self.local, output=False)
1087 if p.returncode != 0:
1088 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
1090 # Check out the git rev equivalent to the svn rev
1091 p = self.git(['checkout', git_rev], cwd=self.local, output=False)
1092 if p.returncode != 0:
1093 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
1095 # Get rid of any uncontrolled files left behind
1096 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1097 if p.returncode != 0:
1098 raise VCSException(_("Git clean failed"), p.output)
1102 for treeish in ['origin/', '']:
1103 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
1104 if os.path.isdir(d):
1105 return os.listdir(d)
1109 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
1110 if p.returncode != 0:
1112 return p.output.strip()
1120 def clientversioncmd(self):
1121 return ['hg', '--version']
1123 def gotorevisionx(self, rev):
1124 if not os.path.exists(self.local):
1125 p = FDroidPopen(['hg', 'clone', '--ssh', '/bin/false', '--', self.remote, self.local],
1127 if p.returncode != 0:
1128 self.clone_failed = True
1129 raise VCSException("Hg clone failed", p.output)
1131 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
1132 if p.returncode != 0:
1133 raise VCSException("Hg status failed", p.output)
1134 for line in p.output.splitlines():
1135 if not line.startswith('? '):
1136 raise VCSException("Unexpected output from hg status -uS: " + line)
1137 FDroidPopen(['rm', '-rf', '--', line[2:]], cwd=self.local, output=False)
1138 if not self.refreshed:
1139 p = FDroidPopen(['hg', 'pull', '--ssh', '/bin/false'], cwd=self.local, output=False)
1140 if p.returncode != 0:
1141 raise VCSException("Hg pull failed", p.output)
1142 self.refreshed = True
1144 rev = rev or 'default'
1147 p = FDroidPopen(['hg', 'update', '-C', '--', rev], cwd=self.local, output=False)
1148 if p.returncode != 0:
1149 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
1150 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1151 # Also delete untracked files, we have to enable purge extension for that:
1152 if "'purge' is provided by the following extension" in p.output:
1153 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
1154 myfile.write("\n[extensions]\nhgext.purge=\n")
1155 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1156 if p.returncode != 0:
1157 raise VCSException("HG purge failed", p.output)
1158 elif p.returncode != 0:
1159 raise VCSException("HG purge failed", p.output)
1162 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
1163 return p.output.splitlines()[1:]
1171 def clientversioncmd(self):
1172 return ['bzr', '--version']
1174 def bzr(self, args, envs=dict(), cwd=None, output=True):
1175 '''Prevent bzr from ever using SSH to avoid security vulns'''
1179 return FDroidPopen(['bzr', ] + args, envs=envs, cwd=cwd, output=output)
1181 def gotorevisionx(self, rev):
1182 if not os.path.exists(self.local):
1183 p = self.bzr(['branch', self.remote, self.local], output=False)
1184 if p.returncode != 0:
1185 self.clone_failed = True
1186 raise VCSException("Bzr branch failed", p.output)
1188 p = self.bzr(['clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
1189 if p.returncode != 0:
1190 raise VCSException("Bzr revert failed", p.output)
1191 if not self.refreshed:
1192 p = self.bzr(['pull'], cwd=self.local, output=False)
1193 if p.returncode != 0:
1194 raise VCSException("Bzr update failed", p.output)
1195 self.refreshed = True
1197 revargs = list(['-r', rev] if rev else [])
1198 p = self.bzr(['revert'] + revargs, cwd=self.local, output=False)
1199 if p.returncode != 0:
1200 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1203 p = self.bzr(['tags'], cwd=self.local, output=False)
1204 return [tag.split(' ')[0].strip() for tag in
1205 p.output.splitlines()]
1208 def unescape_string(string):
1211 if string[0] == '"' and string[-1] == '"':
1214 return string.replace("\\'", "'")
1217 def retrieve_string(app_dir, string, xmlfiles=None):
1219 if not string.startswith('@string/'):
1220 return unescape_string(string)
1222 if xmlfiles is None:
1225 os.path.join(app_dir, 'res'),
1226 os.path.join(app_dir, 'src', 'main', 'res'),
1228 for root, dirs, files in os.walk(res_dir):
1229 if os.path.basename(root) == 'values':
1230 xmlfiles += [os.path.join(root, x) for x in files if x.endswith('.xml')]
1232 name = string[len('@string/'):]
1234 def element_content(element):
1235 if element.text is None:
1237 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1238 return s.decode('utf-8').strip()
1240 for path in xmlfiles:
1241 if not os.path.isfile(path):
1243 xml = parse_xml(path)
1244 element = xml.find('string[@name="' + name + '"]')
1245 if element is not None:
1246 content = element_content(element)
1247 return retrieve_string(app_dir, content, xmlfiles)
1252 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1253 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1256 def manifest_paths(app_dir, flavours):
1257 '''Return list of existing files that will be used to find the highest vercode'''
1259 possible_manifests = \
1260 [os.path.join(app_dir, 'AndroidManifest.xml'),
1261 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1262 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1263 os.path.join(app_dir, 'build.gradle')]
1265 for flavour in flavours:
1266 if flavour == 'yes':
1268 possible_manifests.append(
1269 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1271 return [path for path in possible_manifests if os.path.isfile(path)]
1274 def fetch_real_name(app_dir, flavours):
1275 '''Retrieve the package name. Returns the name, or None if not found.'''
1276 for path in manifest_paths(app_dir, flavours):
1277 if not has_extension(path, 'xml') or not os.path.isfile(path):
1279 logging.debug("fetch_real_name: Checking manifest at " + path)
1280 xml = parse_xml(path)
1281 app = xml.find('application')
1284 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1286 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1287 result = retrieve_string_singleline(app_dir, label)
1289 result = result.strip()
1294 def get_library_references(root_dir):
1296 proppath = os.path.join(root_dir, 'project.properties')
1297 if not os.path.isfile(proppath):
1299 with open(proppath, 'r', encoding='iso-8859-1') as f:
1301 if not line.startswith('android.library.reference.'):
1303 path = line.split('=')[1].strip()
1304 relpath = os.path.join(root_dir, path)
1305 if not os.path.isdir(relpath):
1307 logging.debug("Found subproject at %s" % path)
1308 libraries.append(path)
1312 def ant_subprojects(root_dir):
1313 subprojects = get_library_references(root_dir)
1314 for subpath in subprojects:
1315 subrelpath = os.path.join(root_dir, subpath)
1316 for p in get_library_references(subrelpath):
1317 relp = os.path.normpath(os.path.join(subpath, p))
1318 if relp not in subprojects:
1319 subprojects.insert(0, relp)
1323 def remove_debuggable_flags(root_dir):
1324 # Remove forced debuggable flags
1325 logging.debug("Removing debuggable flags from %s" % root_dir)
1326 for root, dirs, files in os.walk(root_dir):
1327 if 'AndroidManifest.xml' in files and os.path.isfile(os.path.join(root, 'AndroidManifest.xml')):
1328 regsub_file(r'android:debuggable="[^"]*"',
1330 os.path.join(root, 'AndroidManifest.xml'))
1333 vcsearch_g = re.compile(r'''.*[Vv]ersionCode\s*=?\s*["']*([0-9]+)["']*''').search
1334 vnsearch_g = re.compile(r'''.*[Vv]ersionName\s*=?\s*(["'])((?:(?=(\\?))\3.)*?)\1.*''').search
1335 vnssearch_g = re.compile(r'''.*[Vv]ersionNameSuffix\s*=?\s*(["'])((?:(?=(\\?))\3.)*?)\1.*''').search
1336 psearch_g = re.compile(r'''.*(packageName|applicationId)\s*=*\s*["']([^"']+)["'].*''').search
1337 fsearch_g = re.compile(r'''.*(applicationIdSuffix)\s*=*\s*["']([^"']+)["'].*''').search
1340 def app_matches_packagename(app, package):
1343 appid = app.UpdateCheckName or app.id
1344 if appid is None or appid == "Ignore":
1346 return appid == package
1349 def parse_androidmanifests(paths, app):
1351 Extract some information from the AndroidManifest.xml at the given path.
1352 Returns (version, vercode, package), any or all of which might be None.
1353 All values returned are strings.
1356 ignoreversions = app.UpdateCheckIgnore
1357 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1360 return (None, None, None)
1368 if not os.path.isfile(path):
1371 logging.debug(_("Parsing manifest at '{path}'").format(path=path))
1378 temp_version_name = None
1379 if app.builds and 'gradle' in app.builds[-1] and app.builds[-1].gradle:
1380 flavour = app.builds[-1].gradle[-1]
1382 if has_extension(path, 'gradle'):
1383 with open(path, 'r') as f:
1384 inside_flavour_group = 0
1385 inside_required_flavour = 0
1387 if gradle_comment.match(line):
1390 if "applicationId" in line and not temp_app_id:
1391 matches = psearch_g(line)
1393 temp_app_id = matches.group(2)
1395 if "versionName" in line and not temp_version_name:
1396 matches = vnsearch_g(line)
1398 temp_version_name = matches.group(2)
1400 if inside_flavour_group > 0:
1401 if inside_required_flavour > 0:
1402 matches = psearch_g(line)
1404 s = matches.group(2)
1405 if app_matches_packagename(app, s):
1408 # If build.gradle contains applicationIdSuffix add it to the end of package name
1409 matches = fsearch_g(line)
1410 if matches and temp_app_id:
1411 suffix = matches.group(2)
1412 temp_app_id = temp_app_id + suffix
1413 if app_matches_packagename(app, temp_app_id):
1414 package = temp_app_id
1416 matches = vnsearch_g(line)
1418 version = matches.group(2)
1420 # If build.gradle contains applicationNameSuffix add it to the end of version name
1421 matches = vnssearch_g(line)
1422 if matches and temp_version_name:
1423 name_suffix = matches.group(2)
1424 version = temp_version_name + name_suffix
1426 matches = vcsearch_g(line)
1428 vercode = matches.group(1)
1431 inside_required_flavour += 1
1433 inside_required_flavour -= 1
1435 if flavour and (flavour in line):
1436 inside_required_flavour = 1
1439 inside_flavour_group += 1
1441 inside_flavour_group -= 1
1443 if "productFlavors" in line:
1444 inside_flavour_group = 1
1446 matches = psearch_g(line)
1448 s = matches.group(2)
1449 if app_matches_packagename(app, s):
1452 matches = vnsearch_g(line)
1454 version = matches.group(2)
1456 matches = vcsearch_g(line)
1458 vercode = matches.group(1)
1461 xml = parse_xml(path)
1462 if "package" in xml.attrib:
1463 s = xml.attrib["package"]
1464 if app_matches_packagename(app, s):
1466 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1467 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1468 base_dir = os.path.dirname(path)
1469 version = retrieve_string_singleline(base_dir, version)
1470 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1471 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1472 if string_is_integer(a):
1475 logging.warning(_("Problem with xml at '{path}'").format(path=path))
1477 # Remember package name, may be defined separately from version+vercode
1479 package = max_package
1481 logging.debug("..got package={0}, version={1}, vercode={2}"
1482 .format(package, version, vercode))
1484 # Always grab the package name and version name in case they are not
1485 # together with the highest version code
1486 if max_package is None and package is not None:
1487 max_package = package
1488 if max_version is None and version is not None:
1489 max_version = version
1491 if vercode is not None \
1492 and (max_vercode is None or vercode > max_vercode):
1493 if not ignoresearch or not ignoresearch(version):
1494 if version is not None:
1495 max_version = version
1496 if vercode is not None:
1497 max_vercode = vercode
1498 if package is not None:
1499 max_package = package
1501 max_version = "Ignore"
1503 if max_version is None:
1504 max_version = "Unknown"
1506 if max_package and not is_valid_package_name(max_package):
1507 raise FDroidException(_("Invalid package name {0}").format(max_package))
1509 return (max_version, max_vercode, max_package)
1512 def is_valid_package_name(name):
1513 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1516 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1517 raw=False, prepare=True, preponly=False, refresh=True,
1519 """Get the specified source library.
1521 Returns the path to it. Normally this is the path to be used when
1522 referencing it, which may be a subdirectory of the actual project. If
1523 you want the base directory of the project, pass 'basepath=True'.
1532 name, ref = spec.split('@')
1534 number, name = name.split(':', 1)
1536 name, subdir = name.split('/', 1)
1538 if name not in fdroidserver.metadata.srclibs:
1539 raise VCSException('srclib ' + name + ' not found.')
1541 srclib = fdroidserver.metadata.srclibs[name]
1543 sdir = os.path.join(srclib_dir, name)
1546 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1547 vcs.srclib = (name, number, sdir)
1549 vcs.gotorevision(ref, refresh)
1556 libdir = os.path.join(sdir, subdir)
1557 elif srclib["Subdir"]:
1558 for subdir in srclib["Subdir"]:
1559 libdir_candidate = os.path.join(sdir, subdir)
1560 if os.path.exists(libdir_candidate):
1561 libdir = libdir_candidate
1567 remove_signing_keys(sdir)
1568 remove_debuggable_flags(sdir)
1572 if srclib["Prepare"]:
1573 cmd = replace_config_vars(srclib["Prepare"], build)
1575 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=libdir)
1576 if p.returncode != 0:
1577 raise BuildException("Error running prepare command for srclib %s"
1583 return (name, number, libdir)
1586 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1589 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1590 """ Prepare the source code for a particular build
1592 :param vcs: the appropriate vcs object for the application
1593 :param app: the application details from the metadata
1594 :param build: the build details from the metadata
1595 :param build_dir: the path to the build directory, usually 'build/app.id'
1596 :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
1597 :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
1599 Returns the (root, srclibpaths) where:
1600 :param root: is the root directory, which may be the same as 'build_dir' or may
1601 be a subdirectory of it.
1602 :param srclibpaths: is information on the srclibs being used
1605 # Optionally, the actual app source can be in a subdirectory
1607 root_dir = os.path.join(build_dir, build.subdir)
1609 root_dir = build_dir
1611 # Get a working copy of the right revision
1612 logging.info("Getting source for revision " + build.commit)
1613 vcs.gotorevision(build.commit, refresh)
1615 # Initialise submodules if required
1616 if build.submodules:
1617 logging.info(_("Initialising submodules"))
1618 vcs.initsubmodules()
1620 # Check that a subdir (if we're using one) exists. This has to happen
1621 # after the checkout, since it might not exist elsewhere
1622 if not os.path.exists(root_dir):
1623 raise BuildException('Missing subdir ' + root_dir)
1625 # Run an init command if one is required
1627 cmd = replace_config_vars(build.init, build)
1628 logging.info("Running 'init' commands in %s" % root_dir)
1630 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir)
1631 if p.returncode != 0:
1632 raise BuildException("Error running init command for %s:%s" %
1633 (app.id, build.versionName), p.output)
1635 # Apply patches if any
1637 logging.info("Applying patches")
1638 for patch in build.patch:
1639 patch = patch.strip()
1640 logging.info("Applying " + patch)
1641 patch_path = os.path.join('metadata', app.id, patch)
1642 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1643 if p.returncode != 0:
1644 raise BuildException("Failed to apply patch %s" % patch_path)
1646 # Get required source libraries
1649 logging.info("Collecting source libraries")
1650 for lib in build.srclibs:
1651 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1652 refresh=refresh, build=build))
1654 for name, number, libpath in srclibpaths:
1655 place_srclib(root_dir, int(number) if number else None, libpath)
1657 basesrclib = vcs.getsrclib()
1658 # If one was used for the main source, add that too.
1660 srclibpaths.append(basesrclib)
1662 # Update the local.properties file
1663 localprops = [os.path.join(build_dir, 'local.properties')]
1665 parts = build.subdir.split(os.sep)
1668 cur = os.path.join(cur, d)
1669 localprops += [os.path.join(cur, 'local.properties')]
1670 for path in localprops:
1672 if os.path.isfile(path):
1673 logging.info("Updating local.properties file at %s" % path)
1674 with open(path, 'r', encoding='iso-8859-1') as f:
1678 logging.info("Creating local.properties file at %s" % path)
1679 # Fix old-fashioned 'sdk-location' by copying
1680 # from sdk.dir, if necessary
1682 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1683 re.S | re.M).group(1)
1684 props += "sdk-location=%s\n" % sdkloc
1686 props += "sdk.dir=%s\n" % config['sdk_path']
1687 props += "sdk-location=%s\n" % config['sdk_path']
1688 ndk_path = build.ndk_path()
1689 # if for any reason the path isn't valid or the directory
1690 # doesn't exist, some versions of Gradle will error with a
1691 # cryptic message (even if the NDK is not even necessary).
1692 # https://gitlab.com/fdroid/fdroidserver/issues/171
1693 if ndk_path and os.path.exists(ndk_path):
1695 props += "ndk.dir=%s\n" % ndk_path
1696 props += "ndk-location=%s\n" % ndk_path
1697 # Add java.encoding if necessary
1699 props += "java.encoding=%s\n" % build.encoding
1700 with open(path, 'w', encoding='iso-8859-1') as f:
1704 if build.build_method() == 'gradle':
1705 flavours = build.gradle
1708 n = build.target.split('-')[1]
1709 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1710 r'compileSdkVersion %s' % n,
1711 os.path.join(root_dir, 'build.gradle'))
1713 # Remove forced debuggable flags
1714 remove_debuggable_flags(root_dir)
1716 # Insert version code and number into the manifest if necessary
1717 if build.forceversion:
1718 logging.info("Changing the version name")
1719 for path in manifest_paths(root_dir, flavours):
1720 if not os.path.isfile(path):
1722 if has_extension(path, 'xml'):
1723 regsub_file(r'android:versionName="[^"]*"',
1724 r'android:versionName="%s"' % build.versionName,
1726 elif has_extension(path, 'gradle'):
1727 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1728 r"""\1versionName '%s'""" % build.versionName,
1731 if build.forcevercode:
1732 logging.info("Changing the version code")
1733 for path in manifest_paths(root_dir, flavours):
1734 if not os.path.isfile(path):
1736 if has_extension(path, 'xml'):
1737 regsub_file(r'android:versionCode="[^"]*"',
1738 r'android:versionCode="%s"' % build.versionCode,
1740 elif has_extension(path, 'gradle'):
1741 regsub_file(r'versionCode[ =]+[0-9]+',
1742 r'versionCode %s' % build.versionCode,
1745 # Delete unwanted files
1747 logging.info(_("Removing specified files"))
1748 for part in getpaths(build_dir, build.rm):
1749 dest = os.path.join(build_dir, part)
1750 logging.info("Removing {0}".format(part))
1751 if os.path.lexists(dest):
1752 # rmtree can only handle directories that are not symlinks, so catch anything else
1753 if not os.path.isdir(dest) or os.path.islink(dest):
1758 logging.info("...but it didn't exist")
1760 remove_signing_keys(build_dir)
1762 # Add required external libraries
1764 logging.info("Collecting prebuilt libraries")
1765 libsdir = os.path.join(root_dir, 'libs')
1766 if not os.path.exists(libsdir):
1768 for lib in build.extlibs:
1770 logging.info("...installing extlib {0}".format(lib))
1771 libf = os.path.basename(lib)
1772 libsrc = os.path.join(extlib_dir, lib)
1773 if not os.path.exists(libsrc):
1774 raise BuildException("Missing extlib file {0}".format(libsrc))
1775 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1777 # Run a pre-build command if one is required
1779 logging.info("Running 'prebuild' commands in %s" % root_dir)
1781 cmd = replace_config_vars(build.prebuild, build)
1783 # Substitute source library paths into prebuild commands
1784 for name, number, libpath in srclibpaths:
1785 libpath = os.path.relpath(libpath, root_dir)
1786 cmd = cmd.replace('$$' + name + '$$', libpath)
1788 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir)
1789 if p.returncode != 0:
1790 raise BuildException("Error running prebuild command for %s:%s" %
1791 (app.id, build.versionName), p.output)
1793 # Generate (or update) the ant build file, build.xml...
1794 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1795 parms = ['android', 'update', 'lib-project']
1796 lparms = ['android', 'update', 'project']
1799 parms += ['-t', build.target]
1800 lparms += ['-t', build.target]
1801 if build.androidupdate:
1802 update_dirs = build.androidupdate
1804 update_dirs = ant_subprojects(root_dir) + ['.']
1806 for d in update_dirs:
1807 subdir = os.path.join(root_dir, d)
1809 logging.debug("Updating main project")
1810 cmd = parms + ['-p', d]
1812 logging.debug("Updating subproject %s" % d)
1813 cmd = lparms + ['-p', d]
1814 p = SdkToolsPopen(cmd, cwd=root_dir)
1815 # Check to see whether an error was returned without a proper exit
1816 # code (this is the case for the 'no target set or target invalid'
1818 if p.returncode != 0 or p.output.startswith("Error: "):
1819 raise BuildException("Failed to update project at %s" % d, p.output)
1820 # Clean update dirs via ant
1822 logging.info("Cleaning subproject %s" % d)
1823 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1825 return (root_dir, srclibpaths)
1828 def getpaths_map(build_dir, globpaths):
1829 """Extend via globbing the paths from a field and return them as a map from original path to resulting paths"""
1833 full_path = os.path.join(build_dir, p)
1834 full_path = os.path.normpath(full_path)
1835 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1837 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1841 def getpaths(build_dir, globpaths):
1842 """Extend via globbing the paths from a field and return them as a set"""
1843 paths_map = getpaths_map(build_dir, globpaths)
1845 for k, v in paths_map.items():
1852 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1855 def check_system_clock(dt_obj, path):
1856 """Check if system clock is updated based on provided date
1858 If an APK has files newer than the system time, suggest updating
1859 the system clock. This is useful for offline systems, used for
1860 signing, which do not have another source of clock sync info. It
1861 has to be more than 24 hours newer because ZIP/APK files do not
1865 checkdt = dt_obj - timedelta(1)
1866 if datetime.today() < checkdt:
1867 logging.warning(_('System clock is older than date in {path}!').format(path=path)
1868 + '\n' + _('Set clock to that time using:') + '\n'
1869 + 'sudo date -s "' + str(dt_obj) + '"')
1873 """permanent store of existing APKs with the date they were added
1875 This is currently the only way to permanently store the "updated"
1880 '''Load filename/date info about previously seen APKs
1882 Since the appid and date strings both will never have spaces,
1883 this is parsed as a list from the end to allow the filename to
1884 have any combo of spaces.
1887 self.path = os.path.join('stats', 'known_apks.txt')
1889 if os.path.isfile(self.path):
1890 with open(self.path, 'r', encoding='utf8') as f:
1892 t = line.rstrip().split(' ')
1894 self.apks[t[0]] = (t[1], None)
1897 date = datetime.strptime(t[-1], '%Y-%m-%d')
1898 filename = line[0:line.rfind(appid) - 1]
1899 self.apks[filename] = (appid, date)
1900 check_system_clock(date, self.path)
1901 self.changed = False
1903 def writeifchanged(self):
1904 if not self.changed:
1907 if not os.path.exists('stats'):
1911 for apk, app in self.apks.items():
1913 line = apk + ' ' + appid
1915 line += ' ' + added.strftime('%Y-%m-%d')
1918 with open(self.path, 'w', encoding='utf8') as f:
1919 for line in sorted(lst, key=natural_key):
1920 f.write(line + '\n')
1922 def recordapk(self, apkName, app, default_date=None):
1924 Record an apk (if it's new, otherwise does nothing)
1925 Returns the date it was added as a datetime instance
1927 if apkName not in self.apks:
1928 if default_date is None:
1929 default_date = datetime.utcnow()
1930 self.apks[apkName] = (app, default_date)
1932 _ignored, added = self.apks[apkName]
1935 def getapp(self, apkname):
1936 """Look up information - given the 'apkname', returns (app id, date added/None).
1938 Or returns None for an unknown apk.
1940 if apkname in self.apks:
1941 return self.apks[apkname]
1944 def getlatest(self, num):
1945 """Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first"""
1947 for apk, app in self.apks.items():
1951 if apps[appid] > added:
1955 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1956 lst = [app for app, _ignored in sortedapps]
1961 def get_file_extension(filename):
1962 """get the normalized file extension, can be blank string but never None"""
1963 if isinstance(filename, bytes):
1964 filename = filename.decode('utf-8')
1965 return os.path.splitext(filename)[1].lower()[1:]
1968 def use_androguard():
1969 """Report if androguard is available, and config its debug logging"""
1973 if use_androguard.show_path:
1974 logging.debug(_('Using androguard from "{path}"').format(path=androguard.__file__))
1975 use_androguard.show_path = False
1976 if options and options.verbose:
1977 logging.getLogger("androguard.axml").setLevel(logging.INFO)
1983 use_androguard.show_path = True
1986 def _get_androguard_APK(apkfile):
1988 from androguard.core.bytecodes.apk import APK
1990 raise FDroidException("androguard library is not installed and aapt not present")
1995 def ensure_final_value(packageName, arsc, value):
1996 """Ensure incoming value is always the value, not the resid
1998 androguard will sometimes return the Android "resId" aka
1999 Resource ID instead of the actual value. This checks whether
2000 the value is actually a resId, then performs the Android
2001 Resource lookup as needed.
2007 try: # can be a literal value or a resId
2008 res_id = int('0x' + value[1:], 16)
2009 res_id = arsc.get_id(packageName, res_id)[1]
2010 returnValue = arsc.get_string(packageName, res_id)[1]
2016 def is_apk_and_debuggable_aapt(apkfile):
2017 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
2019 if p.returncode != 0:
2020 raise FDroidException(_("Failed to get APK manifest information"))
2021 for line in p.output.splitlines():
2022 if 'android:debuggable' in line and not line.endswith('0x0'):
2027 def is_apk_and_debuggable_androguard(apkfile):
2028 apkobject = _get_androguard_APK(apkfile)
2029 if apkobject.is_valid_APK():
2030 debuggable = apkobject.get_element("application", "debuggable")
2031 if debuggable is not None:
2032 return bool(strtobool(debuggable))
2036 def is_apk_and_debuggable(apkfile):
2037 """Returns True if the given file is an APK and is debuggable
2039 :param apkfile: full path to the apk to check"""
2041 if get_file_extension(apkfile) != 'apk':
2044 if use_androguard():
2045 return is_apk_and_debuggable_androguard(apkfile)
2047 return is_apk_and_debuggable_aapt(apkfile)
2050 def get_apk_id(apkfile):
2051 """Extract identification information from APK using aapt.
2053 :param apkfile: path to an APK file.
2054 :returns: triplet (appid, version code, version name)
2056 if use_androguard():
2057 return get_apk_id_androguard(apkfile)
2059 return get_apk_id_aapt(apkfile)
2062 def get_apk_id_androguard(apkfile):
2063 if not os.path.exists(apkfile):
2064 raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'")
2065 .format(apkfilename=apkfile))
2066 a = _get_androguard_APK(apkfile)
2067 versionName = ensure_final_value(a.package, a.get_android_resources(), a.get_androidversion_name())
2069 versionName = '' # versionName is expected to always be a str
2070 return a.package, a.get_androidversion_code(), versionName
2073 def get_apk_id_aapt(apkfile):
2074 r = re.compile("^package: name='(?P<appid>.*)' versionCode='(?P<vercode>.*)' versionName='(?P<vername>.*?)'(?: platformBuildVersionName='.*')?")
2075 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
2076 for line in p.output.splitlines():
2079 return m.group('appid'), m.group('vercode'), m.group('vername')
2080 raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'")
2081 .format(apkfilename=apkfile))
2084 def get_minSdkVersion_aapt(apkfile):
2085 """Extract the minimum supported Android SDK from an APK using aapt
2087 :param apkfile: path to an APK file.
2088 :returns: the integer representing the SDK version
2090 r = re.compile(r"^sdkVersion:'([0-9]+)'")
2091 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
2092 for line in p.output.splitlines():
2095 return int(m.group(1))
2096 raise FDroidException(_('Reading minSdkVersion failed: "{apkfilename}"')
2097 .format(apkfilename=apkfile))
2102 self.returncode = None
2106 def SdkToolsPopen(commands, cwd=None, output=True):
2108 if cmd not in config:
2109 config[cmd] = find_sdk_tools_cmd(commands[0])
2110 abscmd = config[cmd]
2112 raise FDroidException(_("Could not find '{command}' on your system").format(command=cmd))
2114 test_aapt_version(config['aapt'])
2115 return FDroidPopen([abscmd] + commands[1:],
2116 cwd=cwd, output=output)
2119 def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2121 Run a command and capture the possibly huge output as bytes.
2123 :param commands: command and argument list like in subprocess.Popen
2124 :param cwd: optionally specifies a working directory
2125 :param envs: a optional dictionary of environment variables and their values
2126 :returns: A PopenResult.
2131 set_FDroidPopen_env()
2133 process_env = env.copy()
2134 if envs is not None and len(envs) > 0:
2135 process_env.update(envs)
2138 cwd = os.path.normpath(cwd)
2139 logging.debug("Directory: %s" % cwd)
2140 logging.debug("> %s" % ' '.join(commands))
2142 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
2143 result = PopenResult()
2146 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
2147 stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
2148 stderr=stderr_param)
2149 except OSError as e:
2150 raise BuildException("OSError while trying to execute " +
2151 ' '.join(commands) + ': ' + str(e))
2153 # TODO are these AsynchronousFileReader threads always exiting?
2154 if not stderr_to_stdout and options.verbose:
2155 stderr_queue = Queue()
2156 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
2158 while not stderr_reader.eof():
2159 while not stderr_queue.empty():
2160 line = stderr_queue.get()
2161 sys.stderr.buffer.write(line)
2166 stdout_queue = Queue()
2167 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
2170 # Check the queue for output (until there is no more to get)
2171 while not stdout_reader.eof():
2172 while not stdout_queue.empty():
2173 line = stdout_queue.get()
2174 if output and options.verbose:
2175 # Output directly to console
2176 sys.stderr.buffer.write(line)
2182 result.returncode = p.wait()
2183 result.output = buf.getvalue()
2185 # make sure all filestreams of the subprocess are closed
2186 for streamvar in ['stdin', 'stdout', 'stderr']:
2187 if hasattr(p, streamvar):
2188 stream = getattr(p, streamvar)
2194 def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2196 Run a command and capture the possibly huge output as a str.
2198 :param commands: command and argument list like in subprocess.Popen
2199 :param cwd: optionally specifies a working directory
2200 :param envs: a optional dictionary of environment variables and their values
2201 :returns: A PopenResult.
2203 result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
2204 result.output = result.output.decode('utf-8', 'ignore')
2208 gradle_comment = re.compile(r'[ ]*//')
2209 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
2210 gradle_line_matches = [
2211 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
2212 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
2213 re.compile(r'.*\.readLine\(.*'),
2217 def remove_signing_keys(build_dir):
2218 for root, dirs, files in os.walk(build_dir):
2219 if 'build.gradle' in files:
2220 path = os.path.join(root, 'build.gradle')
2222 with open(path, "r", encoding='utf8') as o:
2223 lines = o.readlines()
2229 with open(path, "w", encoding='utf8') as o:
2230 while i < len(lines):
2233 while line.endswith('\\\n'):
2234 line = line.rstrip('\\\n') + lines[i]
2237 if gradle_comment.match(line):
2242 opened += line.count('{')
2243 opened -= line.count('}')
2246 if gradle_signing_configs.match(line):
2251 if any(s.match(line) for s in gradle_line_matches):
2259 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
2262 'project.properties',
2264 'default.properties',
2265 'ant.properties', ]:
2266 if propfile in files:
2267 path = os.path.join(root, propfile)
2269 with open(path, "r", encoding='iso-8859-1') as o:
2270 lines = o.readlines()
2274 with open(path, "w", encoding='iso-8859-1') as o:
2276 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
2283 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
2286 def set_FDroidPopen_env(build=None):
2288 set up the environment variables for the build environment
2290 There is only a weak standard, the variables used by gradle, so also set
2291 up the most commonly used environment variables for SDK and NDK. Also, if
2292 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
2294 global env, orig_path
2298 orig_path = env['PATH']
2299 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
2300 env[n] = config['sdk_path']
2301 for k, v in config['java_paths'].items():
2302 env['JAVA%s_HOME' % k] = v
2304 missinglocale = True
2305 for k, v in env.items():
2306 if k == 'LANG' and v != 'C':
2307 missinglocale = False
2309 missinglocale = False
2311 env['LANG'] = 'en_US.UTF-8'
2313 if build is not None:
2314 path = build.ndk_path()
2315 paths = orig_path.split(os.pathsep)
2316 if path not in paths:
2317 paths = [path] + paths
2318 env['PATH'] = os.pathsep.join(paths)
2319 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
2320 env[n] = build.ndk_path()
2323 def replace_build_vars(cmd, build):
2324 cmd = cmd.replace('$$COMMIT$$', build.commit)
2325 cmd = cmd.replace('$$VERSION$$', build.versionName)
2326 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
2330 def replace_config_vars(cmd, build):
2331 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
2332 cmd = cmd.replace('$$NDK$$', build.ndk_path())
2333 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
2334 if build is not None:
2335 cmd = replace_build_vars(cmd, build)
2339 def place_srclib(root_dir, number, libpath):
2342 relpath = os.path.relpath(libpath, root_dir)
2343 proppath = os.path.join(root_dir, 'project.properties')
2346 if os.path.isfile(proppath):
2347 with open(proppath, "r", encoding='iso-8859-1') as o:
2348 lines = o.readlines()
2350 with open(proppath, "w", encoding='iso-8859-1') as o:
2353 if line.startswith('android.library.reference.%d=' % number):
2354 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2359 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2362 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z_\-]+\.(SF|RSA|DSA|EC)')
2365 def signer_fingerprint_short(sig):
2366 """Obtain shortened sha256 signing-key fingerprint for pkcs7 signature.
2368 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2369 for a given pkcs7 signature.
2371 :param sig: Contents of an APK signing certificate.
2372 :returns: shortened signing-key fingerprint.
2374 return signer_fingerprint(sig)[:7]
2377 def signer_fingerprint(sig):
2378 """Obtain sha256 signing-key fingerprint for pkcs7 signature.
2380 Extracts hexadecimal sha256 signing-key fingerprint string
2381 for a given pkcs7 signature.
2383 :param: Contents of an APK signature.
2384 :returns: shortened signature fingerprint.
2386 cert_encoded = get_certificate(sig)
2387 return hashlib.sha256(cert_encoded).hexdigest()
2390 def apk_signer_fingerprint(apk_path):
2391 """Obtain sha256 signing-key fingerprint for APK.
2393 Extracts hexadecimal sha256 signing-key fingerprint string
2396 :param apkpath: path to APK
2397 :returns: signature fingerprint
2400 with zipfile.ZipFile(apk_path, 'r') as apk:
2401 certs = [n for n in apk.namelist() if CERT_PATH_REGEX.match(n)]
2404 logging.error("Found no signing certificates on %s" % apk_path)
2407 logging.error("Found multiple signing certificates on %s" % apk_path)
2410 cert = apk.read(certs[0])
2411 return signer_fingerprint(cert)
2414 def apk_signer_fingerprint_short(apk_path):
2415 """Obtain shortened sha256 signing-key fingerprint for APK.
2417 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2418 for a given pkcs7 APK.
2420 :param apk_path: path to APK
2421 :returns: shortened signing-key fingerprint
2423 return apk_signer_fingerprint(apk_path)[:7]
2426 def metadata_get_sigdir(appid, vercode=None):
2427 """Get signature directory for app"""
2429 return os.path.join('metadata', appid, 'signatures', vercode)
2431 return os.path.join('metadata', appid, 'signatures')
2434 def metadata_find_developer_signature(appid, vercode=None):
2435 """Tires to find the developer signature for given appid.
2437 This picks the first signature file found in metadata an returns its
2440 :returns: sha256 signing key fingerprint of the developer signing key.
2441 None in case no signature can not be found."""
2443 # fetch list of dirs for all versions of signatures
2446 appversigdirs.append(metadata_get_sigdir(appid, vercode))
2448 appsigdir = metadata_get_sigdir(appid)
2449 if os.path.isdir(appsigdir):
2450 numre = re.compile('[0-9]+')
2451 for ver in os.listdir(appsigdir):
2452 if numre.match(ver):
2453 appversigdir = os.path.join(appsigdir, ver)
2454 appversigdirs.append(appversigdir)
2456 for sigdir in appversigdirs:
2457 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2458 glob.glob(os.path.join(sigdir, '*.EC')) + \
2459 glob.glob(os.path.join(sigdir, '*.RSA'))
2461 raise FDroidException('ambiguous signatures, please make sure there is only one signature in \'{}\'. (The signature has to be the App maintainers signature for version of the APK.)'.format(sigdir))
2463 with open(sig, 'rb') as f:
2464 return signer_fingerprint(f.read())
2468 def metadata_find_signing_files(appid, vercode):
2469 """Gets a list of singed manifests and signatures.
2471 :param appid: app id string
2472 :param vercode: app version code
2473 :returns: a list of triplets for each signing key with following paths:
2474 (signature_file, singed_file, manifest_file)
2477 sigdir = metadata_get_sigdir(appid, vercode)
2478 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2479 glob.glob(os.path.join(sigdir, '*.EC')) + \
2480 glob.glob(os.path.join(sigdir, '*.RSA'))
2481 extre = re.compile('(\.DSA|\.EC|\.RSA)$')
2483 sf = extre.sub('.SF', sig)
2484 if os.path.isfile(sf):
2485 mf = os.path.join(sigdir, 'MANIFEST.MF')
2486 if os.path.isfile(mf):
2487 ret.append((sig, sf, mf))
2491 def metadata_find_developer_signing_files(appid, vercode):
2492 """Get developer signature files for specified app from metadata.
2494 :returns: A triplet of paths for signing files from metadata:
2495 (signature_file, singed_file, manifest_file)
2497 allsigningfiles = metadata_find_signing_files(appid, vercode)
2498 if allsigningfiles and len(allsigningfiles) == 1:
2499 return allsigningfiles[0]
2504 def apk_strip_signatures(signed_apk, strip_manifest=False):
2505 """Removes signatures from APK.
2507 :param signed_apk: path to apk file.
2508 :param strip_manifest: when set to True also the manifest file will
2509 be removed from the APK.
2511 with tempfile.TemporaryDirectory() as tmpdir:
2512 tmp_apk = os.path.join(tmpdir, 'tmp.apk')
2513 shutil.move(signed_apk, tmp_apk)
2514 with ZipFile(tmp_apk, 'r') as in_apk:
2515 with ZipFile(signed_apk, 'w') as out_apk:
2516 for info in in_apk.infolist():
2517 if not apk_sigfile.match(info.filename):
2519 if info.filename != 'META-INF/MANIFEST.MF':
2520 buf = in_apk.read(info.filename)
2521 out_apk.writestr(info, buf)
2523 buf = in_apk.read(info.filename)
2524 out_apk.writestr(info, buf)
2527 def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest):
2528 """Implats a signature from metadata into an APK.
2530 Note: this changes there supplied APK in place. So copy it if you
2531 need the original to be preserved.
2533 :param apkpath: location of the apk
2535 # get list of available signature files in metadata
2536 with tempfile.TemporaryDirectory() as tmpdir:
2537 apkwithnewsig = os.path.join(tmpdir, 'newsig.apk')
2538 with ZipFile(apkpath, 'r') as in_apk:
2539 with ZipFile(apkwithnewsig, 'w') as out_apk:
2540 for sig_file in [signaturefile, signedfile, manifest]:
2541 with open(sig_file, 'rb') as fp:
2543 info = zipfile.ZipInfo('META-INF/' + os.path.basename(sig_file))
2544 info.compress_type = zipfile.ZIP_DEFLATED
2545 info.create_system = 0 # "Windows" aka "FAT", what Android SDK uses
2546 out_apk.writestr(info, buf)
2547 for info in in_apk.infolist():
2548 if not apk_sigfile.match(info.filename):
2549 if info.filename != 'META-INF/MANIFEST.MF':
2550 buf = in_apk.read(info.filename)
2551 out_apk.writestr(info, buf)
2553 p = SdkToolsPopen(['zipalign', '-v', '4', apkwithnewsig, apkpath])
2554 if p.returncode != 0:
2555 raise BuildException("Failed to align application")
2558 def apk_extract_signatures(apkpath, outdir, manifest=True):
2559 """Extracts a signature files from APK and puts them into target directory.
2561 :param apkpath: location of the apk
2562 :param outdir: folder where the extracted signature files will be stored
2563 :param manifest: (optionally) disable extracting manifest file
2565 with ZipFile(apkpath, 'r') as in_apk:
2566 for f in in_apk.infolist():
2567 if apk_sigfile.match(f.filename) or \
2568 (manifest and f.filename == 'META-INF/MANIFEST.MF'):
2569 newpath = os.path.join(outdir, os.path.basename(f.filename))
2570 with open(newpath, 'wb') as out_file:
2571 out_file.write(in_apk.read(f.filename))
2574 def sign_apk(unsigned_path, signed_path, keyalias):
2575 """Sign and zipalign an unsigned APK, then save to a new file, deleting the unsigned
2577 android-18 (4.3) finally added support for reasonable hash
2578 algorithms, like SHA-256, before then, the only options were MD5
2579 and SHA1 :-/ This aims to use SHA-256 when the APK does not target
2580 older Android versions, and is therefore safe to do so.
2582 https://issuetracker.google.com/issues/36956587
2583 https://android-review.googlesource.com/c/platform/libcore/+/44491
2587 if get_minSdkVersion_aapt(unsigned_path) < 18:
2588 signature_algorithm = ['-sigalg', 'SHA1withRSA', '-digestalg', 'SHA1']
2590 signature_algorithm = ['-sigalg', 'SHA256withRSA', '-digestalg', 'SHA-256']
2592 p = FDroidPopen([config['jarsigner'], '-keystore', config['keystore'],
2593 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2594 '-keypass:env', 'FDROID_KEY_PASS']
2595 + signature_algorithm + [unsigned_path, keyalias],
2597 'FDROID_KEY_STORE_PASS': config['keystorepass'],
2598 'FDROID_KEY_PASS': config['keypass'], })
2599 if p.returncode != 0:
2600 raise BuildException(_("Failed to sign application"), p.output)
2602 p = SdkToolsPopen(['zipalign', '-v', '4', unsigned_path, signed_path])
2603 if p.returncode != 0:
2604 raise BuildException(_("Failed to zipalign application"))
2605 os.remove(unsigned_path)
2608 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
2609 """Verify that two apks are the same
2611 One of the inputs is signed, the other is unsigned. The signature metadata
2612 is transferred from the signed to the unsigned apk, and then jarsigner is
2613 used to verify that the signature from the signed apk is also varlid for
2614 the unsigned one. If the APK given as unsigned actually does have a
2615 signature, it will be stripped out and ignored.
2617 There are two SHA1 git commit IDs that fdroidserver includes in the builds
2618 it makes: fdroidserverid and buildserverid. Originally, these were inserted
2619 into AndroidManifest.xml, but that makes the build not reproducible. So
2620 instead they are included as separate files in the APK's META-INF/ folder.
2621 If those files exist in the signed APK, they will be part of the signature
2622 and need to also be included in the unsigned APK for it to validate.
2624 :param signed_apk: Path to a signed apk file
2625 :param unsigned_apk: Path to an unsigned apk file expected to match it
2626 :param tmp_dir: Path to directory for temporary files
2627 :returns: None if the verification is successful, otherwise a string
2628 describing what went wrong.
2631 if not os.path.isfile(signed_apk):
2632 return 'can not verify: file does not exists: {}'.format(signed_apk)
2634 if not os.path.isfile(unsigned_apk):
2635 return 'can not verify: file does not exists: {}'.format(unsigned_apk)
2637 with ZipFile(signed_apk, 'r') as signed:
2638 meta_inf_files = ['META-INF/MANIFEST.MF']
2639 for f in signed.namelist():
2640 if apk_sigfile.match(f) \
2641 or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
2642 meta_inf_files.append(f)
2643 if len(meta_inf_files) < 3:
2644 return "Signature files missing from {0}".format(signed_apk)
2646 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2647 with ZipFile(unsigned_apk, 'r') as unsigned:
2648 # only read the signature from the signed APK, everything else from unsigned
2649 with ZipFile(tmp_apk, 'w') as tmp:
2650 for filename in meta_inf_files:
2651 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2652 for info in unsigned.infolist():
2653 if info.filename in meta_inf_files:
2654 logging.warning('Ignoring %s from %s',
2655 info.filename, unsigned_apk)
2657 if info.filename in tmp.namelist():
2658 return "duplicate filename found: " + info.filename
2659 tmp.writestr(info, unsigned.read(info.filename))
2661 verified = verify_apk_signature(tmp_apk)
2664 logging.info("...NOT verified - {0}".format(tmp_apk))
2665 return compare_apks(signed_apk, tmp_apk, tmp_dir,
2666 os.path.dirname(unsigned_apk))
2668 logging.info("...successfully verified")
2672 def verify_jar_signature(jar):
2673 """Verifies the signature of a given JAR file.
2675 jarsigner is very shitty: unsigned JARs pass as "verified"! So
2676 this has to turn on -strict then check for result 4, since this
2677 does not expect the signature to be from a CA-signed certificate.
2679 :raises: VerificationException() if the JAR's signature could not be verified
2683 error = _('JAR signature failed to verify: {path}').format(path=jar)
2685 output = subprocess.check_output([config['jarsigner'], '-strict', '-verify', jar],
2686 stderr=subprocess.STDOUT)
2687 raise VerificationException(error + '\n' + output.decode('utf-8'))
2688 except subprocess.CalledProcessError as e:
2689 if e.returncode == 4:
2690 logging.debug(_('JAR signature verified: {path}').format(path=jar))
2692 raise VerificationException(error + '\n' + e.output.decode('utf-8'))
2695 def verify_apk_signature(apk, min_sdk_version=None):
2696 """verify the signature on an APK
2698 Try to use apksigner whenever possible since jarsigner is very
2699 shitty: unsigned APKs pass as "verified"! Warning, this does
2700 not work on JARs with apksigner >= 0.7 (build-tools 26.0.1)
2702 :returns: boolean whether the APK was verified
2704 if set_command_in_config('apksigner'):
2705 args = [config['apksigner'], 'verify']
2707 args += ['--min-sdk-version=' + min_sdk_version]
2709 args += ['--verbose']
2711 output = subprocess.check_output(args + [apk])
2713 logging.debug(apk + ': ' + output.decode('utf-8'))
2715 except subprocess.CalledProcessError as e:
2716 logging.error('\n' + apk + ': ' + e.output.decode('utf-8'))
2718 if not config.get('jarsigner_warning_displayed'):
2719 config['jarsigner_warning_displayed'] = True
2720 logging.warning(_("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner"))
2722 verify_jar_signature(apk)
2724 except Exception as e:
2729 def verify_old_apk_signature(apk):
2730 """verify the signature on an archived APK, supporting deprecated algorithms
2732 F-Droid aims to keep every single binary that it ever published. Therefore,
2733 it needs to be able to verify APK signatures that include deprecated/removed
2734 algorithms. For example, jarsigner treats an MD5 signature as unsigned.
2736 jarsigner passes unsigned APKs as "verified"! So this has to turn
2737 on -strict then check for result 4.
2739 Just to be safe, this never reuses the file, and locks down the
2740 file permissions while in use. That should prevent a bad actor
2741 from changing the settings during operation.
2743 :returns: boolean whether the APK was verified
2747 _java_security = os.path.join(os.getcwd(), '.java.security')
2748 if os.path.exists(_java_security):
2749 os.remove(_java_security)
2750 with open(_java_security, 'w') as fp:
2751 fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
2752 os.chmod(_java_security, 0o400)
2756 config['jarsigner'],
2757 '-J-Djava.security.properties=' + _java_security,
2758 '-strict', '-verify', apk
2760 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
2761 except subprocess.CalledProcessError as e:
2762 if e.returncode != 4:
2765 logging.debug(_('JAR signature verified: {path}').format(path=apk))
2768 if os.path.exists(_java_security):
2769 os.chmod(_java_security, 0o600)
2770 os.remove(_java_security)
2772 logging.error(_('Old APK signature failed to verify: {path}').format(path=apk)
2773 + '\n' + output.decode('utf-8'))
2777 apk_badchars = re.compile('''[/ :;'"]''')
2780 def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
2783 Returns None if the apk content is the same (apart from the signing key),
2784 otherwise a string describing what's different, or what went wrong when
2785 trying to do the comparison.
2791 absapk1 = os.path.abspath(apk1)
2792 absapk2 = os.path.abspath(apk2)
2794 if set_command_in_config('diffoscope'):
2795 logfilename = os.path.join(log_dir, os.path.basename(absapk1))
2796 htmlfile = logfilename + '.diffoscope.html'
2797 textfile = logfilename + '.diffoscope.txt'
2798 if subprocess.call([config['diffoscope'],
2799 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2800 '--html', htmlfile, '--text', textfile,
2801 absapk1, absapk2]) != 0:
2802 return("Failed to unpack " + apk1)
2804 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2805 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2806 for d in [apk1dir, apk2dir]:
2807 if os.path.exists(d):
2810 os.mkdir(os.path.join(d, 'jar-xf'))
2812 if subprocess.call(['jar', 'xf',
2813 os.path.abspath(apk1)],
2814 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2815 return("Failed to unpack " + apk1)
2816 if subprocess.call(['jar', 'xf',
2817 os.path.abspath(apk2)],
2818 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2819 return("Failed to unpack " + apk2)
2821 if set_command_in_config('apktool'):
2822 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2824 return("Failed to unpack " + apk1)
2825 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2827 return("Failed to unpack " + apk2)
2829 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2830 lines = p.output.splitlines()
2831 if len(lines) != 1 or 'META-INF' not in lines[0]:
2832 if set_command_in_config('meld'):
2833 p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
2834 return("Unexpected diff output - " + p.output)
2836 # since everything verifies, delete the comparison to keep cruft down
2837 shutil.rmtree(apk1dir)
2838 shutil.rmtree(apk2dir)
2840 # If we get here, it seems like they're the same!
2844 def set_command_in_config(command):
2845 '''Try to find specified command in the path, if it hasn't been
2846 manually set in config.py. If found, it is added to the config
2847 dict. The return value says whether the command is available.
2850 if command in config:
2853 tmp = find_command(command)
2855 config[command] = tmp
2860 def find_command(command):
2861 '''find the full path of a command, or None if it can't be found in the PATH'''
2864 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2866 fpath, fname = os.path.split(command)
2871 for path in os.environ["PATH"].split(os.pathsep):
2872 path = path.strip('"')
2873 exe_file = os.path.join(path, command)
2874 if is_exe(exe_file):
2881 '''generate a random password for when generating keys'''
2882 h = hashlib.sha256()
2883 h.update(os.urandom(16)) # salt
2884 h.update(socket.getfqdn().encode('utf-8'))
2885 passwd = base64.b64encode(h.digest()).strip()
2886 return passwd.decode('utf-8')
2889 def genkeystore(localconfig):
2891 Generate a new key with password provided in :param localconfig and add it to new keystore
2892 :return: hexed public key, public key fingerprint
2894 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2895 keystoredir = os.path.dirname(localconfig['keystore'])
2896 if keystoredir is None or keystoredir == '':
2897 keystoredir = os.path.join(os.getcwd(), keystoredir)
2898 if not os.path.exists(keystoredir):
2899 os.makedirs(keystoredir, mode=0o700)
2902 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
2903 'FDROID_KEY_PASS': localconfig['keypass'],
2905 p = FDroidPopen([config['keytool'], '-genkey',
2906 '-keystore', localconfig['keystore'],
2907 '-alias', localconfig['repo_keyalias'],
2908 '-keyalg', 'RSA', '-keysize', '4096',
2909 '-sigalg', 'SHA256withRSA',
2910 '-validity', '10000',
2911 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2912 '-keypass:env', 'FDROID_KEY_PASS',
2913 '-dname', localconfig['keydname']], envs=env_vars)
2914 if p.returncode != 0:
2915 raise BuildException("Failed to generate key", p.output)
2916 os.chmod(localconfig['keystore'], 0o0600)
2917 if not options.quiet:
2918 # now show the lovely key that was just generated
2919 p = FDroidPopen([config['keytool'], '-list', '-v',
2920 '-keystore', localconfig['keystore'],
2921 '-alias', localconfig['repo_keyalias'],
2922 '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
2923 logging.info(p.output.strip() + '\n\n')
2924 # get the public key
2925 p = FDroidPopenBytes([config['keytool'], '-exportcert',
2926 '-keystore', localconfig['keystore'],
2927 '-alias', localconfig['repo_keyalias'],
2928 '-storepass:env', 'FDROID_KEY_STORE_PASS']
2929 + config['smartcardoptions'],
2930 envs=env_vars, output=False, stderr_to_stdout=False)
2931 if p.returncode != 0 or len(p.output) < 20:
2932 raise BuildException("Failed to get public key", p.output)
2934 fingerprint = get_cert_fingerprint(pubkey)
2935 return hexlify(pubkey), fingerprint
2938 def get_cert_fingerprint(pubkey):
2940 Generate a certificate fingerprint the same way keytool does it
2941 (but with slightly different formatting)
2943 digest = hashlib.sha256(pubkey).digest()
2944 ret = [' '.join("%02X" % b for b in bytearray(digest))]
2945 return " ".join(ret)
2948 def get_certificate(certificate_file):
2950 Extracts a certificate from the given file.
2951 :param certificate_file: file bytes (as string) representing the certificate
2952 :return: A binary representation of the certificate's public key, or None in case of error
2954 content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
2955 if content.getComponentByName('contentType') != rfc2315.signedData:
2957 content = decoder.decode(content.getComponentByName('content'),
2958 asn1Spec=rfc2315.SignedData())[0]
2960 certificates = content.getComponentByName('certificates')
2961 cert = certificates[0].getComponentByName('certificate')
2963 logging.error("Certificates not found.")
2965 return encoder.encode(cert)
2968 def load_stats_fdroid_signing_key_fingerprints():
2969 """Load list of signing-key fingerprints stored by fdroid publish from file.
2971 :returns: list of dictionanryies containing the singing-key fingerprints.
2973 jar_file = os.path.join('stats', 'publishsigkeys.jar')
2974 if not os.path.isfile(jar_file):
2976 cmd = [config['jarsigner'], '-strict', '-verify', jar_file]
2977 p = FDroidPopen(cmd, output=False)
2978 if p.returncode != 4:
2979 raise FDroidException("Signature validation of '{}' failed! "
2980 "Please run publish again to rebuild this file.".format(jar_file))
2982 jar_sigkey = apk_signer_fingerprint(jar_file)
2983 repo_key_sig = config.get('repo_key_sha256')
2985 if jar_sigkey != repo_key_sig:
2986 raise FDroidException("Signature key fingerprint of file '{}' does not match repo_key_sha256 in config.py (found fingerprint: '{}')".format(jar_file, jar_sigkey))
2988 logging.warning("repo_key_sha256 not in config.py, setting it to the signature key fingerprint of '{}'".format(jar_file))
2989 config['repo_key_sha256'] = jar_sigkey
2990 write_to_config(config, 'repo_key_sha256')
2992 with zipfile.ZipFile(jar_file, 'r') as f:
2993 return json.loads(str(f.read('publishsigkeys.json'), 'utf-8'))
2996 def write_to_config(thisconfig, key, value=None, config_file=None):
2997 '''write a key/value to the local config.py
2999 NOTE: only supports writing string variables.
3001 :param thisconfig: config dictionary
3002 :param key: variable name in config.py to be overwritten/added
3003 :param value: optional value to be written, instead of fetched
3004 from 'thisconfig' dictionary.
3007 origkey = key + '_orig'
3008 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
3009 cfg = config_file if config_file else 'config.py'
3011 # load config file, create one if it doesn't exist
3012 if not os.path.exists(cfg):
3013 open(cfg, 'a').close()
3014 logging.info("Creating empty " + cfg)
3015 with open(cfg, 'r', encoding="utf-8") as f:
3016 lines = f.readlines()
3018 # make sure the file ends with a carraige return
3020 if not lines[-1].endswith('\n'):
3023 # regex for finding and replacing python string variable
3024 # definitions/initializations
3025 pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
3026 repl = key + ' = "' + value + '"'
3027 pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
3028 repl2 = key + " = '" + value + "'"
3030 # If we replaced this line once, we make sure won't be a
3031 # second instance of this line for this key in the document.
3034 with open(cfg, 'w', encoding="utf-8") as f:
3036 if pattern.match(line) or pattern2.match(line):
3038 line = pattern.sub(repl, line)
3039 line = pattern2.sub(repl2, line)
3050 def parse_xml(path):
3051 return XMLElementTree.parse(path).getroot()
3054 def string_is_integer(string):
3062 def local_rsync(options, fromdir, todir):
3063 '''Rsync method for local to local copying of things
3065 This is an rsync wrapper with all the settings for safe use within
3066 the various fdroidserver use cases. This uses stricter rsync
3067 checking on all files since people using offline mode are already
3068 prioritizing security above ease and speed.
3071 rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
3072 '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
3073 if not options.no_checksum:
3074 rsyncargs.append('--checksum')
3076 rsyncargs += ['--verbose']
3078 rsyncargs += ['--quiet']
3079 logging.debug(' '.join(rsyncargs + [fromdir, todir]))
3080 if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
3081 raise FDroidException()
3084 def get_per_app_repos():
3085 '''per-app repos are dirs named with the packageName of a single app'''
3087 # Android packageNames are Java packages, they may contain uppercase or
3088 # lowercase letters ('A' through 'Z'), numbers, and underscores
3089 # ('_'). However, individual package name parts may only start with
3090 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
3091 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
3094 for root, dirs, files in os.walk(os.getcwd()):
3096 print('checking', root, 'for', d)
3097 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
3098 # standard parts of an fdroid repo, so never packageNames
3101 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
3107 def is_repo_file(filename):
3108 '''Whether the file in a repo is a build product to be delivered to users'''
3109 if isinstance(filename, str):
3110 filename = filename.encode('utf-8', errors="surrogateescape")
3111 return os.path.isfile(filename) \
3112 and not filename.endswith(b'.asc') \
3113 and not filename.endswith(b'.sig') \
3114 and os.path.basename(filename) not in [
3116 b'index_unsigned.jar',
3125 def get_examples_dir():
3126 '''Return the dir where the fdroidserver example files are available'''
3128 tmp = os.path.dirname(sys.argv[0])
3129 if os.path.basename(tmp) == 'bin':
3130 egg_links = glob.glob(os.path.join(tmp, '..',
3131 'local/lib/python3.*/site-packages/fdroidserver.egg-link'))
3133 # installed from local git repo
3134 examplesdir = os.path.join(open(egg_links[0]).readline().rstrip(), 'examples')
3137 examplesdir = os.path.dirname(os.path.dirname(__file__)) + '/share/doc/fdroidserver/examples'
3138 if not os.path.exists(examplesdir): # use UNIX layout
3139 examplesdir = os.path.dirname(tmp) + '/share/doc/fdroidserver/examples'
3141 # we're running straight out of the git repo
3142 prefix = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
3143 examplesdir = prefix + '/examples'
3148 def get_wiki_timestamp(timestamp=None):
3149 """Return current time in the standard format for posting to the wiki"""
3151 if timestamp is None:
3152 timestamp = time.gmtime()
3153 return time.strftime("%Y-%m-%d %H:%M:%SZ", timestamp)
3156 def get_android_tools_versions(ndk_path=None):
3157 '''get a list of the versions of all installed Android SDK/NDK components'''
3160 sdk_path = config['sdk_path']
3161 if sdk_path[-1] != '/':
3165 ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT')
3166 if os.path.isfile(ndk_release_txt):
3167 with open(ndk_release_txt, 'r') as fp:
3168 components.append((os.path.basename(ndk_path), fp.read()[:-1]))
3170 pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE)
3171 for root, dirs, files in os.walk(sdk_path):
3172 if 'source.properties' in files:
3173 source_properties = os.path.join(root, 'source.properties')
3174 with open(source_properties, 'r') as fp:
3175 m = pattern.search(fp.read())
3177 components.append((root[len(sdk_path):], m.group(1)))
3182 def get_android_tools_version_log(ndk_path=None):
3183 '''get a list of the versions of all installed Android SDK/NDK components'''
3184 log = '== Installed Android Tools ==\n\n'
3185 components = get_android_tools_versions(ndk_path)
3186 for name, version in sorted(components):
3187 log += '* ' + name + ' (' + version + ')\n'
3192 def get_git_describe_link():
3193 """Get a link to the current fdroiddata commit, to post to the wiki
3197 output = subprocess.check_output(['git', 'describe', '--always', '--dirty', '--abbrev=0'],
3198 universal_newlines=True).strip()
3199 except subprocess.CalledProcessError:
3202 commit = output.replace('-dirty', '')
3203 return ('* fdroiddata: [https://gitlab.com/fdroid/fdroiddata/commit/{commit} {id}]\n'
3204 .format(commit=commit, id=output))
3206 logging.error(_("'{path}' failed to execute!").format(path='git describe'))