3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
40 import xml.etree.ElementTree as XMLElementTree
42 from binascii import hexlify
43 from datetime import datetime, timedelta
44 from distutils.version import LooseVersion
45 from queue import Queue
46 from zipfile import ZipFile
48 from pyasn1.codec.der import decoder, encoder
49 from pyasn1_modules import rfc2315
50 from pyasn1.error import PyAsn1Error
52 from distutils.util import strtobool
54 import fdroidserver.metadata
55 from fdroidserver import _
56 from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException,\
57 BuildException, VerificationException
58 from .asynchronousfilereader import AsynchronousFileReader
60 # this is the build-tools version, aapt has a separate version that
61 # has to be manually set in test_aapt_version()
62 MINIMUM_AAPT_VERSION = '26.0.0'
64 VERCODE_OPERATION_RE = re.compile(r'^([ 0-9/*+-]|%c)+$')
66 # A signature block file with a .DSA, .RSA, or .EC extension
67 CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
68 APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
69 STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
71 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
80 'sdk_path': "$ANDROID_HOME",
84 'r12b': "$ANDROID_NDK",
90 'build_tools': MINIMUM_AAPT_VERSION,
91 'force_build_tools': False,
96 'accepted_formats': ['txt', 'yml'],
97 'sync_from_local_copy_dir': False,
98 'allow_disabled_algorithms': False,
99 'per_app_repos': False,
100 'make_current_version_link': True,
101 'current_version_name_source': 'Name',
102 'update_stats': False,
104 'stats_server': None,
106 'stats_to_carbon': False,
108 'build_server_always': False,
109 'keystore': 'keystore.jks',
110 'smartcardoptions': [],
120 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
121 'repo_name': "My First FDroid Repo Demo",
122 'repo_icon': "fdroid-icon.png",
123 'repo_description': '''
124 This is a repository of apps to be used with FDroid. Applications in this
125 repository are either official binaries built by the original application
126 developers, or are binaries built from source by the admin of f-droid.org
127 using the tools on https://gitlab.com/u/fdroid.
133 def setup_global_opts(parser):
134 try: # the buildserver VM might not have PIL installed
135 from PIL import PngImagePlugin
136 logger = logging.getLogger(PngImagePlugin.__name__)
137 logger.setLevel(logging.INFO) # tame the "STREAM" debug messages
141 parser.add_argument("-v", "--verbose", action="store_true", default=False,
142 help=_("Spew out even more information than normal"))
143 parser.add_argument("-q", "--quiet", action="store_true", default=False,
144 help=_("Restrict output to warnings and errors"))
147 def _add_java_paths_to_config(pathlist, thisconfig):
148 def path_version_key(s):
150 for u in re.split('[^0-9]+', s):
152 versionlist.append(int(u))
157 for d in sorted(pathlist, key=path_version_key):
158 if os.path.islink(d):
160 j = os.path.basename(d)
161 # the last one found will be the canonical one, so order appropriately
163 r'^1\.([6-9])\.0\.jdk$', # OSX
164 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
165 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
166 r'^jdk([6-9])-openjdk$', # Arch
167 r'^java-([6-9])-openjdk$', # Arch
168 r'^java-([6-9])-jdk$', # Arch (oracle)
169 r'^java-1\.([6-9])\.0-.*$', # RedHat
170 r'^java-([6-9])-oracle$', # Debian WebUpd8
171 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
172 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
174 m = re.match(regex, j)
177 for p in [d, os.path.join(d, 'Contents', 'Home')]:
178 if os.path.exists(os.path.join(p, 'bin', 'javac')):
179 thisconfig['java_paths'][m.group(1)] = p
182 def fill_config_defaults(thisconfig):
183 for k, v in default_config.items():
184 if k not in thisconfig:
187 # Expand paths (~users and $vars)
188 def expand_path(path):
192 path = os.path.expanduser(path)
193 path = os.path.expandvars(path)
198 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
203 thisconfig[k + '_orig'] = v
205 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
206 if thisconfig['java_paths'] is None:
207 thisconfig['java_paths'] = dict()
209 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
210 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
211 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
212 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
213 if os.getenv('JAVA_HOME') is not None:
214 pathlist.append(os.getenv('JAVA_HOME'))
215 if os.getenv('PROGRAMFILES') is not None:
216 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
217 _add_java_paths_to_config(pathlist, thisconfig)
219 for java_version in ('7', '8', '9'):
220 if java_version not in thisconfig['java_paths']:
222 java_home = thisconfig['java_paths'][java_version]
223 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
224 if os.path.exists(jarsigner):
225 thisconfig['jarsigner'] = jarsigner
226 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
227 break # Java7 is preferred, so quit if found
229 for k in ['ndk_paths', 'java_paths']:
235 thisconfig[k][k2] = exp
236 thisconfig[k][k2 + '_orig'] = v
239 def regsub_file(pattern, repl, path):
240 with open(path, 'rb') as f:
242 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
243 with open(path, 'wb') as f:
247 def read_config(opts, config_file='config.py'):
248 """Read the repository config
250 The config is read from config_file, which is in the current
251 directory when any of the repo management commands are used. If
252 there is a local metadata file in the git repo, then config.py is
253 not required, just use defaults.
256 global config, options
258 if config is not None:
265 if os.path.isfile(config_file):
266 logging.debug(_("Reading '{config_file}'").format(config_file=config_file))
267 with io.open(config_file, "rb") as f:
268 code = compile(f.read(), config_file, 'exec')
269 exec(code, None, config)
271 logging.warning(_("No 'config.py' found, using defaults."))
273 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
275 if not type(config[k]) in (str, list, tuple):
277 _("'{field}' will be in random order! Use () or [] brackets if order is important!")
280 # smartcardoptions must be a list since its command line args for Popen
281 if 'smartcardoptions' in config:
282 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
283 elif 'keystore' in config and config['keystore'] == 'NONE':
284 # keystore='NONE' means use smartcard, these are required defaults
285 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
286 'SunPKCS11-OpenSC', '-providerClass',
287 'sun.security.pkcs11.SunPKCS11',
288 '-providerArg', 'opensc-fdroid.cfg']
290 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
291 st = os.stat(config_file)
292 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
293 logging.warning(_("unsafe permissions on '{config_file}' (should be 0600)!")
294 .format(config_file=config_file))
296 fill_config_defaults(config)
298 for k in ["repo_description", "archive_description"]:
300 config[k] = clean_description(config[k])
302 if 'serverwebroot' in config:
303 if isinstance(config['serverwebroot'], str):
304 roots = [config['serverwebroot']]
305 elif all(isinstance(item, str) for item in config['serverwebroot']):
306 roots = config['serverwebroot']
308 raise TypeError(_('only accepts strings, lists, and tuples'))
310 for rootstr in roots:
311 # since this is used with rsync, where trailing slashes have
312 # meaning, ensure there is always a trailing slash
313 if rootstr[-1] != '/':
315 rootlist.append(rootstr.replace('//', '/'))
316 config['serverwebroot'] = rootlist
318 if 'servergitmirrors' in config:
319 if isinstance(config['servergitmirrors'], str):
320 roots = [config['servergitmirrors']]
321 elif all(isinstance(item, str) for item in config['servergitmirrors']):
322 roots = config['servergitmirrors']
324 raise TypeError(_('only accepts strings, lists, and tuples'))
325 config['servergitmirrors'] = roots
330 def assert_config_keystore(config):
331 """Check weather keystore is configured correctly and raise exception if not."""
334 if 'repo_keyalias' not in config:
336 logging.critical(_("'repo_keyalias' not found in config.py!"))
337 if 'keystore' not in config:
339 logging.critical(_("'keystore' not found in config.py!"))
340 elif not os.path.exists(config['keystore']):
342 logging.critical("'" + config['keystore'] + "' does not exist!")
343 if 'keystorepass' not in config:
345 logging.critical(_("'keystorepass' not found in config.py!"))
346 if 'keypass' not in config:
348 logging.critical(_("'keypass' not found in config.py!"))
350 raise FDroidException("This command requires a signing key, " +
351 "you can create one using: fdroid update --create-key")
354 def find_sdk_tools_cmd(cmd):
355 '''find a working path to a tool from the Android SDK'''
358 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
359 # try to find a working path to this command, in all the recent possible paths
360 if 'build_tools' in config:
361 build_tools = os.path.join(config['sdk_path'], 'build-tools')
362 # if 'build_tools' was manually set and exists, check only that one
363 configed_build_tools = os.path.join(build_tools, config['build_tools'])
364 if os.path.exists(configed_build_tools):
365 tooldirs.append(configed_build_tools)
367 # no configed version, so hunt known paths for it
368 for f in sorted(os.listdir(build_tools), reverse=True):
369 if os.path.isdir(os.path.join(build_tools, f)):
370 tooldirs.append(os.path.join(build_tools, f))
371 tooldirs.append(build_tools)
372 sdk_tools = os.path.join(config['sdk_path'], 'tools')
373 if os.path.exists(sdk_tools):
374 tooldirs.append(sdk_tools)
375 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
376 if os.path.exists(sdk_platform_tools):
377 tooldirs.append(sdk_platform_tools)
378 tooldirs.append('/usr/bin')
380 path = os.path.join(d, cmd)
381 if os.path.isfile(path):
383 test_aapt_version(path)
385 # did not find the command, exit with error message
386 ensure_build_tools_exists(config)
389 def test_aapt_version(aapt):
390 '''Check whether the version of aapt is new enough'''
391 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
392 if output is None or output == '':
393 logging.error(_("'{path}' failed to execute!").format(path=aapt))
395 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
400 # the Debian package has the version string like "v0.2-23.0.2"
403 if LooseVersion(bugfix) < LooseVersion(MINIMUM_AAPT_VERSION):
405 elif LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.4062713'):
408 logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-{version} or newer!")
409 .format(aapt=aapt, version=MINIMUM_AAPT_VERSION))
411 logging.warning(_('Unknown version of aapt, might cause problems: ') + output)
414 def test_sdk_exists(thisconfig):
415 if 'sdk_path' not in thisconfig:
416 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
417 test_aapt_version(thisconfig['aapt'])
420 logging.error(_("'sdk_path' not set in 'config.py'!"))
422 if thisconfig['sdk_path'] == default_config['sdk_path']:
423 logging.error(_('No Android SDK found!'))
424 logging.error(_('You can use ANDROID_HOME to set the path to your SDK, i.e.:'))
425 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
427 if not os.path.exists(thisconfig['sdk_path']):
428 logging.critical(_("Android SDK path '{path}' does not exist!")
429 .format(path=thisconfig['sdk_path']))
431 if not os.path.isdir(thisconfig['sdk_path']):
432 logging.critical(_("Android SDK path '{path}' is not a directory!")
433 .format(path=thisconfig['sdk_path']))
435 for d in ['build-tools', 'platform-tools', 'tools']:
436 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
437 logging.critical(_("Android SDK '{path}' does not have '{dirname}' installed!")
438 .format(path=thisconfig['sdk_path'], dirname=d))
443 def ensure_build_tools_exists(thisconfig):
444 if not test_sdk_exists(thisconfig):
445 raise FDroidException(_("Android SDK not found!"))
446 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
447 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
448 if not os.path.isdir(versioned_build_tools):
449 raise FDroidException(
450 _("Android build-tools path '{path}' does not exist!")
451 .format(path=versioned_build_tools))
454 def get_local_metadata_files():
455 '''get any metadata files local to an app's source repo
457 This tries to ignore anything that does not count as app metdata,
458 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
461 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
464 def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False):
466 :param appids: arguments in the form of multiple appid:[vc] strings
467 :returns: a dictionary with the set of vercodes specified for each package
470 if not appid_versionCode_pairs:
473 for p in appid_versionCode_pairs:
474 if allow_vercodes and ':' in p:
475 package, vercode = p.split(':')
477 package, vercode = p, None
478 if package not in vercodes:
479 vercodes[package] = [vercode] if vercode else []
481 elif vercode and vercode not in vercodes[package]:
482 vercodes[package] += [vercode] if vercode else []
487 def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
488 """Build a list of App instances for processing
490 On top of what read_pkg_args does, this returns the whole app
491 metadata, but limiting the builds list to the builds matching the
492 appid_versionCode_pairs and vercodes specified. If no appid_versionCode_pairs are specified, then
493 all App and Build instances are returned.
497 vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes)
503 for appid, app in allapps.items():
504 if appid in vercodes:
507 if len(apps) != len(vercodes):
510 logging.critical(_("No such package: %s") % p)
511 raise FDroidException(_("Found invalid appids in arguments"))
513 raise FDroidException(_("No packages specified"))
516 for appid, app in apps.items():
520 app.builds = [b for b in app.builds if b.versionCode in vc]
521 if len(app.builds) != len(vercodes[appid]):
523 allvcs = [b.versionCode for b in app.builds]
524 for v in vercodes[appid]:
526 logging.critical(_("No such versionCode {versionCode} for app {appid}")
527 .format(versionCode=v, appid=appid))
530 raise FDroidException(_("Found invalid versionCodes for some apps"))
535 def get_extension(filename):
536 base, ext = os.path.splitext(filename)
539 return base, ext.lower()[1:]
542 def has_extension(filename, ext):
543 _ignored, f_ext = get_extension(filename)
547 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
550 def clean_description(description):
551 'Remove unneeded newlines and spaces from a block of description text'
553 # this is split up by paragraph to make removing the newlines easier
554 for paragraph in re.split(r'\n\n', description):
555 paragraph = re.sub('\r', '', paragraph)
556 paragraph = re.sub('\n', ' ', paragraph)
557 paragraph = re.sub(' {2,}', ' ', paragraph)
558 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
559 returnstring += paragraph + '\n\n'
560 return returnstring.rstrip('\n')
563 def publishednameinfo(filename):
564 filename = os.path.basename(filename)
565 m = publish_name_regex.match(filename)
567 result = (m.group(1), m.group(2))
568 except AttributeError:
569 raise FDroidException(_("Invalid name for published file: %s") % filename)
573 apk_release_filename = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)\.apk')
574 apk_release_filename_with_sigfp = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)_(?P<sigfp>[0-9a-f]{7})\.apk')
577 def apk_parse_release_filename(apkname):
578 """Parses the name of an APK file according the F-Droids APK naming
579 scheme and returns the tokens.
581 WARNING: Returned values don't necessarily represent the APKs actual
582 properties, the are just paresed from the file name.
584 :returns: A triplet containing (appid, versionCode, signer), where appid
585 should be the package name, versionCode should be the integer
586 represion of the APKs version and signer should be the first 7 hex
587 digists of the sha256 signing key fingerprint which was used to sign
590 m = apk_release_filename_with_sigfp.match(apkname)
592 return m.group('appid'), m.group('vercode'), m.group('sigfp')
593 m = apk_release_filename.match(apkname)
595 return m.group('appid'), m.group('vercode'), None
596 return None, None, None
599 def get_release_filename(app, build):
601 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
603 return "%s_%s.apk" % (app.id, build.versionCode)
606 def get_toolsversion_logname(app, build):
607 return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
610 def getsrcname(app, build):
611 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
623 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
626 def get_build_dir(app):
627 '''get the dir that this app will be built in'''
629 if app.RepoType == 'srclib':
630 return os.path.join('build', 'srclib', app.Repo)
632 return os.path.join('build', app.id)
636 '''checkout code from VCS and return instance of vcs and the build dir'''
637 build_dir = get_build_dir(app)
639 # Set up vcs interface and make sure we have the latest code...
640 logging.debug("Getting {0} vcs interface for {1}"
641 .format(app.RepoType, app.Repo))
642 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
646 vcs = getvcs(app.RepoType, remote, build_dir)
648 return vcs, build_dir
651 def getvcs(vcstype, remote, local):
653 return vcs_git(remote, local)
654 if vcstype == 'git-svn':
655 return vcs_gitsvn(remote, local)
657 return vcs_hg(remote, local)
659 return vcs_bzr(remote, local)
660 if vcstype == 'srclib':
661 if local != os.path.join('build', 'srclib', remote):
662 raise VCSException("Error: srclib paths are hard-coded!")
663 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
665 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
666 raise VCSException("Invalid vcs type " + vcstype)
669 def getsrclibvcs(name):
670 if name not in fdroidserver.metadata.srclibs:
671 raise VCSException("Missing srclib " + name)
672 return fdroidserver.metadata.srclibs[name]['Repo Type']
677 def __init__(self, remote, local):
679 # svn, git-svn and bzr may require auth
681 if self.repotype() in ('git-svn', 'bzr'):
683 if self.repotype == 'git-svn':
684 raise VCSException("Authentication is not supported for git-svn")
685 self.username, remote = remote.split('@')
686 if ':' not in self.username:
687 raise VCSException(_("Password required with username"))
688 self.username, self.password = self.username.split(':')
692 self.clone_failed = False
693 self.refreshed = False
699 def clientversion(self):
700 versionstr = FDroidPopen(self.clientversioncmd()).output
701 return versionstr[0:versionstr.find('\n')]
703 def clientversioncmd(self):
706 def gotorevision(self, rev, refresh=True):
707 """Take the local repository to a clean version of the given
708 revision, which is specificed in the VCS's native
709 format. Beforehand, the repository can be dirty, or even
710 non-existent. If the repository does already exist locally, it
711 will be updated from the origin, but only once in the lifetime
712 of the vcs object. None is acceptable for 'rev' if you know
713 you are cloning a clean copy of the repo - otherwise it must
714 specify a valid revision.
717 if self.clone_failed:
718 raise VCSException(_("Downloading the repository already failed once, not trying again."))
720 # The .fdroidvcs-id file for a repo tells us what VCS type
721 # and remote that directory was created from, allowing us to drop it
722 # automatically if either of those things changes.
723 fdpath = os.path.join(self.local, '..',
724 '.fdroidvcs-' + os.path.basename(self.local))
725 fdpath = os.path.normpath(fdpath)
726 cdata = self.repotype() + ' ' + self.remote
729 if os.path.exists(self.local):
730 if os.path.exists(fdpath):
731 with open(fdpath, 'r') as f:
732 fsdata = f.read().strip()
737 logging.info("Repository details for %s changed - deleting" % (
741 logging.info("Repository details for %s missing - deleting" % (
744 shutil.rmtree(self.local)
748 self.refreshed = True
751 self.gotorevisionx(rev)
752 except FDroidException as e:
755 # If necessary, write the .fdroidvcs file.
756 if writeback and not self.clone_failed:
757 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
758 with open(fdpath, 'w+') as f:
764 def gotorevisionx(self, rev): # pylint: disable=unused-argument
765 """Derived classes need to implement this.
767 It's called once basic checking has been performed.
769 raise VCSException("This VCS type doesn't define gotorevisionx")
771 # Initialise and update submodules
772 def initsubmodules(self):
773 raise VCSException('Submodules not supported for this vcs type')
775 # Get a list of all known tags
777 if not self._gettags:
778 raise VCSException('gettags not supported for this vcs type')
780 for tag in self._gettags():
781 if re.match('[-A-Za-z0-9_. /]+$', tag):
785 def latesttags(self):
786 """Get a list of all the known tags, sorted from newest to oldest"""
787 raise VCSException('latesttags not supported for this vcs type')
790 """Get current commit reference (hash, revision, etc)"""
791 raise VCSException('getref not supported for this vcs type')
794 """Returns the srclib (name, path) used in setting up the current revision, or None."""
803 def clientversioncmd(self):
804 return ['git', '--version']
806 def git(self, args, envs=dict(), cwd=None, output=True):
807 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
809 While fetch/pull/clone respect the command line option flags,
810 it seems that submodule commands do not. They do seem to
811 follow whatever is in env vars, if the version of git is new
812 enough. So we just throw the kitchen sink at it to see what
815 Also, because of CVE-2017-1000117, block all SSH URLs.
818 # supported in git >= 2.3
820 '-c', 'core.askpass=/bin/true',
821 '-c', 'core.sshCommand=/bin/false',
822 '-c', 'url.https://.insteadOf=ssh://',
824 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
825 git_config.append('-c')
826 git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':')
827 git_config.append('-c')
828 git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain)
829 git_config.append('-c')
830 git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain)
832 'GIT_TERMINAL_PROMPT': '0',
833 'GIT_ASKPASS': '/bin/true',
834 'SSH_ASKPASS': '/bin/true',
835 'GIT_SSH': '/bin/false', # for git < 2.3
837 return FDroidPopen(['git', ] + git_config + args,
838 envs=envs, cwd=cwd, output=output)
841 """If the local directory exists, but is somehow not a git repository,
842 git will traverse up the directory tree until it finds one
843 that is (i.e. fdroidserver) and then we'll proceed to destroy
844 it! This is called as a safety check.
848 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
849 result = p.output.rstrip()
850 if not result.endswith(self.local):
851 raise VCSException('Repository mismatch')
853 def gotorevisionx(self, rev):
854 if not os.path.exists(self.local):
856 p = self.git(['clone', '--', self.remote, self.local])
857 if p.returncode != 0:
858 self.clone_failed = True
859 raise VCSException("Git clone failed", p.output)
863 # Discard any working tree changes
864 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
865 'git', 'reset', '--hard'], cwd=self.local, output=False)
866 if p.returncode != 0:
867 raise VCSException(_("Git reset failed"), p.output)
868 # Remove untracked files now, in case they're tracked in the target
869 # revision (it happens!)
870 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
871 'git', 'clean', '-dffx'], cwd=self.local, output=False)
872 if p.returncode != 0:
873 raise VCSException(_("Git clean failed"), p.output)
874 if not self.refreshed:
875 # Get latest commits and tags from remote
876 p = self.git(['fetch', 'origin'], cwd=self.local)
877 if p.returncode != 0:
878 raise VCSException(_("Git fetch failed"), p.output)
879 p = self.git(['fetch', '--prune', '--tags', 'origin'], output=False, cwd=self.local)
880 if p.returncode != 0:
881 raise VCSException(_("Git fetch failed"), p.output)
882 # Recreate origin/HEAD as git clone would do it, in case it disappeared
883 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
884 if p.returncode != 0:
885 lines = p.output.splitlines()
886 if 'Multiple remote HEAD branches' not in lines[0]:
887 raise VCSException(_("Git remote set-head failed"), p.output)
888 branch = lines[1].split(' ')[-1]
889 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--', branch],
890 cwd=self.local, output=False)
891 if p2.returncode != 0:
892 raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output)
893 self.refreshed = True
894 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
895 # a github repo. Most of the time this is the same as origin/master.
896 rev = rev or 'origin/HEAD'
897 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
898 if p.returncode != 0:
899 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
900 # Get rid of any uncontrolled files left behind
901 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
902 if p.returncode != 0:
903 raise VCSException(_("Git clean failed"), p.output)
905 def initsubmodules(self):
907 submfile = os.path.join(self.local, '.gitmodules')
908 if not os.path.isfile(submfile):
909 raise NoSubmodulesException(_("No git submodules available"))
911 # fix submodules not accessible without an account and public key auth
912 with open(submfile, 'r') as f:
913 lines = f.readlines()
914 with open(submfile, 'w') as f:
916 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
917 line = re.sub('git@' + domain + ':', 'https://u:p@' + domain + '/', line)
920 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
921 if p.returncode != 0:
922 raise VCSException(_("Git submodule sync failed"), p.output)
923 p = self.git(['submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
924 if p.returncode != 0:
925 raise VCSException(_("Git submodule update failed"), p.output)
929 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
930 return p.output.splitlines()
932 tag_format = re.compile(r'tag: ([^),]*)')
934 def latesttags(self):
936 p = FDroidPopen(['git', 'log', '--tags',
937 '--simplify-by-decoration', '--pretty=format:%d'],
938 cwd=self.local, output=False)
940 for line in p.output.splitlines():
941 for tag in self.tag_format.findall(line):
946 class vcs_gitsvn(vcs):
951 def clientversioncmd(self):
952 return ['git', 'svn', '--version']
955 """If the local directory exists, but is somehow not a git repository,
956 git will traverse up the directory tree until it finds one that
957 is (i.e. fdroidserver) and then we'll proceed to destory it!
958 This is called as a safety check.
961 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
962 result = p.output.rstrip()
963 if not result.endswith(self.local):
964 raise VCSException('Repository mismatch')
966 def git(self, args, envs=dict(), cwd=None, output=True):
967 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
969 AskPass is set to /bin/true to let the process try to connect
970 without a username/password.
972 The SSH command is set to /bin/false to block all SSH URLs
973 (supported in git >= 2.3). This protects against
978 '-c', 'core.askpass=/bin/true',
979 '-c', 'core.sshCommand=/bin/false',
982 'GIT_TERMINAL_PROMPT': '0',
983 'GIT_ASKPASS': '/bin/true',
984 'SSH_ASKPASS': '/bin/true',
985 'GIT_SSH': '/bin/false', # for git < 2.3
986 'SVN_SSH': '/bin/false',
988 return FDroidPopen(['git', ] + git_config + args,
989 envs=envs, cwd=cwd, output=output)
991 def gotorevisionx(self, rev):
992 if not os.path.exists(self.local):
994 gitsvn_args = ['svn', 'clone']
996 if ';' in self.remote:
997 remote_split = self.remote.split(';')
998 for i in remote_split[1:]:
999 if i.startswith('trunk='):
1000 gitsvn_args.extend(['-T', i[6:]])
1001 elif i.startswith('tags='):
1002 gitsvn_args.extend(['-t', i[5:]])
1003 elif i.startswith('branches='):
1004 gitsvn_args.extend(['-b', i[9:]])
1005 remote = remote_split[0]
1007 remote = self.remote
1009 if not remote.startswith('https://'):
1010 raise VCSException(_('HTTPS must be used with Subversion URLs!'))
1012 # git-svn sucks at certificate validation, this throws useful errors:
1014 r = requests.head(remote)
1015 r.raise_for_status()
1016 location = r.headers.get('location')
1017 if location and not location.startswith('https://'):
1018 raise VCSException(_('Invalid redirect to non-HTTPS: {before} -> {after} ')
1019 .format(before=remote, after=location))
1021 gitsvn_args.extend(['--', remote, self.local])
1022 p = self.git(gitsvn_args)
1023 if p.returncode != 0:
1024 self.clone_failed = True
1025 raise VCSException(_('git svn clone failed'), p.output)
1029 # Discard any working tree changes
1030 p = self.git(['reset', '--hard'], cwd=self.local, output=False)
1031 if p.returncode != 0:
1032 raise VCSException("Git reset failed", p.output)
1033 # Remove untracked files now, in case they're tracked in the target
1034 # revision (it happens!)
1035 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1036 if p.returncode != 0:
1037 raise VCSException("Git clean failed", p.output)
1038 if not self.refreshed:
1039 # Get new commits, branches and tags from repo
1040 p = self.git(['svn', 'fetch'], cwd=self.local, output=False)
1041 if p.returncode != 0:
1042 raise VCSException("Git svn fetch failed")
1043 p = self.git(['svn', 'rebase'], cwd=self.local, output=False)
1044 if p.returncode != 0:
1045 raise VCSException("Git svn rebase failed", p.output)
1046 self.refreshed = True
1048 rev = rev or 'master'
1050 nospaces_rev = rev.replace(' ', '%20')
1051 # Try finding a svn tag
1052 for treeish in ['origin/', '']:
1053 p = self.git(['checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
1054 if p.returncode == 0:
1056 if p.returncode != 0:
1057 # No tag found, normal svn rev translation
1058 # Translate svn rev into git format
1059 rev_split = rev.split('/')
1062 for treeish in ['origin/', '']:
1063 if len(rev_split) > 1:
1064 treeish += rev_split[0]
1065 svn_rev = rev_split[1]
1068 # if no branch is specified, then assume trunk (i.e. 'master' branch):
1072 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
1074 p = self.git(['svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
1075 git_rev = p.output.rstrip()
1077 if p.returncode == 0 and git_rev:
1080 if p.returncode != 0 or not git_rev:
1081 # Try a plain git checkout as a last resort
1082 p = self.git(['checkout', rev], cwd=self.local, output=False)
1083 if p.returncode != 0:
1084 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
1086 # Check out the git rev equivalent to the svn rev
1087 p = self.git(['checkout', git_rev], cwd=self.local, output=False)
1088 if p.returncode != 0:
1089 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
1091 # Get rid of any uncontrolled files left behind
1092 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1093 if p.returncode != 0:
1094 raise VCSException(_("Git clean failed"), p.output)
1098 for treeish in ['origin/', '']:
1099 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
1100 if os.path.isdir(d):
1101 return os.listdir(d)
1105 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
1106 if p.returncode != 0:
1108 return p.output.strip()
1116 def clientversioncmd(self):
1117 return ['hg', '--version']
1119 def gotorevisionx(self, rev):
1120 if not os.path.exists(self.local):
1121 p = FDroidPopen(['hg', 'clone', '--ssh', '/bin/false', '--', self.remote, self.local],
1123 if p.returncode != 0:
1124 self.clone_failed = True
1125 raise VCSException("Hg clone failed", p.output)
1127 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
1128 if p.returncode != 0:
1129 raise VCSException("Hg status failed", p.output)
1130 for line in p.output.splitlines():
1131 if not line.startswith('? '):
1132 raise VCSException("Unexpected output from hg status -uS: " + line)
1133 FDroidPopen(['rm', '-rf', '--', line[2:]], cwd=self.local, output=False)
1134 if not self.refreshed:
1135 p = FDroidPopen(['hg', 'pull', '--ssh', '/bin/false'], cwd=self.local, output=False)
1136 if p.returncode != 0:
1137 raise VCSException("Hg pull failed", p.output)
1138 self.refreshed = True
1140 rev = rev or 'default'
1143 p = FDroidPopen(['hg', 'update', '-C', '--', rev], cwd=self.local, output=False)
1144 if p.returncode != 0:
1145 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
1146 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1147 # Also delete untracked files, we have to enable purge extension for that:
1148 if "'purge' is provided by the following extension" in p.output:
1149 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
1150 myfile.write("\n[extensions]\nhgext.purge=\n")
1151 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1152 if p.returncode != 0:
1153 raise VCSException("HG purge failed", p.output)
1154 elif p.returncode != 0:
1155 raise VCSException("HG purge failed", p.output)
1158 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
1159 return p.output.splitlines()[1:]
1167 def clientversioncmd(self):
1168 return ['bzr', '--version']
1170 def bzr(self, args, envs=dict(), cwd=None, output=True):
1171 '''Prevent bzr from ever using SSH to avoid security vulns'''
1175 return FDroidPopen(['bzr', ] + args, envs=envs, cwd=cwd, output=output)
1177 def gotorevisionx(self, rev):
1178 if not os.path.exists(self.local):
1179 p = self.bzr(['branch', self.remote, self.local], output=False)
1180 if p.returncode != 0:
1181 self.clone_failed = True
1182 raise VCSException("Bzr branch failed", p.output)
1184 p = self.bzr(['clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
1185 if p.returncode != 0:
1186 raise VCSException("Bzr revert failed", p.output)
1187 if not self.refreshed:
1188 p = self.bzr(['pull'], cwd=self.local, output=False)
1189 if p.returncode != 0:
1190 raise VCSException("Bzr update failed", p.output)
1191 self.refreshed = True
1193 revargs = list(['-r', rev] if rev else [])
1194 p = self.bzr(['revert'] + revargs, cwd=self.local, output=False)
1195 if p.returncode != 0:
1196 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1199 p = self.bzr(['tags'], cwd=self.local, output=False)
1200 return [tag.split(' ')[0].strip() for tag in
1201 p.output.splitlines()]
1204 def unescape_string(string):
1207 if string[0] == '"' and string[-1] == '"':
1210 return string.replace("\\'", "'")
1213 def retrieve_string(app_dir, string, xmlfiles=None):
1215 if not string.startswith('@string/'):
1216 return unescape_string(string)
1218 if xmlfiles is None:
1221 os.path.join(app_dir, 'res'),
1222 os.path.join(app_dir, 'src', 'main', 'res'),
1224 for root, dirs, files in os.walk(res_dir):
1225 if os.path.basename(root) == 'values':
1226 xmlfiles += [os.path.join(root, x) for x in files if x.endswith('.xml')]
1228 name = string[len('@string/'):]
1230 def element_content(element):
1231 if element.text is None:
1233 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1234 return s.decode('utf-8').strip()
1236 for path in xmlfiles:
1237 if not os.path.isfile(path):
1239 xml = parse_xml(path)
1240 element = xml.find('string[@name="' + name + '"]')
1241 if element is not None:
1242 content = element_content(element)
1243 return retrieve_string(app_dir, content, xmlfiles)
1248 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1249 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1252 def manifest_paths(app_dir, flavours):
1253 '''Return list of existing files that will be used to find the highest vercode'''
1255 possible_manifests = \
1256 [os.path.join(app_dir, 'AndroidManifest.xml'),
1257 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1258 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1259 os.path.join(app_dir, 'build.gradle')]
1261 for flavour in flavours:
1262 if flavour == 'yes':
1264 possible_manifests.append(
1265 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1267 return [path for path in possible_manifests if os.path.isfile(path)]
1270 def fetch_real_name(app_dir, flavours):
1271 '''Retrieve the package name. Returns the name, or None if not found.'''
1272 for path in manifest_paths(app_dir, flavours):
1273 if not has_extension(path, 'xml') or not os.path.isfile(path):
1275 logging.debug("fetch_real_name: Checking manifest at " + path)
1276 xml = parse_xml(path)
1277 app = xml.find('application')
1280 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1282 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1283 result = retrieve_string_singleline(app_dir, label)
1285 result = result.strip()
1290 def get_library_references(root_dir):
1292 proppath = os.path.join(root_dir, 'project.properties')
1293 if not os.path.isfile(proppath):
1295 with open(proppath, 'r', encoding='iso-8859-1') as f:
1297 if not line.startswith('android.library.reference.'):
1299 path = line.split('=')[1].strip()
1300 relpath = os.path.join(root_dir, path)
1301 if not os.path.isdir(relpath):
1303 logging.debug("Found subproject at %s" % path)
1304 libraries.append(path)
1308 def ant_subprojects(root_dir):
1309 subprojects = get_library_references(root_dir)
1310 for subpath in subprojects:
1311 subrelpath = os.path.join(root_dir, subpath)
1312 for p in get_library_references(subrelpath):
1313 relp = os.path.normpath(os.path.join(subpath, p))
1314 if relp not in subprojects:
1315 subprojects.insert(0, relp)
1319 def remove_debuggable_flags(root_dir):
1320 # Remove forced debuggable flags
1321 logging.debug("Removing debuggable flags from %s" % root_dir)
1322 for root, dirs, files in os.walk(root_dir):
1323 if 'AndroidManifest.xml' in files and os.path.isfile(os.path.join(root, 'AndroidManifest.xml')):
1324 regsub_file(r'android:debuggable="[^"]*"',
1326 os.path.join(root, 'AndroidManifest.xml'))
1329 vcsearch_g = re.compile(r'''.*[Vv]ersionCode\s*=?\s*["']*([0-9]+)["']*''').search
1330 vnsearch_g = re.compile(r'''.*[Vv]ersionName\s*=?\s*(["'])((?:(?=(\\?))\3.)*?)\1.*''').search
1331 psearch_g = re.compile(r'''.*(packageName|applicationId)\s*=*\s*["']([^"']+)["'].*''').search
1334 def app_matches_packagename(app, package):
1337 appid = app.UpdateCheckName or app.id
1338 if appid is None or appid == "Ignore":
1340 return appid == package
1343 def parse_androidmanifests(paths, app):
1345 Extract some information from the AndroidManifest.xml at the given path.
1346 Returns (version, vercode, package), any or all of which might be None.
1347 All values returned are strings.
1350 ignoreversions = app.UpdateCheckIgnore
1351 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1354 return (None, None, None)
1362 if not os.path.isfile(path):
1365 logging.debug(_("Parsing manifest at '{path}'").format(path=path))
1371 if app.builds and 'gradle' in app.builds[-1] and app.builds[-1].gradle:
1372 flavour = app.builds[-1].gradle[-1]
1374 if has_extension(path, 'gradle'):
1375 with open(path, 'r') as f:
1376 inside_flavour_group = 0
1377 inside_required_flavour = 0
1379 if gradle_comment.match(line):
1382 if inside_flavour_group > 0:
1383 if inside_required_flavour > 0:
1384 matches = psearch_g(line)
1386 s = matches.group(2)
1387 if app_matches_packagename(app, s):
1390 matches = vnsearch_g(line)
1392 version = matches.group(2)
1394 matches = vcsearch_g(line)
1396 vercode = matches.group(1)
1399 inside_required_flavour += 1
1401 inside_required_flavour -= 1
1403 if flavour and (flavour in line):
1404 inside_required_flavour = 1
1407 inside_flavour_group += 1
1409 inside_flavour_group -= 1
1411 if "productFlavors" in line:
1412 inside_flavour_group = 1
1414 matches = psearch_g(line)
1416 s = matches.group(2)
1417 if app_matches_packagename(app, s):
1420 matches = vnsearch_g(line)
1422 version = matches.group(2)
1424 matches = vcsearch_g(line)
1426 vercode = matches.group(1)
1429 xml = parse_xml(path)
1430 if "package" in xml.attrib:
1431 s = xml.attrib["package"]
1432 if app_matches_packagename(app, s):
1434 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1435 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1436 base_dir = os.path.dirname(path)
1437 version = retrieve_string_singleline(base_dir, version)
1438 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1439 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1440 if string_is_integer(a):
1443 logging.warning(_("Problem with xml at '{path}'").format(path=path))
1445 # Remember package name, may be defined separately from version+vercode
1447 package = max_package
1449 logging.debug("..got package={0}, version={1}, vercode={2}"
1450 .format(package, version, vercode))
1452 # Always grab the package name and version name in case they are not
1453 # together with the highest version code
1454 if max_package is None and package is not None:
1455 max_package = package
1456 if max_version is None and version is not None:
1457 max_version = version
1459 if vercode is not None \
1460 and (max_vercode is None or vercode > max_vercode):
1461 if not ignoresearch or not ignoresearch(version):
1462 if version is not None:
1463 max_version = version
1464 if vercode is not None:
1465 max_vercode = vercode
1466 if package is not None:
1467 max_package = package
1469 max_version = "Ignore"
1471 if max_version is None:
1472 max_version = "Unknown"
1474 if max_package and not is_valid_package_name(max_package):
1475 raise FDroidException(_("Invalid package name {0}").format(max_package))
1477 return (max_version, max_vercode, max_package)
1480 def is_valid_package_name(name):
1481 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1484 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1485 raw=False, prepare=True, preponly=False, refresh=True,
1487 """Get the specified source library.
1489 Returns the path to it. Normally this is the path to be used when
1490 referencing it, which may be a subdirectory of the actual project. If
1491 you want the base directory of the project, pass 'basepath=True'.
1500 name, ref = spec.split('@')
1502 number, name = name.split(':', 1)
1504 name, subdir = name.split('/', 1)
1506 if name not in fdroidserver.metadata.srclibs:
1507 raise VCSException('srclib ' + name + ' not found.')
1509 srclib = fdroidserver.metadata.srclibs[name]
1511 sdir = os.path.join(srclib_dir, name)
1514 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1515 vcs.srclib = (name, number, sdir)
1517 vcs.gotorevision(ref, refresh)
1524 libdir = os.path.join(sdir, subdir)
1525 elif srclib["Subdir"]:
1526 for subdir in srclib["Subdir"]:
1527 libdir_candidate = os.path.join(sdir, subdir)
1528 if os.path.exists(libdir_candidate):
1529 libdir = libdir_candidate
1535 remove_signing_keys(sdir)
1536 remove_debuggable_flags(sdir)
1540 if srclib["Prepare"]:
1541 cmd = replace_config_vars(srclib["Prepare"], build)
1543 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=libdir)
1544 if p.returncode != 0:
1545 raise BuildException("Error running prepare command for srclib %s"
1551 return (name, number, libdir)
1554 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1557 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1558 """ Prepare the source code for a particular build
1560 :param vcs: the appropriate vcs object for the application
1561 :param app: the application details from the metadata
1562 :param build: the build details from the metadata
1563 :param build_dir: the path to the build directory, usually 'build/app.id'
1564 :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
1565 :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
1567 Returns the (root, srclibpaths) where:
1568 :param root: is the root directory, which may be the same as 'build_dir' or may
1569 be a subdirectory of it.
1570 :param srclibpaths: is information on the srclibs being used
1573 # Optionally, the actual app source can be in a subdirectory
1575 root_dir = os.path.join(build_dir, build.subdir)
1577 root_dir = build_dir
1579 # Get a working copy of the right revision
1580 logging.info("Getting source for revision " + build.commit)
1581 vcs.gotorevision(build.commit, refresh)
1583 # Initialise submodules if required
1584 if build.submodules:
1585 logging.info(_("Initialising submodules"))
1586 vcs.initsubmodules()
1588 # Check that a subdir (if we're using one) exists. This has to happen
1589 # after the checkout, since it might not exist elsewhere
1590 if not os.path.exists(root_dir):
1591 raise BuildException('Missing subdir ' + root_dir)
1593 # Run an init command if one is required
1595 cmd = replace_config_vars(build.init, build)
1596 logging.info("Running 'init' commands in %s" % root_dir)
1598 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir)
1599 if p.returncode != 0:
1600 raise BuildException("Error running init command for %s:%s" %
1601 (app.id, build.versionName), p.output)
1603 # Apply patches if any
1605 logging.info("Applying patches")
1606 for patch in build.patch:
1607 patch = patch.strip()
1608 logging.info("Applying " + patch)
1609 patch_path = os.path.join('metadata', app.id, patch)
1610 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1611 if p.returncode != 0:
1612 raise BuildException("Failed to apply patch %s" % patch_path)
1614 # Get required source libraries
1617 logging.info("Collecting source libraries")
1618 for lib in build.srclibs:
1619 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1620 refresh=refresh, build=build))
1622 for name, number, libpath in srclibpaths:
1623 place_srclib(root_dir, int(number) if number else None, libpath)
1625 basesrclib = vcs.getsrclib()
1626 # If one was used for the main source, add that too.
1628 srclibpaths.append(basesrclib)
1630 # Update the local.properties file
1631 localprops = [os.path.join(build_dir, 'local.properties')]
1633 parts = build.subdir.split(os.sep)
1636 cur = os.path.join(cur, d)
1637 localprops += [os.path.join(cur, 'local.properties')]
1638 for path in localprops:
1640 if os.path.isfile(path):
1641 logging.info("Updating local.properties file at %s" % path)
1642 with open(path, 'r', encoding='iso-8859-1') as f:
1646 logging.info("Creating local.properties file at %s" % path)
1647 # Fix old-fashioned 'sdk-location' by copying
1648 # from sdk.dir, if necessary
1650 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1651 re.S | re.M).group(1)
1652 props += "sdk-location=%s\n" % sdkloc
1654 props += "sdk.dir=%s\n" % config['sdk_path']
1655 props += "sdk-location=%s\n" % config['sdk_path']
1656 ndk_path = build.ndk_path()
1657 # if for any reason the path isn't valid or the directory
1658 # doesn't exist, some versions of Gradle will error with a
1659 # cryptic message (even if the NDK is not even necessary).
1660 # https://gitlab.com/fdroid/fdroidserver/issues/171
1661 if ndk_path and os.path.exists(ndk_path):
1663 props += "ndk.dir=%s\n" % ndk_path
1664 props += "ndk-location=%s\n" % ndk_path
1665 # Add java.encoding if necessary
1667 props += "java.encoding=%s\n" % build.encoding
1668 with open(path, 'w', encoding='iso-8859-1') as f:
1672 if build.build_method() == 'gradle':
1673 flavours = build.gradle
1676 n = build.target.split('-')[1]
1677 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1678 r'compileSdkVersion %s' % n,
1679 os.path.join(root_dir, 'build.gradle'))
1681 # Remove forced debuggable flags
1682 remove_debuggable_flags(root_dir)
1684 # Insert version code and number into the manifest if necessary
1685 if build.forceversion:
1686 logging.info("Changing the version name")
1687 for path in manifest_paths(root_dir, flavours):
1688 if not os.path.isfile(path):
1690 if has_extension(path, 'xml'):
1691 regsub_file(r'android:versionName="[^"]*"',
1692 r'android:versionName="%s"' % build.versionName,
1694 elif has_extension(path, 'gradle'):
1695 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1696 r"""\1versionName '%s'""" % build.versionName,
1699 if build.forcevercode:
1700 logging.info("Changing the version code")
1701 for path in manifest_paths(root_dir, flavours):
1702 if not os.path.isfile(path):
1704 if has_extension(path, 'xml'):
1705 regsub_file(r'android:versionCode="[^"]*"',
1706 r'android:versionCode="%s"' % build.versionCode,
1708 elif has_extension(path, 'gradle'):
1709 regsub_file(r'versionCode[ =]+[0-9]+',
1710 r'versionCode %s' % build.versionCode,
1713 # Delete unwanted files
1715 logging.info(_("Removing specified files"))
1716 for part in getpaths(build_dir, build.rm):
1717 dest = os.path.join(build_dir, part)
1718 logging.info("Removing {0}".format(part))
1719 if os.path.lexists(dest):
1720 # rmtree can only handle directories that are not symlinks, so catch anything else
1721 if not os.path.isdir(dest) or os.path.islink(dest):
1726 logging.info("...but it didn't exist")
1728 remove_signing_keys(build_dir)
1730 # Add required external libraries
1732 logging.info("Collecting prebuilt libraries")
1733 libsdir = os.path.join(root_dir, 'libs')
1734 if not os.path.exists(libsdir):
1736 for lib in build.extlibs:
1738 logging.info("...installing extlib {0}".format(lib))
1739 libf = os.path.basename(lib)
1740 libsrc = os.path.join(extlib_dir, lib)
1741 if not os.path.exists(libsrc):
1742 raise BuildException("Missing extlib file {0}".format(libsrc))
1743 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1745 # Run a pre-build command if one is required
1747 logging.info("Running 'prebuild' commands in %s" % root_dir)
1749 cmd = replace_config_vars(build.prebuild, build)
1751 # Substitute source library paths into prebuild commands
1752 for name, number, libpath in srclibpaths:
1753 libpath = os.path.relpath(libpath, root_dir)
1754 cmd = cmd.replace('$$' + name + '$$', libpath)
1756 p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir)
1757 if p.returncode != 0:
1758 raise BuildException("Error running prebuild command for %s:%s" %
1759 (app.id, build.versionName), p.output)
1761 # Generate (or update) the ant build file, build.xml...
1762 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1763 parms = ['android', 'update', 'lib-project']
1764 lparms = ['android', 'update', 'project']
1767 parms += ['-t', build.target]
1768 lparms += ['-t', build.target]
1769 if build.androidupdate:
1770 update_dirs = build.androidupdate
1772 update_dirs = ant_subprojects(root_dir) + ['.']
1774 for d in update_dirs:
1775 subdir = os.path.join(root_dir, d)
1777 logging.debug("Updating main project")
1778 cmd = parms + ['-p', d]
1780 logging.debug("Updating subproject %s" % d)
1781 cmd = lparms + ['-p', d]
1782 p = SdkToolsPopen(cmd, cwd=root_dir)
1783 # Check to see whether an error was returned without a proper exit
1784 # code (this is the case for the 'no target set or target invalid'
1786 if p.returncode != 0 or p.output.startswith("Error: "):
1787 raise BuildException("Failed to update project at %s" % d, p.output)
1788 # Clean update dirs via ant
1790 logging.info("Cleaning subproject %s" % d)
1791 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1793 return (root_dir, srclibpaths)
1796 def getpaths_map(build_dir, globpaths):
1797 """Extend via globbing the paths from a field and return them as a map from original path to resulting paths"""
1801 full_path = os.path.join(build_dir, p)
1802 full_path = os.path.normpath(full_path)
1803 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1805 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1809 def getpaths(build_dir, globpaths):
1810 """Extend via globbing the paths from a field and return them as a set"""
1811 paths_map = getpaths_map(build_dir, globpaths)
1813 for k, v in paths_map.items():
1820 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1823 def check_system_clock(dt_obj, path):
1824 """Check if system clock is updated based on provided date
1826 If an APK has files newer than the system time, suggest updating
1827 the system clock. This is useful for offline systems, used for
1828 signing, which do not have another source of clock sync info. It
1829 has to be more than 24 hours newer because ZIP/APK files do not
1833 checkdt = dt_obj - timedelta(1)
1834 if datetime.today() < checkdt:
1835 logging.warning(_('System clock is older than date in {path}!').format(path=path)
1836 + '\n' + _('Set clock to that time using:') + '\n'
1837 + 'sudo date -s "' + str(dt_obj) + '"')
1841 """permanent store of existing APKs with the date they were added
1843 This is currently the only way to permanently store the "updated"
1848 '''Load filename/date info about previously seen APKs
1850 Since the appid and date strings both will never have spaces,
1851 this is parsed as a list from the end to allow the filename to
1852 have any combo of spaces.
1855 self.path = os.path.join('stats', 'known_apks.txt')
1857 if os.path.isfile(self.path):
1858 with open(self.path, 'r', encoding='utf8') as f:
1860 t = line.rstrip().split(' ')
1862 self.apks[t[0]] = (t[1], None)
1865 date = datetime.strptime(t[-1], '%Y-%m-%d')
1866 filename = line[0:line.rfind(appid) - 1]
1867 self.apks[filename] = (appid, date)
1868 check_system_clock(date, self.path)
1869 self.changed = False
1871 def writeifchanged(self):
1872 if not self.changed:
1875 if not os.path.exists('stats'):
1879 for apk, app in self.apks.items():
1881 line = apk + ' ' + appid
1883 line += ' ' + added.strftime('%Y-%m-%d')
1886 with open(self.path, 'w', encoding='utf8') as f:
1887 for line in sorted(lst, key=natural_key):
1888 f.write(line + '\n')
1890 def recordapk(self, apkName, app, default_date=None):
1892 Record an apk (if it's new, otherwise does nothing)
1893 Returns the date it was added as a datetime instance
1895 if apkName not in self.apks:
1896 if default_date is None:
1897 default_date = datetime.utcnow()
1898 self.apks[apkName] = (app, default_date)
1900 _ignored, added = self.apks[apkName]
1903 def getapp(self, apkname):
1904 """Look up information - given the 'apkname', returns (app id, date added/None).
1906 Or returns None for an unknown apk.
1908 if apkname in self.apks:
1909 return self.apks[apkname]
1912 def getlatest(self, num):
1913 """Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first"""
1915 for apk, app in self.apks.items():
1919 if apps[appid] > added:
1923 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1924 lst = [app for app, _ignored in sortedapps]
1929 def get_file_extension(filename):
1930 """get the normalized file extension, can be blank string but never None"""
1931 if isinstance(filename, bytes):
1932 filename = filename.decode('utf-8')
1933 return os.path.splitext(filename)[1].lower()[1:]
1936 def use_androguard():
1937 """Report if androguard is available, and config its debug logging"""
1941 if use_androguard.show_path:
1942 logging.debug(_('Using androguard from "{path}"').format(path=androguard.__file__))
1943 use_androguard.show_path = False
1944 if options and options.verbose:
1945 logging.getLogger("androguard.axml").setLevel(logging.INFO)
1951 use_androguard.show_path = True
1954 def is_apk_and_debuggable_aapt(apkfile):
1955 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1957 if p.returncode != 0:
1958 raise FDroidException(_("Failed to get APK manifest information"))
1959 for line in p.output.splitlines():
1960 if 'android:debuggable' in line and not line.endswith('0x0'):
1965 def is_apk_and_debuggable_androguard(apkfile):
1967 from androguard.core.bytecodes.apk import APK
1969 raise FDroidException("androguard library is not installed and aapt not present")
1971 apkobject = APK(apkfile)
1972 if apkobject.is_valid_APK():
1973 debuggable = apkobject.get_element("application", "debuggable")
1974 if debuggable is not None:
1975 return bool(strtobool(debuggable))
1979 def is_apk_and_debuggable(apkfile):
1980 """Returns True if the given file is an APK and is debuggable
1982 :param apkfile: full path to the apk to check"""
1984 if get_file_extension(apkfile) != 'apk':
1987 if use_androguard():
1988 return is_apk_and_debuggable_androguard(apkfile)
1990 return is_apk_and_debuggable_aapt(apkfile)
1993 def get_apk_id_aapt(apkfile):
1994 """Extrat identification information from APK using aapt.
1996 :param apkfile: path to an APK file.
1997 :returns: triplet (appid, version code, version name)
1999 r = re.compile("^package: name='(?P<appid>.*)' versionCode='(?P<vercode>.*)' versionName='(?P<vername>.*)'.*")
2000 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
2001 for line in p.output.splitlines():
2004 return m.group('appid'), m.group('vercode'), m.group('vername')
2005 raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'")
2006 .format(apkfilename=apkfile))
2009 def get_minSdkVersion_aapt(apkfile):
2010 """Extract the minimum supported Android SDK from an APK using aapt
2012 :param apkfile: path to an APK file.
2013 :returns: the integer representing the SDK version
2015 r = re.compile(r"^sdkVersion:'([0-9]+)'")
2016 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
2017 for line in p.output.splitlines():
2020 return int(m.group(1))
2021 raise FDroidException(_('Reading minSdkVersion failed: "{apkfilename}"')
2022 .format(apkfilename=apkfile))
2027 self.returncode = None
2031 def SdkToolsPopen(commands, cwd=None, output=True):
2033 if cmd not in config:
2034 config[cmd] = find_sdk_tools_cmd(commands[0])
2035 abscmd = config[cmd]
2037 raise FDroidException(_("Could not find '{command}' on your system").format(command=cmd))
2039 test_aapt_version(config['aapt'])
2040 return FDroidPopen([abscmd] + commands[1:],
2041 cwd=cwd, output=output)
2044 def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2046 Run a command and capture the possibly huge output as bytes.
2048 :param commands: command and argument list like in subprocess.Popen
2049 :param cwd: optionally specifies a working directory
2050 :param envs: a optional dictionary of environment variables and their values
2051 :returns: A PopenResult.
2056 set_FDroidPopen_env()
2058 process_env = env.copy()
2059 if envs is not None and len(envs) > 0:
2060 process_env.update(envs)
2063 cwd = os.path.normpath(cwd)
2064 logging.debug("Directory: %s" % cwd)
2065 logging.debug("> %s" % ' '.join(commands))
2067 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
2068 result = PopenResult()
2071 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
2072 stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
2073 stderr=stderr_param)
2074 except OSError as e:
2075 raise BuildException("OSError while trying to execute " +
2076 ' '.join(commands) + ': ' + str(e))
2078 # TODO are these AsynchronousFileReader threads always exiting?
2079 if not stderr_to_stdout and options.verbose:
2080 stderr_queue = Queue()
2081 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
2083 while not stderr_reader.eof():
2084 while not stderr_queue.empty():
2085 line = stderr_queue.get()
2086 sys.stderr.buffer.write(line)
2091 stdout_queue = Queue()
2092 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
2095 # Check the queue for output (until there is no more to get)
2096 while not stdout_reader.eof():
2097 while not stdout_queue.empty():
2098 line = stdout_queue.get()
2099 if output and options.verbose:
2100 # Output directly to console
2101 sys.stderr.buffer.write(line)
2107 result.returncode = p.wait()
2108 result.output = buf.getvalue()
2110 # make sure all filestreams of the subprocess are closed
2111 for streamvar in ['stdin', 'stdout', 'stderr']:
2112 if hasattr(p, streamvar):
2113 stream = getattr(p, streamvar)
2119 def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2121 Run a command and capture the possibly huge output as a str.
2123 :param commands: command and argument list like in subprocess.Popen
2124 :param cwd: optionally specifies a working directory
2125 :param envs: a optional dictionary of environment variables and their values
2126 :returns: A PopenResult.
2128 result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
2129 result.output = result.output.decode('utf-8', 'ignore')
2133 gradle_comment = re.compile(r'[ ]*//')
2134 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
2135 gradle_line_matches = [
2136 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
2137 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
2138 re.compile(r'.*\.readLine\(.*'),
2142 def remove_signing_keys(build_dir):
2143 for root, dirs, files in os.walk(build_dir):
2144 if 'build.gradle' in files:
2145 path = os.path.join(root, 'build.gradle')
2147 with open(path, "r", encoding='utf8') as o:
2148 lines = o.readlines()
2154 with open(path, "w", encoding='utf8') as o:
2155 while i < len(lines):
2158 while line.endswith('\\\n'):
2159 line = line.rstrip('\\\n') + lines[i]
2162 if gradle_comment.match(line):
2167 opened += line.count('{')
2168 opened -= line.count('}')
2171 if gradle_signing_configs.match(line):
2176 if any(s.match(line) for s in gradle_line_matches):
2184 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
2187 'project.properties',
2189 'default.properties',
2190 'ant.properties', ]:
2191 if propfile in files:
2192 path = os.path.join(root, propfile)
2194 with open(path, "r", encoding='iso-8859-1') as o:
2195 lines = o.readlines()
2199 with open(path, "w", encoding='iso-8859-1') as o:
2201 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
2208 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
2211 def set_FDroidPopen_env(build=None):
2213 set up the environment variables for the build environment
2215 There is only a weak standard, the variables used by gradle, so also set
2216 up the most commonly used environment variables for SDK and NDK. Also, if
2217 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
2219 global env, orig_path
2223 orig_path = env['PATH']
2224 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
2225 env[n] = config['sdk_path']
2226 for k, v in config['java_paths'].items():
2227 env['JAVA%s_HOME' % k] = v
2229 missinglocale = True
2230 for k, v in env.items():
2231 if k == 'LANG' and v != 'C':
2232 missinglocale = False
2234 missinglocale = False
2236 env['LANG'] = 'en_US.UTF-8'
2238 if build is not None:
2239 path = build.ndk_path()
2240 paths = orig_path.split(os.pathsep)
2241 if path not in paths:
2242 paths = [path] + paths
2243 env['PATH'] = os.pathsep.join(paths)
2244 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
2245 env[n] = build.ndk_path()
2248 def replace_build_vars(cmd, build):
2249 cmd = cmd.replace('$$COMMIT$$', build.commit)
2250 cmd = cmd.replace('$$VERSION$$', build.versionName)
2251 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
2255 def replace_config_vars(cmd, build):
2256 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
2257 cmd = cmd.replace('$$NDK$$', build.ndk_path())
2258 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
2259 if build is not None:
2260 cmd = replace_build_vars(cmd, build)
2264 def place_srclib(root_dir, number, libpath):
2267 relpath = os.path.relpath(libpath, root_dir)
2268 proppath = os.path.join(root_dir, 'project.properties')
2271 if os.path.isfile(proppath):
2272 with open(proppath, "r", encoding='iso-8859-1') as o:
2273 lines = o.readlines()
2275 with open(proppath, "w", encoding='iso-8859-1') as o:
2278 if line.startswith('android.library.reference.%d=' % number):
2279 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2284 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2287 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z_\-]+\.(SF|RSA|DSA|EC)')
2290 def signer_fingerprint_short(sig):
2291 """Obtain shortened sha256 signing-key fingerprint for pkcs7 signature.
2293 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2294 for a given pkcs7 signature.
2296 :param sig: Contents of an APK signing certificate.
2297 :returns: shortened signing-key fingerprint.
2299 return signer_fingerprint(sig)[:7]
2302 def signer_fingerprint(sig):
2303 """Obtain sha256 signing-key fingerprint for pkcs7 signature.
2305 Extracts hexadecimal sha256 signing-key fingerprint string
2306 for a given pkcs7 signature.
2308 :param: Contents of an APK signature.
2309 :returns: shortened signature fingerprint.
2311 cert_encoded = get_certificate(sig)
2312 return hashlib.sha256(cert_encoded).hexdigest()
2315 def apk_signer_fingerprint(apk_path):
2316 """Obtain sha256 signing-key fingerprint for APK.
2318 Extracts hexadecimal sha256 signing-key fingerprint string
2321 :param apkpath: path to APK
2322 :returns: signature fingerprint
2325 with zipfile.ZipFile(apk_path, 'r') as apk:
2326 certs = [n for n in apk.namelist() if CERT_PATH_REGEX.match(n)]
2329 logging.error("Found no signing certificates on %s" % apk_path)
2332 logging.error("Found multiple signing certificates on %s" % apk_path)
2335 cert = apk.read(certs[0])
2336 return signer_fingerprint(cert)
2339 def apk_signer_fingerprint_short(apk_path):
2340 """Obtain shortened sha256 signing-key fingerprint for APK.
2342 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2343 for a given pkcs7 APK.
2345 :param apk_path: path to APK
2346 :returns: shortened signing-key fingerprint
2348 return apk_signer_fingerprint(apk_path)[:7]
2351 def metadata_get_sigdir(appid, vercode=None):
2352 """Get signature directory for app"""
2354 return os.path.join('metadata', appid, 'signatures', vercode)
2356 return os.path.join('metadata', appid, 'signatures')
2359 def metadata_find_developer_signature(appid, vercode=None):
2360 """Tires to find the developer signature for given appid.
2362 This picks the first signature file found in metadata an returns its
2365 :returns: sha256 signing key fingerprint of the developer signing key.
2366 None in case no signature can not be found."""
2368 # fetch list of dirs for all versions of signatures
2371 appversigdirs.append(metadata_get_sigdir(appid, vercode))
2373 appsigdir = metadata_get_sigdir(appid)
2374 if os.path.isdir(appsigdir):
2375 numre = re.compile('[0-9]+')
2376 for ver in os.listdir(appsigdir):
2377 if numre.match(ver):
2378 appversigdir = os.path.join(appsigdir, ver)
2379 appversigdirs.append(appversigdir)
2381 for sigdir in appversigdirs:
2382 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2383 glob.glob(os.path.join(sigdir, '*.EC')) + \
2384 glob.glob(os.path.join(sigdir, '*.RSA'))
2386 raise FDroidException('ambiguous signatures, please make sure there is only one signature in \'{}\'. (The signature has to be the App maintainers signature for version of the APK.)'.format(sigdir))
2388 with open(sig, 'rb') as f:
2389 return signer_fingerprint(f.read())
2393 def metadata_find_signing_files(appid, vercode):
2394 """Gets a list of singed manifests and signatures.
2396 :param appid: app id string
2397 :param vercode: app version code
2398 :returns: a list of triplets for each signing key with following paths:
2399 (signature_file, singed_file, manifest_file)
2402 sigdir = metadata_get_sigdir(appid, vercode)
2403 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2404 glob.glob(os.path.join(sigdir, '*.EC')) + \
2405 glob.glob(os.path.join(sigdir, '*.RSA'))
2406 extre = re.compile('(\.DSA|\.EC|\.RSA)$')
2408 sf = extre.sub('.SF', sig)
2409 if os.path.isfile(sf):
2410 mf = os.path.join(sigdir, 'MANIFEST.MF')
2411 if os.path.isfile(mf):
2412 ret.append((sig, sf, mf))
2416 def metadata_find_developer_signing_files(appid, vercode):
2417 """Get developer signature files for specified app from metadata.
2419 :returns: A triplet of paths for signing files from metadata:
2420 (signature_file, singed_file, manifest_file)
2422 allsigningfiles = metadata_find_signing_files(appid, vercode)
2423 if allsigningfiles and len(allsigningfiles) == 1:
2424 return allsigningfiles[0]
2429 def apk_strip_signatures(signed_apk, strip_manifest=False):
2430 """Removes signatures from APK.
2432 :param signed_apk: path to apk file.
2433 :param strip_manifest: when set to True also the manifest file will
2434 be removed from the APK.
2436 with tempfile.TemporaryDirectory() as tmpdir:
2437 tmp_apk = os.path.join(tmpdir, 'tmp.apk')
2438 shutil.move(signed_apk, tmp_apk)
2439 with ZipFile(tmp_apk, 'r') as in_apk:
2440 with ZipFile(signed_apk, 'w') as out_apk:
2441 for info in in_apk.infolist():
2442 if not apk_sigfile.match(info.filename):
2444 if info.filename != 'META-INF/MANIFEST.MF':
2445 buf = in_apk.read(info.filename)
2446 out_apk.writestr(info, buf)
2448 buf = in_apk.read(info.filename)
2449 out_apk.writestr(info, buf)
2452 def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest):
2453 """Implats a signature from metadata into an APK.
2455 Note: this changes there supplied APK in place. So copy it if you
2456 need the original to be preserved.
2458 :param apkpath: location of the apk
2460 # get list of available signature files in metadata
2461 with tempfile.TemporaryDirectory() as tmpdir:
2462 apkwithnewsig = os.path.join(tmpdir, 'newsig.apk')
2463 with ZipFile(apkpath, 'r') as in_apk:
2464 with ZipFile(apkwithnewsig, 'w') as out_apk:
2465 for sig_file in [signaturefile, signedfile, manifest]:
2466 with open(sig_file, 'rb') as fp:
2468 info = zipfile.ZipInfo('META-INF/' + os.path.basename(sig_file))
2469 info.compress_type = zipfile.ZIP_DEFLATED
2470 info.create_system = 0 # "Windows" aka "FAT", what Android SDK uses
2471 out_apk.writestr(info, buf)
2472 for info in in_apk.infolist():
2473 if not apk_sigfile.match(info.filename):
2474 if info.filename != 'META-INF/MANIFEST.MF':
2475 buf = in_apk.read(info.filename)
2476 out_apk.writestr(info, buf)
2478 p = SdkToolsPopen(['zipalign', '-v', '4', apkwithnewsig, apkpath])
2479 if p.returncode != 0:
2480 raise BuildException("Failed to align application")
2483 def apk_extract_signatures(apkpath, outdir, manifest=True):
2484 """Extracts a signature files from APK and puts them into target directory.
2486 :param apkpath: location of the apk
2487 :param outdir: folder where the extracted signature files will be stored
2488 :param manifest: (optionally) disable extracting manifest file
2490 with ZipFile(apkpath, 'r') as in_apk:
2491 for f in in_apk.infolist():
2492 if apk_sigfile.match(f.filename) or \
2493 (manifest and f.filename == 'META-INF/MANIFEST.MF'):
2494 newpath = os.path.join(outdir, os.path.basename(f.filename))
2495 with open(newpath, 'wb') as out_file:
2496 out_file.write(in_apk.read(f.filename))
2499 def sign_apk(unsigned_path, signed_path, keyalias):
2500 """Sign and zipalign an unsigned APK, then save to a new file, deleting the unsigned
2502 android-18 (4.3) finally added support for reasonable hash
2503 algorithms, like SHA-256, before then, the only options were MD5
2504 and SHA1 :-/ This aims to use SHA-256 when the APK does not target
2505 older Android versions, and is therefore safe to do so.
2507 https://issuetracker.google.com/issues/36956587
2508 https://android-review.googlesource.com/c/platform/libcore/+/44491
2512 if get_minSdkVersion_aapt(unsigned_path) < 18:
2513 signature_algorithm = ['-sigalg', 'SHA1withRSA', '-digestalg', 'SHA1']
2515 signature_algorithm = ['-sigalg', 'SHA256withRSA', '-digestalg', 'SHA-256']
2517 p = FDroidPopen([config['jarsigner'], '-keystore', config['keystore'],
2518 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2519 '-keypass:env', 'FDROID_KEY_PASS']
2520 + signature_algorithm + [unsigned_path, keyalias],
2522 'FDROID_KEY_STORE_PASS': config['keystorepass'],
2523 'FDROID_KEY_PASS': config['keypass'], })
2524 if p.returncode != 0:
2525 raise BuildException(_("Failed to sign application"), p.output)
2527 p = SdkToolsPopen(['zipalign', '-v', '4', unsigned_path, signed_path])
2528 if p.returncode != 0:
2529 raise BuildException(_("Failed to zipalign application"))
2530 os.remove(unsigned_path)
2533 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
2534 """Verify that two apks are the same
2536 One of the inputs is signed, the other is unsigned. The signature metadata
2537 is transferred from the signed to the unsigned apk, and then jarsigner is
2538 used to verify that the signature from the signed apk is also varlid for
2539 the unsigned one. If the APK given as unsigned actually does have a
2540 signature, it will be stripped out and ignored.
2542 There are two SHA1 git commit IDs that fdroidserver includes in the builds
2543 it makes: fdroidserverid and buildserverid. Originally, these were inserted
2544 into AndroidManifest.xml, but that makes the build not reproducible. So
2545 instead they are included as separate files in the APK's META-INF/ folder.
2546 If those files exist in the signed APK, they will be part of the signature
2547 and need to also be included in the unsigned APK for it to validate.
2549 :param signed_apk: Path to a signed apk file
2550 :param unsigned_apk: Path to an unsigned apk file expected to match it
2551 :param tmp_dir: Path to directory for temporary files
2552 :returns: None if the verification is successful, otherwise a string
2553 describing what went wrong.
2556 if not os.path.isfile(signed_apk):
2557 return 'can not verify: file does not exists: {}'.format(signed_apk)
2559 if not os.path.isfile(unsigned_apk):
2560 return 'can not verify: file does not exists: {}'.format(unsigned_apk)
2562 with ZipFile(signed_apk, 'r') as signed:
2563 meta_inf_files = ['META-INF/MANIFEST.MF']
2564 for f in signed.namelist():
2565 if apk_sigfile.match(f) \
2566 or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
2567 meta_inf_files.append(f)
2568 if len(meta_inf_files) < 3:
2569 return "Signature files missing from {0}".format(signed_apk)
2571 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2572 with ZipFile(unsigned_apk, 'r') as unsigned:
2573 # only read the signature from the signed APK, everything else from unsigned
2574 with ZipFile(tmp_apk, 'w') as tmp:
2575 for filename in meta_inf_files:
2576 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2577 for info in unsigned.infolist():
2578 if info.filename in meta_inf_files:
2579 logging.warning('Ignoring %s from %s',
2580 info.filename, unsigned_apk)
2582 if info.filename in tmp.namelist():
2583 return "duplicate filename found: " + info.filename
2584 tmp.writestr(info, unsigned.read(info.filename))
2586 verified = verify_apk_signature(tmp_apk)
2589 logging.info("...NOT verified - {0}".format(tmp_apk))
2590 return compare_apks(signed_apk, tmp_apk, tmp_dir,
2591 os.path.dirname(unsigned_apk))
2593 logging.info("...successfully verified")
2597 def verify_jar_signature(jar):
2598 """Verifies the signature of a given JAR file.
2600 jarsigner is very shitty: unsigned JARs pass as "verified"! So
2601 this has to turn on -strict then check for result 4, since this
2602 does not expect the signature to be from a CA-signed certificate.
2604 :raises: VerificationException() if the JAR's signature could not be verified
2608 error = _('JAR signature failed to verify: {path}').format(path=jar)
2610 output = subprocess.check_output([config['jarsigner'], '-strict', '-verify', jar],
2611 stderr=subprocess.STDOUT)
2612 raise VerificationException(error + '\n' + output.decode('utf-8'))
2613 except subprocess.CalledProcessError as e:
2614 if e.returncode == 4:
2615 logging.debug(_('JAR signature verified: {path}').format(path=jar))
2617 raise VerificationException(error + '\n' + e.output.decode('utf-8'))
2620 def verify_apk_signature(apk, min_sdk_version=None):
2621 """verify the signature on an APK
2623 Try to use apksigner whenever possible since jarsigner is very
2624 shitty: unsigned APKs pass as "verified"! Warning, this does
2625 not work on JARs with apksigner >= 0.7 (build-tools 26.0.1)
2627 :returns: boolean whether the APK was verified
2629 if set_command_in_config('apksigner'):
2630 args = [config['apksigner'], 'verify']
2632 args += ['--min-sdk-version=' + min_sdk_version]
2634 args += ['--verbose']
2636 output = subprocess.check_output(args + [apk])
2638 logging.debug(apk + ': ' + output.decode('utf-8'))
2640 except subprocess.CalledProcessError as e:
2641 logging.error('\n' + apk + ': ' + e.output.decode('utf-8'))
2643 if not config.get('jarsigner_warning_displayed'):
2644 config['jarsigner_warning_displayed'] = True
2645 logging.warning(_("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner"))
2647 verify_jar_signature(apk)
2649 except Exception as e:
2654 def verify_old_apk_signature(apk):
2655 """verify the signature on an archived APK, supporting deprecated algorithms
2657 F-Droid aims to keep every single binary that it ever published. Therefore,
2658 it needs to be able to verify APK signatures that include deprecated/removed
2659 algorithms. For example, jarsigner treats an MD5 signature as unsigned.
2661 jarsigner passes unsigned APKs as "verified"! So this has to turn
2662 on -strict then check for result 4.
2664 :returns: boolean whether the APK was verified
2667 _java_security = os.path.join(os.getcwd(), '.java.security')
2668 with open(_java_security, 'w') as fp:
2669 fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
2673 config['jarsigner'],
2674 '-J-Djava.security.properties=' + _java_security,
2675 '-strict', '-verify', apk
2677 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
2678 except subprocess.CalledProcessError as e:
2679 if e.returncode != 4:
2682 logging.debug(_('JAR signature verified: {path}').format(path=apk))
2685 logging.error(_('Old APK signature failed to verify: {path}').format(path=apk)
2686 + '\n' + output.decode('utf-8'))
2690 apk_badchars = re.compile('''[/ :;'"]''')
2693 def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
2696 Returns None if the apk content is the same (apart from the signing key),
2697 otherwise a string describing what's different, or what went wrong when
2698 trying to do the comparison.
2704 absapk1 = os.path.abspath(apk1)
2705 absapk2 = os.path.abspath(apk2)
2707 if set_command_in_config('diffoscope'):
2708 logfilename = os.path.join(log_dir, os.path.basename(absapk1))
2709 htmlfile = logfilename + '.diffoscope.html'
2710 textfile = logfilename + '.diffoscope.txt'
2711 if subprocess.call([config['diffoscope'],
2712 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2713 '--html', htmlfile, '--text', textfile,
2714 absapk1, absapk2]) != 0:
2715 return("Failed to unpack " + apk1)
2717 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2718 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2719 for d in [apk1dir, apk2dir]:
2720 if os.path.exists(d):
2723 os.mkdir(os.path.join(d, 'jar-xf'))
2725 if subprocess.call(['jar', 'xf',
2726 os.path.abspath(apk1)],
2727 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2728 return("Failed to unpack " + apk1)
2729 if subprocess.call(['jar', 'xf',
2730 os.path.abspath(apk2)],
2731 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2732 return("Failed to unpack " + apk2)
2734 if set_command_in_config('apktool'):
2735 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2737 return("Failed to unpack " + apk1)
2738 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2740 return("Failed to unpack " + apk2)
2742 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2743 lines = p.output.splitlines()
2744 if len(lines) != 1 or 'META-INF' not in lines[0]:
2745 if set_command_in_config('meld'):
2746 p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
2747 return("Unexpected diff output - " + p.output)
2749 # since everything verifies, delete the comparison to keep cruft down
2750 shutil.rmtree(apk1dir)
2751 shutil.rmtree(apk2dir)
2753 # If we get here, it seems like they're the same!
2757 def set_command_in_config(command):
2758 '''Try to find specified command in the path, if it hasn't been
2759 manually set in config.py. If found, it is added to the config
2760 dict. The return value says whether the command is available.
2763 if command in config:
2766 tmp = find_command(command)
2768 config[command] = tmp
2773 def find_command(command):
2774 '''find the full path of a command, or None if it can't be found in the PATH'''
2777 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2779 fpath, fname = os.path.split(command)
2784 for path in os.environ["PATH"].split(os.pathsep):
2785 path = path.strip('"')
2786 exe_file = os.path.join(path, command)
2787 if is_exe(exe_file):
2794 '''generate a random password for when generating keys'''
2795 h = hashlib.sha256()
2796 h.update(os.urandom(16)) # salt
2797 h.update(socket.getfqdn().encode('utf-8'))
2798 passwd = base64.b64encode(h.digest()).strip()
2799 return passwd.decode('utf-8')
2802 def genkeystore(localconfig):
2804 Generate a new key with password provided in :param localconfig and add it to new keystore
2805 :return: hexed public key, public key fingerprint
2807 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2808 keystoredir = os.path.dirname(localconfig['keystore'])
2809 if keystoredir is None or keystoredir == '':
2810 keystoredir = os.path.join(os.getcwd(), keystoredir)
2811 if not os.path.exists(keystoredir):
2812 os.makedirs(keystoredir, mode=0o700)
2815 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
2816 'FDROID_KEY_PASS': localconfig['keypass'],
2818 p = FDroidPopen([config['keytool'], '-genkey',
2819 '-keystore', localconfig['keystore'],
2820 '-alias', localconfig['repo_keyalias'],
2821 '-keyalg', 'RSA', '-keysize', '4096',
2822 '-sigalg', 'SHA256withRSA',
2823 '-validity', '10000',
2824 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2825 '-keypass:env', 'FDROID_KEY_PASS',
2826 '-dname', localconfig['keydname']], envs=env_vars)
2827 if p.returncode != 0:
2828 raise BuildException("Failed to generate key", p.output)
2829 os.chmod(localconfig['keystore'], 0o0600)
2830 if not options.quiet:
2831 # now show the lovely key that was just generated
2832 p = FDroidPopen([config['keytool'], '-list', '-v',
2833 '-keystore', localconfig['keystore'],
2834 '-alias', localconfig['repo_keyalias'],
2835 '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
2836 logging.info(p.output.strip() + '\n\n')
2837 # get the public key
2838 p = FDroidPopenBytes([config['keytool'], '-exportcert',
2839 '-keystore', localconfig['keystore'],
2840 '-alias', localconfig['repo_keyalias'],
2841 '-storepass:env', 'FDROID_KEY_STORE_PASS']
2842 + config['smartcardoptions'],
2843 envs=env_vars, output=False, stderr_to_stdout=False)
2844 if p.returncode != 0 or len(p.output) < 20:
2845 raise BuildException("Failed to get public key", p.output)
2847 fingerprint = get_cert_fingerprint(pubkey)
2848 return hexlify(pubkey), fingerprint
2851 def get_cert_fingerprint(pubkey):
2853 Generate a certificate fingerprint the same way keytool does it
2854 (but with slightly different formatting)
2856 digest = hashlib.sha256(pubkey).digest()
2857 ret = [' '.join("%02X" % b for b in bytearray(digest))]
2858 return " ".join(ret)
2861 def get_certificate(certificate_file):
2863 Extracts a certificate from the given file.
2864 :param certificate_file: file bytes (as string) representing the certificate
2865 :return: A binary representation of the certificate's public key, or None in case of error
2867 content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
2868 if content.getComponentByName('contentType') != rfc2315.signedData:
2870 content = decoder.decode(content.getComponentByName('content'),
2871 asn1Spec=rfc2315.SignedData())[0]
2873 certificates = content.getComponentByName('certificates')
2874 cert = certificates[0].getComponentByName('certificate')
2876 logging.error("Certificates not found.")
2878 return encoder.encode(cert)
2881 def load_stats_fdroid_signing_key_fingerprints():
2882 """Load list of signing-key fingerprints stored by fdroid publish from file.
2884 :returns: list of dictionanryies containing the singing-key fingerprints.
2886 jar_file = os.path.join('stats', 'publishsigkeys.jar')
2887 if not os.path.isfile(jar_file):
2889 cmd = [config['jarsigner'], '-strict', '-verify', jar_file]
2890 p = FDroidPopen(cmd, output=False)
2891 if p.returncode != 4:
2892 raise FDroidException("Signature validation of '{}' failed! "
2893 "Please run publish again to rebuild this file.".format(jar_file))
2895 jar_sigkey = apk_signer_fingerprint(jar_file)
2896 repo_key_sig = config.get('repo_key_sha256')
2898 if jar_sigkey != repo_key_sig:
2899 raise FDroidException("Signature key fingerprint of file '{}' does not match repo_key_sha256 in config.py (found fingerprint: '{}')".format(jar_file, jar_sigkey))
2901 logging.warning("repo_key_sha256 not in config.py, setting it to the signature key fingerprint of '{}'".format(jar_file))
2902 config['repo_key_sha256'] = jar_sigkey
2903 write_to_config(config, 'repo_key_sha256')
2905 with zipfile.ZipFile(jar_file, 'r') as f:
2906 return json.loads(str(f.read('publishsigkeys.json'), 'utf-8'))
2909 def write_to_config(thisconfig, key, value=None, config_file=None):
2910 '''write a key/value to the local config.py
2912 NOTE: only supports writing string variables.
2914 :param thisconfig: config dictionary
2915 :param key: variable name in config.py to be overwritten/added
2916 :param value: optional value to be written, instead of fetched
2917 from 'thisconfig' dictionary.
2920 origkey = key + '_orig'
2921 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2922 cfg = config_file if config_file else 'config.py'
2924 # load config file, create one if it doesn't exist
2925 if not os.path.exists(cfg):
2926 open(cfg, 'a').close()
2927 logging.info("Creating empty " + cfg)
2928 with open(cfg, 'r', encoding="utf-8") as f:
2929 lines = f.readlines()
2931 # make sure the file ends with a carraige return
2933 if not lines[-1].endswith('\n'):
2936 # regex for finding and replacing python string variable
2937 # definitions/initializations
2938 pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
2939 repl = key + ' = "' + value + '"'
2940 pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
2941 repl2 = key + " = '" + value + "'"
2943 # If we replaced this line once, we make sure won't be a
2944 # second instance of this line for this key in the document.
2947 with open(cfg, 'w', encoding="utf-8") as f:
2949 if pattern.match(line) or pattern2.match(line):
2951 line = pattern.sub(repl, line)
2952 line = pattern2.sub(repl2, line)
2963 def parse_xml(path):
2964 return XMLElementTree.parse(path).getroot()
2967 def string_is_integer(string):
2975 def local_rsync(options, fromdir, todir):
2976 '''Rsync method for local to local copying of things
2978 This is an rsync wrapper with all the settings for safe use within
2979 the various fdroidserver use cases. This uses stricter rsync
2980 checking on all files since people using offline mode are already
2981 prioritizing security above ease and speed.
2984 rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
2985 '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
2986 if not options.no_checksum:
2987 rsyncargs.append('--checksum')
2989 rsyncargs += ['--verbose']
2991 rsyncargs += ['--quiet']
2992 logging.debug(' '.join(rsyncargs + [fromdir, todir]))
2993 if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
2994 raise FDroidException()
2997 def get_per_app_repos():
2998 '''per-app repos are dirs named with the packageName of a single app'''
3000 # Android packageNames are Java packages, they may contain uppercase or
3001 # lowercase letters ('A' through 'Z'), numbers, and underscores
3002 # ('_'). However, individual package name parts may only start with
3003 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
3004 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
3007 for root, dirs, files in os.walk(os.getcwd()):
3009 print('checking', root, 'for', d)
3010 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
3011 # standard parts of an fdroid repo, so never packageNames
3014 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
3020 def is_repo_file(filename):
3021 '''Whether the file in a repo is a build product to be delivered to users'''
3022 if isinstance(filename, str):
3023 filename = filename.encode('utf-8', errors="surrogateescape")
3024 return os.path.isfile(filename) \
3025 and not filename.endswith(b'.asc') \
3026 and not filename.endswith(b'.sig') \
3027 and os.path.basename(filename) not in [
3029 b'index_unsigned.jar',
3038 def get_examples_dir():
3039 '''Return the dir where the fdroidserver example files are available'''
3041 tmp = os.path.dirname(sys.argv[0])
3042 if os.path.basename(tmp) == 'bin':
3043 egg_links = glob.glob(os.path.join(tmp, '..',
3044 'local/lib/python3.*/site-packages/fdroidserver.egg-link'))
3046 # installed from local git repo
3047 examplesdir = os.path.join(open(egg_links[0]).readline().rstrip(), 'examples')
3050 examplesdir = os.path.dirname(os.path.dirname(__file__)) + '/share/doc/fdroidserver/examples'
3051 if not os.path.exists(examplesdir): # use UNIX layout
3052 examplesdir = os.path.dirname(tmp) + '/share/doc/fdroidserver/examples'
3054 # we're running straight out of the git repo
3055 prefix = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
3056 examplesdir = prefix + '/examples'
3061 def get_wiki_timestamp(timestamp=None):
3062 """Return current time in the standard format for posting to the wiki"""
3064 if timestamp is None:
3065 timestamp = time.gmtime()
3066 return time.strftime("%Y-%m-%d %H:%M:%SZ", timestamp)
3069 def get_android_tools_versions(ndk_path=None):
3070 '''get a list of the versions of all installed Android SDK/NDK components'''
3073 sdk_path = config['sdk_path']
3074 if sdk_path[-1] != '/':
3078 ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT')
3079 if os.path.isfile(ndk_release_txt):
3080 with open(ndk_release_txt, 'r') as fp:
3081 components.append((os.path.basename(ndk_path), fp.read()[:-1]))
3083 pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE)
3084 for root, dirs, files in os.walk(sdk_path):
3085 if 'source.properties' in files:
3086 source_properties = os.path.join(root, 'source.properties')
3087 with open(source_properties, 'r') as fp:
3088 m = pattern.search(fp.read())
3090 components.append((root[len(sdk_path):], m.group(1)))
3095 def get_android_tools_version_log(ndk_path=None):
3096 '''get a list of the versions of all installed Android SDK/NDK components'''
3097 log = '== Installed Android Tools ==\n\n'
3098 components = get_android_tools_versions(ndk_path)
3099 for name, version in sorted(components):
3100 log += '* ' + name + ' (' + version + ')\n'
3105 def get_git_describe_link():
3106 """Get a link to the current fdroiddata commit, to post to the wiki
3110 output = subprocess.check_output(['git', 'describe', '--always', '--dirty', '--abbrev=0'],
3111 universal_newlines=True).strip()
3112 except subprocess.CalledProcessError:
3115 commit = output.replace('-dirty', '')
3116 return ('* fdroiddata: [https://gitlab.com/fdroid/fdroiddata/commit/{commit} {id}]\n'
3117 .format(commit=commit, id=output))
3119 logging.error(_("'{path}' failed to execute!").format(path='git describe'))