3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
37 import xml.etree.ElementTree as XMLElementTree
39 from binascii import hexlify
40 from datetime import datetime
41 from distutils.version import LooseVersion
42 from queue import Queue
43 from zipfile import ZipFile
45 from pyasn1.codec.der import decoder, encoder
46 from pyasn1_modules import rfc2315
47 from pyasn1.error import PyAsn1Error
49 from distutils.util import strtobool
51 import fdroidserver.metadata
52 from fdroidserver import _
53 from fdroidserver.exception import FDroidException, VCSException, BuildException
54 from .asynchronousfilereader import AsynchronousFileReader
57 # A signature block file with a .DSA, .RSA, or .EC extension
58 CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
59 APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
60 STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
62 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
71 'sdk_path': "$ANDROID_HOME",
76 'r12b': "$ANDROID_NDK",
82 'build_tools': "25.0.2",
83 'force_build_tools': False,
88 'accepted_formats': ['txt', 'yml'],
89 'sync_from_local_copy_dir': False,
90 'allow_disabled_algorithms': False,
91 'per_app_repos': False,
92 'make_current_version_link': True,
93 'current_version_name_source': 'Name',
94 'update_stats': False,
98 'stats_to_carbon': False,
100 'build_server_always': False,
101 'keystore': 'keystore.jks',
102 'smartcardoptions': [],
112 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
113 'repo_name': "My First FDroid Repo Demo",
114 'repo_icon': "fdroid-icon.png",
115 'repo_description': '''
116 This is a repository of apps to be used with FDroid. Applications in this
117 repository are either official binaries built by the original application
118 developers, or are binaries built from source by the admin of f-droid.org
119 using the tools on https://gitlab.com/u/fdroid.
125 def setup_global_opts(parser):
126 parser.add_argument("-v", "--verbose", action="store_true", default=False,
127 help=_("Spew out even more information than normal"))
128 parser.add_argument("-q", "--quiet", action="store_true", default=False,
129 help=_("Restrict output to warnings and errors"))
132 def fill_config_defaults(thisconfig):
133 for k, v in default_config.items():
134 if k not in thisconfig:
137 # Expand paths (~users and $vars)
138 def expand_path(path):
142 path = os.path.expanduser(path)
143 path = os.path.expandvars(path)
148 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
153 thisconfig[k + '_orig'] = v
155 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
156 if thisconfig['java_paths'] is None:
157 thisconfig['java_paths'] = dict()
159 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
160 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
161 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
162 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
163 if os.getenv('JAVA_HOME') is not None:
164 pathlist.append(os.getenv('JAVA_HOME'))
165 if os.getenv('PROGRAMFILES') is not None:
166 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
167 for d in sorted(pathlist):
168 if os.path.islink(d):
170 j = os.path.basename(d)
171 # the last one found will be the canonical one, so order appropriately
173 r'^1\.([6-9])\.0\.jdk$', # OSX
174 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
175 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
176 r'^jdk([6-9])-openjdk$', # Arch
177 r'^java-([6-9])-openjdk$', # Arch
178 r'^java-([6-9])-jdk$', # Arch (oracle)
179 r'^java-1\.([6-9])\.0-.*$', # RedHat
180 r'^java-([6-9])-oracle$', # Debian WebUpd8
181 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
182 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
184 m = re.match(regex, j)
187 for p in [d, os.path.join(d, 'Contents', 'Home')]:
188 if os.path.exists(os.path.join(p, 'bin', 'javac')):
189 thisconfig['java_paths'][m.group(1)] = p
191 for java_version in ('7', '8', '9'):
192 if java_version not in thisconfig['java_paths']:
194 java_home = thisconfig['java_paths'][java_version]
195 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
196 if os.path.exists(jarsigner):
197 thisconfig['jarsigner'] = jarsigner
198 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
199 break # Java7 is preferred, so quit if found
201 for k in ['ndk_paths', 'java_paths']:
207 thisconfig[k][k2] = exp
208 thisconfig[k][k2 + '_orig'] = v
211 def regsub_file(pattern, repl, path):
212 with open(path, 'rb') as f:
214 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
215 with open(path, 'wb') as f:
219 def read_config(opts, config_file='config.py'):
220 """Read the repository config
222 The config is read from config_file, which is in the current
223 directory when any of the repo management commands are used. If
224 there is a local metadata file in the git repo, then config.py is
225 not required, just use defaults.
228 global config, options
230 if config is not None:
237 if os.path.isfile(config_file):
238 logging.debug("Reading %s" % config_file)
239 with io.open(config_file, "rb") as f:
240 code = compile(f.read(), config_file, 'exec')
241 exec(code, None, config)
243 logging.debug("No config.py found - using defaults.")
245 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
247 if not type(config[k]) in (str, list, tuple):
248 logging.warn('"' + k + '" will be in random order!'
249 + ' Use () or [] brackets if order is important!')
251 # smartcardoptions must be a list since its command line args for Popen
252 if 'smartcardoptions' in config:
253 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
254 elif 'keystore' in config and config['keystore'] == 'NONE':
255 # keystore='NONE' means use smartcard, these are required defaults
256 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
257 'SunPKCS11-OpenSC', '-providerClass',
258 'sun.security.pkcs11.SunPKCS11',
259 '-providerArg', 'opensc-fdroid.cfg']
261 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
262 st = os.stat(config_file)
263 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
264 logging.warning("unsafe permissions on {0} (should be 0600)!".format(config_file))
266 fill_config_defaults(config)
268 for k in ["repo_description", "archive_description"]:
270 config[k] = clean_description(config[k])
272 if 'serverwebroot' in config:
273 if isinstance(config['serverwebroot'], str):
274 roots = [config['serverwebroot']]
275 elif all(isinstance(item, str) for item in config['serverwebroot']):
276 roots = config['serverwebroot']
278 raise TypeError('only accepts strings, lists, and tuples')
280 for rootstr in roots:
281 # since this is used with rsync, where trailing slashes have
282 # meaning, ensure there is always a trailing slash
283 if rootstr[-1] != '/':
285 rootlist.append(rootstr.replace('//', '/'))
286 config['serverwebroot'] = rootlist
288 if 'servergitmirrors' in config:
289 if isinstance(config['servergitmirrors'], str):
290 roots = [config['servergitmirrors']]
291 elif all(isinstance(item, str) for item in config['servergitmirrors']):
292 roots = config['servergitmirrors']
294 raise TypeError('only accepts strings, lists, and tuples')
295 config['servergitmirrors'] = roots
300 def find_sdk_tools_cmd(cmd):
301 '''find a working path to a tool from the Android SDK'''
304 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
305 # try to find a working path to this command, in all the recent possible paths
306 if 'build_tools' in config:
307 build_tools = os.path.join(config['sdk_path'], 'build-tools')
308 # if 'build_tools' was manually set and exists, check only that one
309 configed_build_tools = os.path.join(build_tools, config['build_tools'])
310 if os.path.exists(configed_build_tools):
311 tooldirs.append(configed_build_tools)
313 # no configed version, so hunt known paths for it
314 for f in sorted(os.listdir(build_tools), reverse=True):
315 if os.path.isdir(os.path.join(build_tools, f)):
316 tooldirs.append(os.path.join(build_tools, f))
317 tooldirs.append(build_tools)
318 sdk_tools = os.path.join(config['sdk_path'], 'tools')
319 if os.path.exists(sdk_tools):
320 tooldirs.append(sdk_tools)
321 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
322 if os.path.exists(sdk_platform_tools):
323 tooldirs.append(sdk_platform_tools)
324 tooldirs.append('/usr/bin')
326 path = os.path.join(d, cmd)
327 if os.path.isfile(path):
329 test_aapt_version(path)
331 # did not find the command, exit with error message
332 ensure_build_tools_exists(config)
335 def test_aapt_version(aapt):
336 '''Check whether the version of aapt is new enough'''
337 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
338 if output is None or output == '':
339 logging.error(aapt + ' failed to execute!')
341 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
346 # the Debian package has the version string like "v0.2-23.0.2"
347 if '.' not in bugfix and LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.2166767'):
348 logging.warning(aapt + ' is too old, fdroid requires build-tools-23.0.0 or newer!')
350 logging.warning('Unknown version of aapt, might cause problems: ' + output)
353 def test_sdk_exists(thisconfig):
354 if 'sdk_path' not in thisconfig:
355 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
356 test_aapt_version(thisconfig['aapt'])
359 logging.error("'sdk_path' not set in config.py!")
361 if thisconfig['sdk_path'] == default_config['sdk_path']:
362 logging.error('No Android SDK found!')
363 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
364 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
366 if not os.path.exists(thisconfig['sdk_path']):
367 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
369 if not os.path.isdir(thisconfig['sdk_path']):
370 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
372 for d in ['build-tools', 'platform-tools', 'tools']:
373 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
374 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
375 thisconfig['sdk_path'], d))
380 def ensure_build_tools_exists(thisconfig):
381 if not test_sdk_exists(thisconfig):
382 raise FDroidException("Android SDK not found.")
383 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
384 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
385 if not os.path.isdir(versioned_build_tools):
386 raise FDroidException(
387 'Android Build Tools path "' + versioned_build_tools + '" does not exist!')
390 def get_local_metadata_files():
391 '''get any metadata files local to an app's source repo
393 This tries to ignore anything that does not count as app metdata,
394 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
397 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
400 def read_pkg_args(args, allow_vercodes=False):
402 :param args: arguments in the form of multiple appid:[vc] strings
403 :returns: a dictionary with the set of vercodes specified for each package
411 if allow_vercodes and ':' in p:
412 package, vercode = p.split(':')
414 package, vercode = p, None
415 if package not in vercodes:
416 vercodes[package] = [vercode] if vercode else []
418 elif vercode and vercode not in vercodes[package]:
419 vercodes[package] += [vercode] if vercode else []
424 def read_app_args(args, allapps, allow_vercodes=False):
426 On top of what read_pkg_args does, this returns the whole app metadata, but
427 limiting the builds list to the builds matching the vercodes specified.
430 vercodes = read_pkg_args(args, allow_vercodes)
436 for appid, app in allapps.items():
437 if appid in vercodes:
440 if len(apps) != len(vercodes):
443 logging.critical("No such package: %s" % p)
444 raise FDroidException("Found invalid app ids in arguments")
446 raise FDroidException("No packages specified")
449 for appid, app in apps.items():
453 app.builds = [b for b in app.builds if b.versionCode in vc]
454 if len(app.builds) != len(vercodes[appid]):
456 allvcs = [b.versionCode for b in app.builds]
457 for v in vercodes[appid]:
459 logging.critical("No such vercode %s for app %s" % (v, appid))
462 raise FDroidException("Found invalid vercodes for some apps")
467 def get_extension(filename):
468 base, ext = os.path.splitext(filename)
471 return base, ext.lower()[1:]
474 def has_extension(filename, ext):
475 _, f_ext = get_extension(filename)
479 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
482 def clean_description(description):
483 'Remove unneeded newlines and spaces from a block of description text'
485 # this is split up by paragraph to make removing the newlines easier
486 for paragraph in re.split(r'\n\n', description):
487 paragraph = re.sub('\r', '', paragraph)
488 paragraph = re.sub('\n', ' ', paragraph)
489 paragraph = re.sub(' {2,}', ' ', paragraph)
490 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
491 returnstring += paragraph + '\n\n'
492 return returnstring.rstrip('\n')
495 def publishednameinfo(filename):
496 filename = os.path.basename(filename)
497 m = publish_name_regex.match(filename)
499 result = (m.group(1), m.group(2))
500 except AttributeError:
501 raise FDroidException("Invalid name for published file: %s" % filename)
505 def get_release_filename(app, build):
507 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
509 return "%s_%s.apk" % (app.id, build.versionCode)
512 def get_toolsversion_logname(app, build):
513 return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
516 def getsrcname(app, build):
517 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
529 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
532 def get_build_dir(app):
533 '''get the dir that this app will be built in'''
535 if app.RepoType == 'srclib':
536 return os.path.join('build', 'srclib', app.Repo)
538 return os.path.join('build', app.id)
542 '''checkout code from VCS and return instance of vcs and the build dir'''
543 build_dir = get_build_dir(app)
545 # Set up vcs interface and make sure we have the latest code...
546 logging.debug("Getting {0} vcs interface for {1}"
547 .format(app.RepoType, app.Repo))
548 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
552 vcs = getvcs(app.RepoType, remote, build_dir)
554 return vcs, build_dir
557 def getvcs(vcstype, remote, local):
559 return vcs_git(remote, local)
560 if vcstype == 'git-svn':
561 return vcs_gitsvn(remote, local)
563 return vcs_hg(remote, local)
565 return vcs_bzr(remote, local)
566 if vcstype == 'srclib':
567 if local != os.path.join('build', 'srclib', remote):
568 raise VCSException("Error: srclib paths are hard-coded!")
569 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
571 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
572 raise VCSException("Invalid vcs type " + vcstype)
575 def getsrclibvcs(name):
576 if name not in fdroidserver.metadata.srclibs:
577 raise VCSException("Missing srclib " + name)
578 return fdroidserver.metadata.srclibs[name]['Repo Type']
583 def __init__(self, remote, local):
585 # svn, git-svn and bzr may require auth
587 if self.repotype() in ('git-svn', 'bzr'):
589 if self.repotype == 'git-svn':
590 raise VCSException("Authentication is not supported for git-svn")
591 self.username, remote = remote.split('@')
592 if ':' not in self.username:
593 raise VCSException("Password required with username")
594 self.username, self.password = self.username.split(':')
598 self.clone_failed = False
599 self.refreshed = False
605 # Take the local repository to a clean version of the given revision, which
606 # is specificed in the VCS's native format. Beforehand, the repository can
607 # be dirty, or even non-existent. If the repository does already exist
608 # locally, it will be updated from the origin, but only once in the
609 # lifetime of the vcs object.
610 # None is acceptable for 'rev' if you know you are cloning a clean copy of
611 # the repo - otherwise it must specify a valid revision.
612 def gotorevision(self, rev, refresh=True):
614 if self.clone_failed:
615 raise VCSException("Downloading the repository already failed once, not trying again.")
617 # The .fdroidvcs-id file for a repo tells us what VCS type
618 # and remote that directory was created from, allowing us to drop it
619 # automatically if either of those things changes.
620 fdpath = os.path.join(self.local, '..',
621 '.fdroidvcs-' + os.path.basename(self.local))
622 fdpath = os.path.normpath(fdpath)
623 cdata = self.repotype() + ' ' + self.remote
626 if os.path.exists(self.local):
627 if os.path.exists(fdpath):
628 with open(fdpath, 'r') as f:
629 fsdata = f.read().strip()
634 logging.info("Repository details for %s changed - deleting" % (
638 logging.info("Repository details for %s missing - deleting" % (
641 shutil.rmtree(self.local)
645 self.refreshed = True
648 self.gotorevisionx(rev)
649 except FDroidException as e:
652 # If necessary, write the .fdroidvcs file.
653 if writeback and not self.clone_failed:
654 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
655 with open(fdpath, 'w+') as f:
661 # Derived classes need to implement this. It's called once basic checking
662 # has been performend.
663 def gotorevisionx(self, rev): # pylint: disable=unused-argument
664 raise VCSException("This VCS type doesn't define gotorevisionx")
666 # Initialise and update submodules
667 def initsubmodules(self):
668 raise VCSException('Submodules not supported for this vcs type')
670 # Get a list of all known tags
672 if not self._gettags:
673 raise VCSException('gettags not supported for this vcs type')
675 for tag in self._gettags():
676 if re.match('[-A-Za-z0-9_. /]+$', tag):
680 # Get a list of all the known tags, sorted from newest to oldest
681 def latesttags(self):
682 raise VCSException('latesttags not supported for this vcs type')
684 # Get current commit reference (hash, revision, etc)
686 raise VCSException('getref not supported for this vcs type')
688 # Returns the srclib (name, path) used in setting up the current
699 # If the local directory exists, but is somehow not a git repository, git
700 # will traverse up the directory tree until it finds one that is (i.e.
701 # fdroidserver) and then we'll proceed to destroy it! This is called as
704 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
705 result = p.output.rstrip()
706 if not result.endswith(self.local):
707 raise VCSException('Repository mismatch')
709 def gotorevisionx(self, rev):
710 if not os.path.exists(self.local):
712 p = FDroidPopen(['git', 'clone', self.remote, self.local])
713 if p.returncode != 0:
714 self.clone_failed = True
715 raise VCSException("Git clone failed", p.output)
719 # Discard any working tree changes
720 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
721 'git', 'reset', '--hard'], cwd=self.local, output=False)
722 if p.returncode != 0:
723 raise VCSException("Git reset failed", p.output)
724 # Remove untracked files now, in case they're tracked in the target
725 # revision (it happens!)
726 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
727 'git', 'clean', '-dffx'], cwd=self.local, output=False)
728 if p.returncode != 0:
729 raise VCSException("Git clean failed", p.output)
730 if not self.refreshed:
731 # Get latest commits and tags from remote
732 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
733 if p.returncode != 0:
734 raise VCSException("Git fetch failed", p.output)
735 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
736 if p.returncode != 0:
737 raise VCSException("Git fetch failed", p.output)
738 # Recreate origin/HEAD as git clone would do it, in case it disappeared
739 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
740 if p.returncode != 0:
741 lines = p.output.splitlines()
742 if 'Multiple remote HEAD branches' not in lines[0]:
743 raise VCSException("Git remote set-head failed", p.output)
744 branch = lines[1].split(' ')[-1]
745 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
746 if p2.returncode != 0:
747 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
748 self.refreshed = True
749 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
750 # a github repo. Most of the time this is the same as origin/master.
751 rev = rev or 'origin/HEAD'
752 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
753 if p.returncode != 0:
754 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
755 # Get rid of any uncontrolled files left behind
756 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
757 if p.returncode != 0:
758 raise VCSException("Git clean failed", p.output)
760 def initsubmodules(self):
762 submfile = os.path.join(self.local, '.gitmodules')
763 if not os.path.isfile(submfile):
764 raise VCSException("No git submodules available")
766 # fix submodules not accessible without an account and public key auth
767 with open(submfile, 'r') as f:
768 lines = f.readlines()
769 with open(submfile, 'w') as f:
771 if 'git@github.com' in line:
772 line = line.replace('git@github.com:', 'https://github.com/')
773 if 'git@gitlab.com' in line:
774 line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
777 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
778 if p.returncode != 0:
779 raise VCSException("Git submodule sync failed", p.output)
780 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
781 if p.returncode != 0:
782 raise VCSException("Git submodule update failed", p.output)
786 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
787 return p.output.splitlines()
789 tag_format = re.compile(r'tag: ([^),]*)')
791 def latesttags(self):
793 p = FDroidPopen(['git', 'log', '--tags',
794 '--simplify-by-decoration', '--pretty=format:%d'],
795 cwd=self.local, output=False)
797 for line in p.output.splitlines():
798 for tag in self.tag_format.findall(line):
803 class vcs_gitsvn(vcs):
808 # If the local directory exists, but is somehow not a git repository, git
809 # will traverse up the directory tree until it finds one that is (i.e.
810 # fdroidserver) and then we'll proceed to destory it! This is called as
813 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
814 result = p.output.rstrip()
815 if not result.endswith(self.local):
816 raise VCSException('Repository mismatch')
818 def gotorevisionx(self, rev):
819 if not os.path.exists(self.local):
821 gitsvn_args = ['git', 'svn', 'clone']
822 if ';' in self.remote:
823 remote_split = self.remote.split(';')
824 for i in remote_split[1:]:
825 if i.startswith('trunk='):
826 gitsvn_args.extend(['-T', i[6:]])
827 elif i.startswith('tags='):
828 gitsvn_args.extend(['-t', i[5:]])
829 elif i.startswith('branches='):
830 gitsvn_args.extend(['-b', i[9:]])
831 gitsvn_args.extend([remote_split[0], self.local])
832 p = FDroidPopen(gitsvn_args, output=False)
833 if p.returncode != 0:
834 self.clone_failed = True
835 raise VCSException("Git svn clone failed", p.output)
837 gitsvn_args.extend([self.remote, self.local])
838 p = FDroidPopen(gitsvn_args, output=False)
839 if p.returncode != 0:
840 self.clone_failed = True
841 raise VCSException("Git svn clone failed", p.output)
845 # Discard any working tree changes
846 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
847 if p.returncode != 0:
848 raise VCSException("Git reset failed", p.output)
849 # Remove untracked files now, in case they're tracked in the target
850 # revision (it happens!)
851 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
852 if p.returncode != 0:
853 raise VCSException("Git clean failed", p.output)
854 if not self.refreshed:
855 # Get new commits, branches and tags from repo
856 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
857 if p.returncode != 0:
858 raise VCSException("Git svn fetch failed")
859 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
860 if p.returncode != 0:
861 raise VCSException("Git svn rebase failed", p.output)
862 self.refreshed = True
864 rev = rev or 'master'
866 nospaces_rev = rev.replace(' ', '%20')
867 # Try finding a svn tag
868 for treeish in ['origin/', '']:
869 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
870 if p.returncode == 0:
872 if p.returncode != 0:
873 # No tag found, normal svn rev translation
874 # Translate svn rev into git format
875 rev_split = rev.split('/')
878 for treeish in ['origin/', '']:
879 if len(rev_split) > 1:
880 treeish += rev_split[0]
881 svn_rev = rev_split[1]
884 # if no branch is specified, then assume trunk (i.e. 'master' branch):
888 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
890 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
891 git_rev = p.output.rstrip()
893 if p.returncode == 0 and git_rev:
896 if p.returncode != 0 or not git_rev:
897 # Try a plain git checkout as a last resort
898 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
899 if p.returncode != 0:
900 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
902 # Check out the git rev equivalent to the svn rev
903 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
904 if p.returncode != 0:
905 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
907 # Get rid of any uncontrolled files left behind
908 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
909 if p.returncode != 0:
910 raise VCSException("Git clean failed", p.output)
914 for treeish in ['origin/', '']:
915 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
921 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
922 if p.returncode != 0:
924 return p.output.strip()
932 def gotorevisionx(self, rev):
933 if not os.path.exists(self.local):
934 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
935 if p.returncode != 0:
936 self.clone_failed = True
937 raise VCSException("Hg clone failed", p.output)
939 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
940 if p.returncode != 0:
941 raise VCSException("Hg status failed", p.output)
942 for line in p.output.splitlines():
943 if not line.startswith('? '):
944 raise VCSException("Unexpected output from hg status -uS: " + line)
945 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
946 if not self.refreshed:
947 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
948 if p.returncode != 0:
949 raise VCSException("Hg pull failed", p.output)
950 self.refreshed = True
952 rev = rev or 'default'
955 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
956 if p.returncode != 0:
957 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
958 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
959 # Also delete untracked files, we have to enable purge extension for that:
960 if "'purge' is provided by the following extension" in p.output:
961 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
962 myfile.write("\n[extensions]\nhgext.purge=\n")
963 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
964 if p.returncode != 0:
965 raise VCSException("HG purge failed", p.output)
966 elif p.returncode != 0:
967 raise VCSException("HG purge failed", p.output)
970 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
971 return p.output.splitlines()[1:]
979 def gotorevisionx(self, rev):
980 if not os.path.exists(self.local):
981 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
982 if p.returncode != 0:
983 self.clone_failed = True
984 raise VCSException("Bzr branch failed", p.output)
986 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
987 if p.returncode != 0:
988 raise VCSException("Bzr revert failed", p.output)
989 if not self.refreshed:
990 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
991 if p.returncode != 0:
992 raise VCSException("Bzr update failed", p.output)
993 self.refreshed = True
995 revargs = list(['-r', rev] if rev else [])
996 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
997 if p.returncode != 0:
998 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1001 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
1002 return [tag.split(' ')[0].strip() for tag in
1003 p.output.splitlines()]
1006 def unescape_string(string):
1009 if string[0] == '"' and string[-1] == '"':
1012 return string.replace("\\'", "'")
1015 def retrieve_string(app_dir, string, xmlfiles=None):
1017 if not string.startswith('@string/'):
1018 return unescape_string(string)
1020 if xmlfiles is None:
1023 os.path.join(app_dir, 'res'),
1024 os.path.join(app_dir, 'src', 'main', 'res'),
1026 for root, dirs, files in os.walk(res_dir):
1027 if os.path.basename(root) == 'values':
1028 xmlfiles += [os.path.join(root, x) for x in files if x.endswith('.xml')]
1030 name = string[len('@string/'):]
1032 def element_content(element):
1033 if element.text is None:
1035 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1036 return s.decode('utf-8').strip()
1038 for path in xmlfiles:
1039 if not os.path.isfile(path):
1041 xml = parse_xml(path)
1042 element = xml.find('string[@name="' + name + '"]')
1043 if element is not None:
1044 content = element_content(element)
1045 return retrieve_string(app_dir, content, xmlfiles)
1050 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1051 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1054 def manifest_paths(app_dir, flavours):
1055 '''Return list of existing files that will be used to find the highest vercode'''
1057 possible_manifests = \
1058 [os.path.join(app_dir, 'AndroidManifest.xml'),
1059 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1060 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1061 os.path.join(app_dir, 'build.gradle')]
1063 for flavour in flavours:
1064 if flavour == 'yes':
1066 possible_manifests.append(
1067 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1069 return [path for path in possible_manifests if os.path.isfile(path)]
1072 def fetch_real_name(app_dir, flavours):
1073 '''Retrieve the package name. Returns the name, or None if not found.'''
1074 for path in manifest_paths(app_dir, flavours):
1075 if not has_extension(path, 'xml') or not os.path.isfile(path):
1077 logging.debug("fetch_real_name: Checking manifest at " + path)
1078 xml = parse_xml(path)
1079 app = xml.find('application')
1082 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1084 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1085 result = retrieve_string_singleline(app_dir, label)
1087 result = result.strip()
1092 def get_library_references(root_dir):
1094 proppath = os.path.join(root_dir, 'project.properties')
1095 if not os.path.isfile(proppath):
1097 with open(proppath, 'r', encoding='iso-8859-1') as f:
1099 if not line.startswith('android.library.reference.'):
1101 path = line.split('=')[1].strip()
1102 relpath = os.path.join(root_dir, path)
1103 if not os.path.isdir(relpath):
1105 logging.debug("Found subproject at %s" % path)
1106 libraries.append(path)
1110 def ant_subprojects(root_dir):
1111 subprojects = get_library_references(root_dir)
1112 for subpath in subprojects:
1113 subrelpath = os.path.join(root_dir, subpath)
1114 for p in get_library_references(subrelpath):
1115 relp = os.path.normpath(os.path.join(subpath, p))
1116 if relp not in subprojects:
1117 subprojects.insert(0, relp)
1121 def remove_debuggable_flags(root_dir):
1122 # Remove forced debuggable flags
1123 logging.debug("Removing debuggable flags from %s" % root_dir)
1124 for root, dirs, files in os.walk(root_dir):
1125 if 'AndroidManifest.xml' in files:
1126 regsub_file(r'android:debuggable="[^"]*"',
1128 os.path.join(root, 'AndroidManifest.xml'))
1131 vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search
1132 vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1133 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1136 def app_matches_packagename(app, package):
1139 appid = app.UpdateCheckName or app.id
1140 if appid is None or appid == "Ignore":
1142 return appid == package
1145 def parse_androidmanifests(paths, app):
1147 Extract some information from the AndroidManifest.xml at the given path.
1148 Returns (version, vercode, package), any or all of which might be None.
1149 All values returned are strings.
1152 ignoreversions = app.UpdateCheckIgnore
1153 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1156 return (None, None, None)
1164 if not os.path.isfile(path):
1167 logging.debug("Parsing manifest at {0}".format(path))
1172 if has_extension(path, 'gradle'):
1173 with open(path, 'r') as f:
1175 if gradle_comment.match(line):
1177 # Grab first occurence of each to avoid running into
1178 # alternative flavours and builds.
1180 matches = psearch_g(line)
1182 s = matches.group(2)
1183 if app_matches_packagename(app, s):
1186 matches = vnsearch_g(line)
1188 version = matches.group(2)
1190 matches = vcsearch_g(line)
1192 vercode = matches.group(1)
1195 xml = parse_xml(path)
1196 if "package" in xml.attrib:
1197 s = xml.attrib["package"]
1198 if app_matches_packagename(app, s):
1200 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1201 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1202 base_dir = os.path.dirname(path)
1203 version = retrieve_string_singleline(base_dir, version)
1204 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1205 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1206 if string_is_integer(a):
1209 logging.warning("Problem with xml at {0}".format(path))
1211 # Remember package name, may be defined separately from version+vercode
1213 package = max_package
1215 logging.debug("..got package={0}, version={1}, vercode={2}"
1216 .format(package, version, vercode))
1218 # Always grab the package name and version name in case they are not
1219 # together with the highest version code
1220 if max_package is None and package is not None:
1221 max_package = package
1222 if max_version is None and version is not None:
1223 max_version = version
1225 if vercode is not None \
1226 and (max_vercode is None or vercode > max_vercode):
1227 if not ignoresearch or not ignoresearch(version):
1228 if version is not None:
1229 max_version = version
1230 if vercode is not None:
1231 max_vercode = vercode
1232 if package is not None:
1233 max_package = package
1235 max_version = "Ignore"
1237 if max_version is None:
1238 max_version = "Unknown"
1240 if max_package and not is_valid_package_name(max_package):
1241 raise FDroidException("Invalid package name {0}".format(max_package))
1243 return (max_version, max_vercode, max_package)
1246 def is_valid_package_name(name):
1247 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1250 # Get the specified source library.
1251 # Returns the path to it. Normally this is the path to be used when referencing
1252 # it, which may be a subdirectory of the actual project. If you want the base
1253 # directory of the project, pass 'basepath=True'.
1254 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1255 raw=False, prepare=True, preponly=False, refresh=True,
1264 name, ref = spec.split('@')
1266 number, name = name.split(':', 1)
1268 name, subdir = name.split('/', 1)
1270 if name not in fdroidserver.metadata.srclibs:
1271 raise VCSException('srclib ' + name + ' not found.')
1273 srclib = fdroidserver.metadata.srclibs[name]
1275 sdir = os.path.join(srclib_dir, name)
1278 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1279 vcs.srclib = (name, number, sdir)
1281 vcs.gotorevision(ref, refresh)
1288 libdir = os.path.join(sdir, subdir)
1289 elif srclib["Subdir"]:
1290 for subdir in srclib["Subdir"]:
1291 libdir_candidate = os.path.join(sdir, subdir)
1292 if os.path.exists(libdir_candidate):
1293 libdir = libdir_candidate
1299 remove_signing_keys(sdir)
1300 remove_debuggable_flags(sdir)
1304 if srclib["Prepare"]:
1305 cmd = replace_config_vars(srclib["Prepare"], build)
1307 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1308 if p.returncode != 0:
1309 raise BuildException("Error running prepare command for srclib %s"
1315 return (name, number, libdir)
1318 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1321 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1322 """ Prepare the source code for a particular build
1324 :param vcs: the appropriate vcs object for the application
1325 :param app: the application details from the metadata
1326 :param build: the build details from the metadata
1327 :param build_dir: the path to the build directory, usually 'build/app.id'
1328 :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
1329 :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
1331 Returns the (root, srclibpaths) where:
1332 :param root: is the root directory, which may be the same as 'build_dir' or may
1333 be a subdirectory of it.
1334 :param srclibpaths: is information on the srclibs being used
1337 # Optionally, the actual app source can be in a subdirectory
1339 root_dir = os.path.join(build_dir, build.subdir)
1341 root_dir = build_dir
1343 # Get a working copy of the right revision
1344 logging.info("Getting source for revision " + build.commit)
1345 vcs.gotorevision(build.commit, refresh)
1347 # Initialise submodules if required
1348 if build.submodules:
1349 logging.info("Initialising submodules")
1350 vcs.initsubmodules()
1352 # Check that a subdir (if we're using one) exists. This has to happen
1353 # after the checkout, since it might not exist elsewhere
1354 if not os.path.exists(root_dir):
1355 raise BuildException('Missing subdir ' + root_dir)
1357 # Run an init command if one is required
1359 cmd = replace_config_vars(build.init, build)
1360 logging.info("Running 'init' commands in %s" % root_dir)
1362 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1363 if p.returncode != 0:
1364 raise BuildException("Error running init command for %s:%s" %
1365 (app.id, build.versionName), p.output)
1367 # Apply patches if any
1369 logging.info("Applying patches")
1370 for patch in build.patch:
1371 patch = patch.strip()
1372 logging.info("Applying " + patch)
1373 patch_path = os.path.join('metadata', app.id, patch)
1374 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1375 if p.returncode != 0:
1376 raise BuildException("Failed to apply patch %s" % patch_path)
1378 # Get required source libraries
1381 logging.info("Collecting source libraries")
1382 for lib in build.srclibs:
1383 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1384 refresh=refresh, build=build))
1386 for name, number, libpath in srclibpaths:
1387 place_srclib(root_dir, int(number) if number else None, libpath)
1389 basesrclib = vcs.getsrclib()
1390 # If one was used for the main source, add that too.
1392 srclibpaths.append(basesrclib)
1394 # Update the local.properties file
1395 localprops = [os.path.join(build_dir, 'local.properties')]
1397 parts = build.subdir.split(os.sep)
1400 cur = os.path.join(cur, d)
1401 localprops += [os.path.join(cur, 'local.properties')]
1402 for path in localprops:
1404 if os.path.isfile(path):
1405 logging.info("Updating local.properties file at %s" % path)
1406 with open(path, 'r', encoding='iso-8859-1') as f:
1410 logging.info("Creating local.properties file at %s" % path)
1411 # Fix old-fashioned 'sdk-location' by copying
1412 # from sdk.dir, if necessary
1414 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1415 re.S | re.M).group(1)
1416 props += "sdk-location=%s\n" % sdkloc
1418 props += "sdk.dir=%s\n" % config['sdk_path']
1419 props += "sdk-location=%s\n" % config['sdk_path']
1420 ndk_path = build.ndk_path()
1421 # if for any reason the path isn't valid or the directory
1422 # doesn't exist, some versions of Gradle will error with a
1423 # cryptic message (even if the NDK is not even necessary).
1424 # https://gitlab.com/fdroid/fdroidserver/issues/171
1425 if ndk_path and os.path.exists(ndk_path):
1427 props += "ndk.dir=%s\n" % ndk_path
1428 props += "ndk-location=%s\n" % ndk_path
1429 # Add java.encoding if necessary
1431 props += "java.encoding=%s\n" % build.encoding
1432 with open(path, 'w', encoding='iso-8859-1') as f:
1436 if build.build_method() == 'gradle':
1437 flavours = build.gradle
1440 n = build.target.split('-')[1]
1441 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1442 r'compileSdkVersion %s' % n,
1443 os.path.join(root_dir, 'build.gradle'))
1445 # Remove forced debuggable flags
1446 remove_debuggable_flags(root_dir)
1448 # Insert version code and number into the manifest if necessary
1449 if build.forceversion:
1450 logging.info("Changing the version name")
1451 for path in manifest_paths(root_dir, flavours):
1452 if not os.path.isfile(path):
1454 if has_extension(path, 'xml'):
1455 regsub_file(r'android:versionName="[^"]*"',
1456 r'android:versionName="%s"' % build.versionName,
1458 elif has_extension(path, 'gradle'):
1459 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1460 r"""\1versionName '%s'""" % build.versionName,
1463 if build.forcevercode:
1464 logging.info("Changing the version code")
1465 for path in manifest_paths(root_dir, flavours):
1466 if not os.path.isfile(path):
1468 if has_extension(path, 'xml'):
1469 regsub_file(r'android:versionCode="[^"]*"',
1470 r'android:versionCode="%s"' % build.versionCode,
1472 elif has_extension(path, 'gradle'):
1473 regsub_file(r'versionCode[ =]+[0-9]+',
1474 r'versionCode %s' % build.versionCode,
1477 # Delete unwanted files
1479 logging.info("Removing specified files")
1480 for part in getpaths(build_dir, build.rm):
1481 dest = os.path.join(build_dir, part)
1482 logging.info("Removing {0}".format(part))
1483 if os.path.lexists(dest):
1484 if os.path.islink(dest):
1485 FDroidPopen(['unlink', dest], output=False)
1487 FDroidPopen(['rm', '-rf', dest], output=False)
1489 logging.info("...but it didn't exist")
1491 remove_signing_keys(build_dir)
1493 # Add required external libraries
1495 logging.info("Collecting prebuilt libraries")
1496 libsdir = os.path.join(root_dir, 'libs')
1497 if not os.path.exists(libsdir):
1499 for lib in build.extlibs:
1501 logging.info("...installing extlib {0}".format(lib))
1502 libf = os.path.basename(lib)
1503 libsrc = os.path.join(extlib_dir, lib)
1504 if not os.path.exists(libsrc):
1505 raise BuildException("Missing extlib file {0}".format(libsrc))
1506 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1508 # Run a pre-build command if one is required
1510 logging.info("Running 'prebuild' commands in %s" % root_dir)
1512 cmd = replace_config_vars(build.prebuild, build)
1514 # Substitute source library paths into prebuild commands
1515 for name, number, libpath in srclibpaths:
1516 libpath = os.path.relpath(libpath, root_dir)
1517 cmd = cmd.replace('$$' + name + '$$', libpath)
1519 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1520 if p.returncode != 0:
1521 raise BuildException("Error running prebuild command for %s:%s" %
1522 (app.id, build.versionName), p.output)
1524 # Generate (or update) the ant build file, build.xml...
1525 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1526 parms = ['android', 'update', 'lib-project']
1527 lparms = ['android', 'update', 'project']
1530 parms += ['-t', build.target]
1531 lparms += ['-t', build.target]
1532 if build.androidupdate:
1533 update_dirs = build.androidupdate
1535 update_dirs = ant_subprojects(root_dir) + ['.']
1537 for d in update_dirs:
1538 subdir = os.path.join(root_dir, d)
1540 logging.debug("Updating main project")
1541 cmd = parms + ['-p', d]
1543 logging.debug("Updating subproject %s" % d)
1544 cmd = lparms + ['-p', d]
1545 p = SdkToolsPopen(cmd, cwd=root_dir)
1546 # Check to see whether an error was returned without a proper exit
1547 # code (this is the case for the 'no target set or target invalid'
1549 if p.returncode != 0 or p.output.startswith("Error: "):
1550 raise BuildException("Failed to update project at %s" % d, p.output)
1551 # Clean update dirs via ant
1553 logging.info("Cleaning subproject %s" % d)
1554 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1556 return (root_dir, srclibpaths)
1559 # Extend via globbing the paths from a field and return them as a map from
1560 # original path to resulting paths
1561 def getpaths_map(build_dir, globpaths):
1565 full_path = os.path.join(build_dir, p)
1566 full_path = os.path.normpath(full_path)
1567 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1569 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1573 # Extend via globbing the paths from a field and return them as a set
1574 def getpaths(build_dir, globpaths):
1575 paths_map = getpaths_map(build_dir, globpaths)
1577 for k, v in paths_map.items():
1584 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1588 """permanent store of existing APKs with the date they were added
1590 This is currently the only way to permanently store the "updated"
1595 self.path = os.path.join('stats', 'known_apks.txt')
1597 if os.path.isfile(self.path):
1598 with open(self.path, 'r', encoding='utf8') as f:
1600 t = line.rstrip().split(' ')
1602 self.apks[t[0]] = (t[1], None)
1604 self.apks[t[0]] = (t[1], datetime.strptime(t[2], '%Y-%m-%d'))
1605 self.changed = False
1607 def writeifchanged(self):
1608 if not self.changed:
1611 if not os.path.exists('stats'):
1615 for apk, app in self.apks.items():
1617 line = apk + ' ' + appid
1619 line += ' ' + added.strftime('%Y-%m-%d')
1622 with open(self.path, 'w', encoding='utf8') as f:
1623 for line in sorted(lst, key=natural_key):
1624 f.write(line + '\n')
1626 def recordapk(self, apkName, app, default_date=None):
1628 Record an apk (if it's new, otherwise does nothing)
1629 Returns the date it was added as a datetime instance
1631 if apkName not in self.apks:
1632 if default_date is None:
1633 default_date = datetime.utcnow()
1634 self.apks[apkName] = (app, default_date)
1636 _, added = self.apks[apkName]
1639 # Look up information - given the 'apkname', returns (app id, date added/None).
1640 # Or returns None for an unknown apk.
1641 def getapp(self, apkname):
1642 if apkname in self.apks:
1643 return self.apks[apkname]
1646 # Get the most recent 'num' apps added to the repo, as a list of package ids
1647 # with the most recent first.
1648 def getlatest(self, num):
1650 for apk, app in self.apks.items():
1654 if apps[appid] > added:
1658 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1659 lst = [app for app, _ in sortedapps]
1664 def get_file_extension(filename):
1665 """get the normalized file extension, can be blank string but never None"""
1666 if isinstance(filename, bytes):
1667 filename = filename.decode('utf-8')
1668 return os.path.splitext(filename)[1].lower()[1:]
1671 def get_apk_debuggable_aapt(apkfile):
1672 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1674 if p.returncode != 0:
1675 raise FDroidException("Failed to get apk manifest information")
1676 for line in p.output.splitlines():
1677 if 'android:debuggable' in line and not line.endswith('0x0'):
1682 def get_apk_debuggable_androguard(apkfile):
1684 from androguard.core.bytecodes.apk import APK
1686 raise FDroidException("androguard library is not installed and aapt not present")
1688 apkobject = APK(apkfile)
1689 if apkobject.is_valid_APK():
1690 debuggable = apkobject.get_element("application", "debuggable")
1691 if debuggable is not None:
1692 return bool(strtobool(debuggable))
1696 def isApkAndDebuggable(apkfile):
1697 """Returns True if the given file is an APK and is debuggable
1699 :param apkfile: full path to the apk to check"""
1701 if get_file_extension(apkfile) != 'apk':
1704 if SdkToolsPopen(['aapt', 'version'], output=False):
1705 return get_apk_debuggable_aapt(apkfile)
1707 return get_apk_debuggable_androguard(apkfile)
1710 def get_apk_id_aapt(apkfile):
1711 """Extrat identification information from APK using aapt.
1713 :param apkfile: path to an APK file.
1714 :returns: triplet (appid, version code, version name)
1716 r = re.compile("package: name='(?P<appid>.*)' versionCode='(?P<vercode>.*)' versionName='(?P<vername>.*)' platformBuildVersionName='.*'")
1717 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1718 for line in p.output.splitlines():
1721 return m.group('appid'), m.group('vercode'), m.group('vername')
1722 raise FDroidException("reading identification failed, APK invalid: '{}'".format(apkfile))
1727 self.returncode = None
1731 def SdkToolsPopen(commands, cwd=None, output=True):
1733 if cmd not in config:
1734 config[cmd] = find_sdk_tools_cmd(commands[0])
1735 abscmd = config[cmd]
1737 raise FDroidException("Could not find '%s' on your system" % cmd)
1739 test_aapt_version(config['aapt'])
1740 return FDroidPopen([abscmd] + commands[1:],
1741 cwd=cwd, output=output)
1744 def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
1746 Run a command and capture the possibly huge output as bytes.
1748 :param commands: command and argument list like in subprocess.Popen
1749 :param cwd: optionally specifies a working directory
1750 :param envs: a optional dictionary of environment variables and their values
1751 :returns: A PopenResult.
1756 set_FDroidPopen_env()
1758 process_env = env.copy()
1759 if envs is not None and len(envs) > 0:
1760 process_env.update(envs)
1763 cwd = os.path.normpath(cwd)
1764 logging.debug("Directory: %s" % cwd)
1765 logging.debug("> %s" % ' '.join(commands))
1767 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
1768 result = PopenResult()
1771 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
1772 stdout=subprocess.PIPE, stderr=stderr_param)
1773 except OSError as e:
1774 raise BuildException("OSError while trying to execute " +
1775 ' '.join(commands) + ': ' + str(e))
1777 if not stderr_to_stdout and options.verbose:
1778 stderr_queue = Queue()
1779 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
1781 while not stderr_reader.eof():
1782 while not stderr_queue.empty():
1783 line = stderr_queue.get()
1784 sys.stderr.buffer.write(line)
1789 stdout_queue = Queue()
1790 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1793 # Check the queue for output (until there is no more to get)
1794 while not stdout_reader.eof():
1795 while not stdout_queue.empty():
1796 line = stdout_queue.get()
1797 if output and options.verbose:
1798 # Output directly to console
1799 sys.stderr.buffer.write(line)
1805 result.returncode = p.wait()
1806 result.output = buf.getvalue()
1808 # make sure all filestreams of the subprocess are closed
1809 for streamvar in ['stdin', 'stdout', 'stderr']:
1810 if hasattr(p, streamvar):
1811 stream = getattr(p, streamvar)
1817 def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
1819 Run a command and capture the possibly huge output as a str.
1821 :param commands: command and argument list like in subprocess.Popen
1822 :param cwd: optionally specifies a working directory
1823 :param envs: a optional dictionary of environment variables and their values
1824 :returns: A PopenResult.
1826 result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
1827 result.output = result.output.decode('utf-8', 'ignore')
1831 gradle_comment = re.compile(r'[ ]*//')
1832 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1833 gradle_line_matches = [
1834 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1835 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1836 re.compile(r'.*\.readLine\(.*'),
1840 def remove_signing_keys(build_dir):
1841 for root, dirs, files in os.walk(build_dir):
1842 if 'build.gradle' in files:
1843 path = os.path.join(root, 'build.gradle')
1845 with open(path, "r", encoding='utf8') as o:
1846 lines = o.readlines()
1852 with open(path, "w", encoding='utf8') as o:
1853 while i < len(lines):
1856 while line.endswith('\\\n'):
1857 line = line.rstrip('\\\n') + lines[i]
1860 if gradle_comment.match(line):
1865 opened += line.count('{')
1866 opened -= line.count('}')
1869 if gradle_signing_configs.match(line):
1874 if any(s.match(line) for s in gradle_line_matches):
1882 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1885 'project.properties',
1887 'default.properties',
1888 'ant.properties', ]:
1889 if propfile in files:
1890 path = os.path.join(root, propfile)
1892 with open(path, "r", encoding='iso-8859-1') as o:
1893 lines = o.readlines()
1897 with open(path, "w", encoding='iso-8859-1') as o:
1899 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1906 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1909 def set_FDroidPopen_env(build=None):
1911 set up the environment variables for the build environment
1913 There is only a weak standard, the variables used by gradle, so also set
1914 up the most commonly used environment variables for SDK and NDK. Also, if
1915 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
1917 global env, orig_path
1921 orig_path = env['PATH']
1922 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
1923 env[n] = config['sdk_path']
1924 for k, v in config['java_paths'].items():
1925 env['JAVA%s_HOME' % k] = v
1927 missinglocale = True
1928 for k, v in env.items():
1929 if k == 'LANG' and v != 'C':
1930 missinglocale = False
1932 missinglocale = False
1934 env['LANG'] = 'en_US.UTF-8'
1936 if build is not None:
1937 path = build.ndk_path()
1938 paths = orig_path.split(os.pathsep)
1939 if path not in paths:
1940 paths = [path] + paths
1941 env['PATH'] = os.pathsep.join(paths)
1942 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
1943 env[n] = build.ndk_path()
1946 def replace_build_vars(cmd, build):
1947 cmd = cmd.replace('$$COMMIT$$', build.commit)
1948 cmd = cmd.replace('$$VERSION$$', build.versionName)
1949 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
1953 def replace_config_vars(cmd, build):
1954 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1955 cmd = cmd.replace('$$NDK$$', build.ndk_path())
1956 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1957 cmd = cmd.replace('$$QT$$', config['qt_sdk_path'] or '')
1958 if build is not None:
1959 cmd = replace_build_vars(cmd, build)
1963 def place_srclib(root_dir, number, libpath):
1966 relpath = os.path.relpath(libpath, root_dir)
1967 proppath = os.path.join(root_dir, 'project.properties')
1970 if os.path.isfile(proppath):
1971 with open(proppath, "r", encoding='iso-8859-1') as o:
1972 lines = o.readlines()
1974 with open(proppath, "w", encoding='iso-8859-1') as o:
1977 if line.startswith('android.library.reference.%d=' % number):
1978 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1983 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1986 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
1989 def metadata_get_sigdir(appid, vercode=None):
1990 """Get signature directory for app"""
1992 return os.path.join('metadata', appid, 'signatures', vercode)
1994 return os.path.join('metadata', appid, 'signatures')
1997 def apk_extract_signatures(apkpath, outdir, manifest=True):
1998 """Extracts a signature files from APK and puts them into target directory.
2000 :param apkpath: location of the apk
2001 :param outdir: folder where the extracted signature files will be stored
2002 :param manifest: (optionally) disable extracting manifest file
2004 with ZipFile(apkpath, 'r') as in_apk:
2005 for f in in_apk.infolist():
2006 if apk_sigfile.match(f.filename) or \
2007 (manifest and f.filename == 'META-INF/MANIFEST.MF'):
2008 newpath = os.path.join(outdir, os.path.basename(f.filename))
2009 with open(newpath, 'wb') as out_file:
2010 out_file.write(in_apk.read(f.filename))
2013 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
2014 """Verify that two apks are the same
2016 One of the inputs is signed, the other is unsigned. The signature metadata
2017 is transferred from the signed to the unsigned apk, and then jarsigner is
2018 used to verify that the signature from the signed apk is also varlid for
2019 the unsigned one. If the APK given as unsigned actually does have a
2020 signature, it will be stripped out and ignored.
2022 There are two SHA1 git commit IDs that fdroidserver includes in the builds
2023 it makes: fdroidserverid and buildserverid. Originally, these were inserted
2024 into AndroidManifest.xml, but that makes the build not reproducible. So
2025 instead they are included as separate files in the APK's META-INF/ folder.
2026 If those files exist in the signed APK, they will be part of the signature
2027 and need to also be included in the unsigned APK for it to validate.
2029 :param signed_apk: Path to a signed apk file
2030 :param unsigned_apk: Path to an unsigned apk file expected to match it
2031 :param tmp_dir: Path to directory for temporary files
2032 :returns: None if the verification is successful, otherwise a string
2033 describing what went wrong.
2036 signed = ZipFile(signed_apk, 'r')
2037 meta_inf_files = ['META-INF/MANIFEST.MF']
2038 for f in signed.namelist():
2039 if apk_sigfile.match(f) \
2040 or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
2041 meta_inf_files.append(f)
2042 if len(meta_inf_files) < 3:
2043 return "Signature files missing from {0}".format(signed_apk)
2045 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2046 unsigned = ZipFile(unsigned_apk, 'r')
2047 # only read the signature from the signed APK, everything else from unsigned
2048 with ZipFile(tmp_apk, 'w') as tmp:
2049 for filename in meta_inf_files:
2050 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2051 for info in unsigned.infolist():
2052 if info.filename in meta_inf_files:
2053 logging.warning('Ignoring ' + info.filename + ' from ' + unsigned_apk)
2055 if info.filename in tmp.namelist():
2056 return "duplicate filename found: " + info.filename
2057 tmp.writestr(info, unsigned.read(info.filename))
2061 verified = verify_apk_signature(tmp_apk)
2064 logging.info("...NOT verified - {0}".format(tmp_apk))
2065 return compare_apks(signed_apk, tmp_apk, tmp_dir,
2066 os.path.dirname(unsigned_apk))
2068 logging.info("...successfully verified")
2072 def verify_apk_signature(apk, jar=False):
2073 """verify the signature on an APK
2075 Try to use apksigner whenever possible since jarsigner is very
2076 shitty: unsigned APKs pass as "verified"! So this has to turn on
2077 -strict then check for result 4.
2079 You can set :param: jar to True if you want to use this method
2080 to verify jar signatures.
2082 if set_command_in_config('apksigner'):
2083 args = [config['apksigner'], 'verify']
2085 args += ['--min-sdk-version=1']
2086 return subprocess.call(args + [apk]) == 0
2088 logging.warning("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner")
2089 return subprocess.call([config['jarsigner'], '-strict', '-verify', apk]) == 4
2092 def verify_old_apk_signature(apk):
2093 """verify the signature on an archived APK, supporting deprecated algorithms
2095 F-Droid aims to keep every single binary that it ever published. Therefore,
2096 it needs to be able to verify APK signatures that include deprecated/removed
2097 algorithms. For example, jarsigner treats an MD5 signature as unsigned.
2099 jarsigner passes unsigned APKs as "verified"! So this has to turn
2100 on -strict then check for result 4.
2104 _java_security = os.path.join(os.getcwd(), '.java.security')
2105 with open(_java_security, 'w') as fp:
2106 fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
2108 return subprocess.call([config['jarsigner'], '-J-Djava.security.properties=' + _java_security,
2109 '-strict', '-verify', apk]) == 4
2112 apk_badchars = re.compile('''[/ :;'"]''')
2115 def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
2118 Returns None if the apk content is the same (apart from the signing key),
2119 otherwise a string describing what's different, or what went wrong when
2120 trying to do the comparison.
2126 absapk1 = os.path.abspath(apk1)
2127 absapk2 = os.path.abspath(apk2)
2129 if set_command_in_config('diffoscope'):
2130 logfilename = os.path.join(log_dir, os.path.basename(absapk1))
2131 htmlfile = logfilename + '.diffoscope.html'
2132 textfile = logfilename + '.diffoscope.txt'
2133 if subprocess.call([config['diffoscope'],
2134 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2135 '--html', htmlfile, '--text', textfile,
2136 absapk1, absapk2]) != 0:
2137 return("Failed to unpack " + apk1)
2139 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2140 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2141 for d in [apk1dir, apk2dir]:
2142 if os.path.exists(d):
2145 os.mkdir(os.path.join(d, 'jar-xf'))
2147 if subprocess.call(['jar', 'xf',
2148 os.path.abspath(apk1)],
2149 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2150 return("Failed to unpack " + apk1)
2151 if subprocess.call(['jar', 'xf',
2152 os.path.abspath(apk2)],
2153 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2154 return("Failed to unpack " + apk2)
2156 if set_command_in_config('apktool'):
2157 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2159 return("Failed to unpack " + apk1)
2160 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2162 return("Failed to unpack " + apk2)
2164 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2165 lines = p.output.splitlines()
2166 if len(lines) != 1 or 'META-INF' not in lines[0]:
2167 if set_command_in_config('meld'):
2168 p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
2169 return("Unexpected diff output - " + p.output)
2171 # since everything verifies, delete the comparison to keep cruft down
2172 shutil.rmtree(apk1dir)
2173 shutil.rmtree(apk2dir)
2175 # If we get here, it seems like they're the same!
2179 def set_command_in_config(command):
2180 '''Try to find specified command in the path, if it hasn't been
2181 manually set in config.py. If found, it is added to the config
2182 dict. The return value says whether the command is available.
2185 if command in config:
2188 tmp = find_command(command)
2190 config[command] = tmp
2195 def find_command(command):
2196 '''find the full path of a command, or None if it can't be found in the PATH'''
2199 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2201 fpath, fname = os.path.split(command)
2206 for path in os.environ["PATH"].split(os.pathsep):
2207 path = path.strip('"')
2208 exe_file = os.path.join(path, command)
2209 if is_exe(exe_file):
2216 '''generate a random password for when generating keys'''
2217 h = hashlib.sha256()
2218 h.update(os.urandom(16)) # salt
2219 h.update(socket.getfqdn().encode('utf-8'))
2220 passwd = base64.b64encode(h.digest()).strip()
2221 return passwd.decode('utf-8')
2224 def genkeystore(localconfig):
2226 Generate a new key with password provided in :param localconfig and add it to new keystore
2227 :return: hexed public key, public key fingerprint
2229 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2230 keystoredir = os.path.dirname(localconfig['keystore'])
2231 if keystoredir is None or keystoredir == '':
2232 keystoredir = os.path.join(os.getcwd(), keystoredir)
2233 if not os.path.exists(keystoredir):
2234 os.makedirs(keystoredir, mode=0o700)
2237 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
2238 'FDROID_KEY_PASS': localconfig['keypass'],
2240 p = FDroidPopen([config['keytool'], '-genkey',
2241 '-keystore', localconfig['keystore'],
2242 '-alias', localconfig['repo_keyalias'],
2243 '-keyalg', 'RSA', '-keysize', '4096',
2244 '-sigalg', 'SHA256withRSA',
2245 '-validity', '10000',
2246 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2247 '-keypass:env', 'FDROID_KEY_PASS',
2248 '-dname', localconfig['keydname']], envs=env_vars)
2249 if p.returncode != 0:
2250 raise BuildException("Failed to generate key", p.output)
2251 os.chmod(localconfig['keystore'], 0o0600)
2252 if not options.quiet:
2253 # now show the lovely key that was just generated
2254 p = FDroidPopen([config['keytool'], '-list', '-v',
2255 '-keystore', localconfig['keystore'],
2256 '-alias', localconfig['repo_keyalias'],
2257 '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
2258 logging.info(p.output.strip() + '\n\n')
2259 # get the public key
2260 p = FDroidPopenBytes([config['keytool'], '-exportcert',
2261 '-keystore', localconfig['keystore'],
2262 '-alias', localconfig['repo_keyalias'],
2263 '-storepass:env', 'FDROID_KEY_STORE_PASS']
2264 + config['smartcardoptions'],
2265 envs=env_vars, output=False, stderr_to_stdout=False)
2266 if p.returncode != 0 or len(p.output) < 20:
2267 raise BuildException("Failed to get public key", p.output)
2269 fingerprint = get_cert_fingerprint(pubkey)
2270 return hexlify(pubkey), fingerprint
2273 def get_cert_fingerprint(pubkey):
2275 Generate a certificate fingerprint the same way keytool does it
2276 (but with slightly different formatting)
2278 digest = hashlib.sha256(pubkey).digest()
2279 ret = [' '.join("%02X" % b for b in bytearray(digest))]
2280 return " ".join(ret)
2283 def get_certificate(certificate_file):
2285 Extracts a certificate from the given file.
2286 :param certificate_file: file bytes (as string) representing the certificate
2287 :return: A binary representation of the certificate's public key, or None in case of error
2289 content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
2290 if content.getComponentByName('contentType') != rfc2315.signedData:
2292 content = decoder.decode(content.getComponentByName('content'),
2293 asn1Spec=rfc2315.SignedData())[0]
2295 certificates = content.getComponentByName('certificates')
2296 cert = certificates[0].getComponentByName('certificate')
2298 logging.error("Certificates not found.")
2300 return encoder.encode(cert)
2303 def write_to_config(thisconfig, key, value=None, config_file=None):
2304 '''write a key/value to the local config.py
2306 NOTE: only supports writing string variables.
2308 :param thisconfig: config dictionary
2309 :param key: variable name in config.py to be overwritten/added
2310 :param value: optional value to be written, instead of fetched
2311 from 'thisconfig' dictionary.
2314 origkey = key + '_orig'
2315 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2316 cfg = config_file if config_file else 'config.py'
2318 # load config file, create one if it doesn't exist
2319 if not os.path.exists(cfg):
2321 logging.info("Creating empty " + cfg)
2322 with open(cfg, 'r', encoding="utf-8") as f:
2323 lines = f.readlines()
2325 # make sure the file ends with a carraige return
2327 if not lines[-1].endswith('\n'):
2330 # regex for finding and replacing python string variable
2331 # definitions/initializations
2332 pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
2333 repl = key + ' = "' + value + '"'
2334 pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
2335 repl2 = key + " = '" + value + "'"
2337 # If we replaced this line once, we make sure won't be a
2338 # second instance of this line for this key in the document.
2341 with open(cfg, 'w', encoding="utf-8") as f:
2343 if pattern.match(line) or pattern2.match(line):
2345 line = pattern.sub(repl, line)
2346 line = pattern2.sub(repl2, line)
2357 def parse_xml(path):
2358 return XMLElementTree.parse(path).getroot()
2361 def string_is_integer(string):
2369 def get_per_app_repos():
2370 '''per-app repos are dirs named with the packageName of a single app'''
2372 # Android packageNames are Java packages, they may contain uppercase or
2373 # lowercase letters ('A' through 'Z'), numbers, and underscores
2374 # ('_'). However, individual package name parts may only start with
2375 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2376 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2379 for root, dirs, files in os.walk(os.getcwd()):
2381 print('checking', root, 'for', d)
2382 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2383 # standard parts of an fdroid repo, so never packageNames
2386 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
2392 def is_repo_file(filename):
2393 '''Whether the file in a repo is a build product to be delivered to users'''
2394 if isinstance(filename, str):
2395 filename = filename.encode('utf-8', errors="surrogateescape")
2396 return os.path.isfile(filename) \
2397 and not filename.endswith(b'.asc') \
2398 and not filename.endswith(b'.sig') \
2399 and os.path.basename(filename) not in [
2401 b'index_unsigned.jar',