3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
37 import xml.etree.ElementTree as XMLElementTree
39 from binascii import hexlify
40 from datetime import datetime
41 from distutils.version import LooseVersion
42 from queue import Queue
43 from zipfile import ZipFile
45 from pyasn1.codec.der import decoder, encoder
46 from pyasn1_modules import rfc2315
47 from pyasn1.error import PyAsn1Error
49 from distutils.util import strtobool
51 import fdroidserver.metadata
52 from fdroidserver.exception import FDroidException, VCSException, BuildException
53 from .asynchronousfilereader import AsynchronousFileReader
56 # A signature block file with a .DSA, .RSA, or .EC extension
57 CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
58 APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
59 STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
61 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
70 'sdk_path': "$ANDROID_HOME",
75 'r12b': "$ANDROID_NDK",
81 'build_tools': "25.0.2",
82 'force_build_tools': False,
87 'accepted_formats': ['txt', 'yml'],
88 'sync_from_local_copy_dir': False,
89 'allow_disabled_algorithms': False,
90 'per_app_repos': False,
91 'make_current_version_link': True,
92 'current_version_name_source': 'Name',
93 'update_stats': False,
97 'stats_to_carbon': False,
99 'build_server_always': False,
100 'keystore': 'keystore.jks',
101 'smartcardoptions': [],
111 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
112 'repo_name': "My First FDroid Repo Demo",
113 'repo_icon': "fdroid-icon.png",
114 'repo_description': '''
115 This is a repository of apps to be used with FDroid. Applications in this
116 repository are either official binaries built by the original application
117 developers, or are binaries built from source by the admin of f-droid.org
118 using the tools on https://gitlab.com/u/fdroid.
124 def setup_global_opts(parser):
125 parser.add_argument("-v", "--verbose", action="store_true", default=False,
126 help="Spew out even more information than normal")
127 parser.add_argument("-q", "--quiet", action="store_true", default=False,
128 help="Restrict output to warnings and errors")
131 def fill_config_defaults(thisconfig):
132 for k, v in default_config.items():
133 if k not in thisconfig:
136 # Expand paths (~users and $vars)
137 def expand_path(path):
141 path = os.path.expanduser(path)
142 path = os.path.expandvars(path)
147 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
152 thisconfig[k + '_orig'] = v
154 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
155 if thisconfig['java_paths'] is None:
156 thisconfig['java_paths'] = dict()
158 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
159 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
160 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
161 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
162 if os.getenv('JAVA_HOME') is not None:
163 pathlist.append(os.getenv('JAVA_HOME'))
164 if os.getenv('PROGRAMFILES') is not None:
165 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
166 for d in sorted(pathlist):
167 if os.path.islink(d):
169 j = os.path.basename(d)
170 # the last one found will be the canonical one, so order appropriately
172 r'^1\.([6-9])\.0\.jdk$', # OSX
173 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
174 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
175 r'^jdk([6-9])-openjdk$', # Arch
176 r'^java-([6-9])-openjdk$', # Arch
177 r'^java-([6-9])-jdk$', # Arch (oracle)
178 r'^java-1\.([6-9])\.0-.*$', # RedHat
179 r'^java-([6-9])-oracle$', # Debian WebUpd8
180 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
181 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
183 m = re.match(regex, j)
186 for p in [d, os.path.join(d, 'Contents', 'Home')]:
187 if os.path.exists(os.path.join(p, 'bin', 'javac')):
188 thisconfig['java_paths'][m.group(1)] = p
190 for java_version in ('7', '8', '9'):
191 if java_version not in thisconfig['java_paths']:
193 java_home = thisconfig['java_paths'][java_version]
194 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
195 if os.path.exists(jarsigner):
196 thisconfig['jarsigner'] = jarsigner
197 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
198 break # Java7 is preferred, so quit if found
200 for k in ['ndk_paths', 'java_paths']:
206 thisconfig[k][k2] = exp
207 thisconfig[k][k2 + '_orig'] = v
210 def regsub_file(pattern, repl, path):
211 with open(path, 'rb') as f:
213 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
214 with open(path, 'wb') as f:
218 def read_config(opts, config_file='config.py'):
219 """Read the repository config
221 The config is read from config_file, which is in the current
222 directory when any of the repo management commands are used. If
223 there is a local metadata file in the git repo, then config.py is
224 not required, just use defaults.
227 global config, options
229 if config is not None:
236 if os.path.isfile(config_file):
237 logging.debug("Reading %s" % config_file)
238 with io.open(config_file, "rb") as f:
239 code = compile(f.read(), config_file, 'exec')
240 exec(code, None, config)
241 elif len(get_local_metadata_files()) == 0:
242 raise FDroidException("Missing config file - is this a repo directory?")
244 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
246 if not type(config[k]) in (str, list, tuple):
247 logging.warn('"' + k + '" will be in random order!'
248 + ' Use () or [] brackets if order is important!')
250 # smartcardoptions must be a list since its command line args for Popen
251 if 'smartcardoptions' in config:
252 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
253 elif 'keystore' in config and config['keystore'] == 'NONE':
254 # keystore='NONE' means use smartcard, these are required defaults
255 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
256 'SunPKCS11-OpenSC', '-providerClass',
257 'sun.security.pkcs11.SunPKCS11',
258 '-providerArg', 'opensc-fdroid.cfg']
260 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
261 st = os.stat(config_file)
262 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
263 logging.warning("unsafe permissions on {0} (should be 0600)!".format(config_file))
265 fill_config_defaults(config)
267 for k in ["repo_description", "archive_description"]:
269 config[k] = clean_description(config[k])
271 if 'serverwebroot' in config:
272 if isinstance(config['serverwebroot'], str):
273 roots = [config['serverwebroot']]
274 elif all(isinstance(item, str) for item in config['serverwebroot']):
275 roots = config['serverwebroot']
277 raise TypeError('only accepts strings, lists, and tuples')
279 for rootstr in roots:
280 # since this is used with rsync, where trailing slashes have
281 # meaning, ensure there is always a trailing slash
282 if rootstr[-1] != '/':
284 rootlist.append(rootstr.replace('//', '/'))
285 config['serverwebroot'] = rootlist
287 if 'servergitmirrors' in config:
288 if isinstance(config['servergitmirrors'], str):
289 roots = [config['servergitmirrors']]
290 elif all(isinstance(item, str) for item in config['servergitmirrors']):
291 roots = config['servergitmirrors']
293 raise TypeError('only accepts strings, lists, and tuples')
294 config['servergitmirrors'] = roots
299 def find_sdk_tools_cmd(cmd):
300 '''find a working path to a tool from the Android SDK'''
303 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
304 # try to find a working path to this command, in all the recent possible paths
305 if 'build_tools' in config:
306 build_tools = os.path.join(config['sdk_path'], 'build-tools')
307 # if 'build_tools' was manually set and exists, check only that one
308 configed_build_tools = os.path.join(build_tools, config['build_tools'])
309 if os.path.exists(configed_build_tools):
310 tooldirs.append(configed_build_tools)
312 # no configed version, so hunt known paths for it
313 for f in sorted(os.listdir(build_tools), reverse=True):
314 if os.path.isdir(os.path.join(build_tools, f)):
315 tooldirs.append(os.path.join(build_tools, f))
316 tooldirs.append(build_tools)
317 sdk_tools = os.path.join(config['sdk_path'], 'tools')
318 if os.path.exists(sdk_tools):
319 tooldirs.append(sdk_tools)
320 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
321 if os.path.exists(sdk_platform_tools):
322 tooldirs.append(sdk_platform_tools)
323 tooldirs.append('/usr/bin')
325 path = os.path.join(d, cmd)
326 if os.path.isfile(path):
328 test_aapt_version(path)
330 # did not find the command, exit with error message
331 ensure_build_tools_exists(config)
334 def test_aapt_version(aapt):
335 '''Check whether the version of aapt is new enough'''
336 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
337 if output is None or output == '':
338 logging.error(aapt + ' failed to execute!')
340 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
345 # the Debian package has the version string like "v0.2-23.0.2"
346 if '.' not in bugfix and LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.2166767'):
347 logging.warning(aapt + ' is too old, fdroid requires build-tools-23.0.0 or newer!')
349 logging.warning('Unknown version of aapt, might cause problems: ' + output)
352 def test_sdk_exists(thisconfig):
353 if 'sdk_path' not in thisconfig:
354 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
355 test_aapt_version(thisconfig['aapt'])
358 logging.error("'sdk_path' not set in config.py!")
360 if thisconfig['sdk_path'] == default_config['sdk_path']:
361 logging.error('No Android SDK found!')
362 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
363 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
365 if not os.path.exists(thisconfig['sdk_path']):
366 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
368 if not os.path.isdir(thisconfig['sdk_path']):
369 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
371 for d in ['build-tools', 'platform-tools', 'tools']:
372 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
373 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
374 thisconfig['sdk_path'], d))
379 def ensure_build_tools_exists(thisconfig):
380 if not test_sdk_exists(thisconfig):
381 raise FDroidException("Android SDK not found.")
382 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
383 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
384 if not os.path.isdir(versioned_build_tools):
385 raise FDroidException(
386 'Android Build Tools path "' + versioned_build_tools + '" does not exist!')
389 def get_local_metadata_files():
390 '''get any metadata files local to an app's source repo
392 This tries to ignore anything that does not count as app metdata,
393 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
396 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
399 def read_pkg_args(args, allow_vercodes=False):
401 :param args: arguments in the form of multiple appid:[vc] strings
402 :returns: a dictionary with the set of vercodes specified for each package
410 if allow_vercodes and ':' in p:
411 package, vercode = p.split(':')
413 package, vercode = p, None
414 if package not in vercodes:
415 vercodes[package] = [vercode] if vercode else []
417 elif vercode and vercode not in vercodes[package]:
418 vercodes[package] += [vercode] if vercode else []
423 def read_app_args(args, allapps, allow_vercodes=False):
425 On top of what read_pkg_args does, this returns the whole app metadata, but
426 limiting the builds list to the builds matching the vercodes specified.
429 vercodes = read_pkg_args(args, allow_vercodes)
435 for appid, app in allapps.items():
436 if appid in vercodes:
439 if len(apps) != len(vercodes):
442 logging.critical("No such package: %s" % p)
443 raise FDroidException("Found invalid app ids in arguments")
445 raise FDroidException("No packages specified")
448 for appid, app in apps.items():
452 app.builds = [b for b in app.builds if b.versionCode in vc]
453 if len(app.builds) != len(vercodes[appid]):
455 allvcs = [b.versionCode for b in app.builds]
456 for v in vercodes[appid]:
458 logging.critical("No such vercode %s for app %s" % (v, appid))
461 raise FDroidException("Found invalid vercodes for some apps")
466 def get_extension(filename):
467 base, ext = os.path.splitext(filename)
470 return base, ext.lower()[1:]
473 def has_extension(filename, ext):
474 _, f_ext = get_extension(filename)
478 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
481 def clean_description(description):
482 'Remove unneeded newlines and spaces from a block of description text'
484 # this is split up by paragraph to make removing the newlines easier
485 for paragraph in re.split(r'\n\n', description):
486 paragraph = re.sub('\r', '', paragraph)
487 paragraph = re.sub('\n', ' ', paragraph)
488 paragraph = re.sub(' {2,}', ' ', paragraph)
489 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
490 returnstring += paragraph + '\n\n'
491 return returnstring.rstrip('\n')
494 def publishednameinfo(filename):
495 filename = os.path.basename(filename)
496 m = publish_name_regex.match(filename)
498 result = (m.group(1), m.group(2))
499 except AttributeError:
500 raise FDroidException("Invalid name for published file: %s" % filename)
504 def get_release_filename(app, build):
506 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
508 return "%s_%s.apk" % (app.id, build.versionCode)
511 def get_toolsversion_logname(app, build):
512 return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
515 def getsrcname(app, build):
516 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
528 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
531 def get_build_dir(app):
532 '''get the dir that this app will be built in'''
534 if app.RepoType == 'srclib':
535 return os.path.join('build', 'srclib', app.Repo)
537 return os.path.join('build', app.id)
541 '''checkout code from VCS and return instance of vcs and the build dir'''
542 build_dir = get_build_dir(app)
544 # Set up vcs interface and make sure we have the latest code...
545 logging.debug("Getting {0} vcs interface for {1}"
546 .format(app.RepoType, app.Repo))
547 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
551 vcs = getvcs(app.RepoType, remote, build_dir)
553 return vcs, build_dir
556 def getvcs(vcstype, remote, local):
558 return vcs_git(remote, local)
559 if vcstype == 'git-svn':
560 return vcs_gitsvn(remote, local)
562 return vcs_hg(remote, local)
564 return vcs_bzr(remote, local)
565 if vcstype == 'srclib':
566 if local != os.path.join('build', 'srclib', remote):
567 raise VCSException("Error: srclib paths are hard-coded!")
568 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
570 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
571 raise VCSException("Invalid vcs type " + vcstype)
574 def getsrclibvcs(name):
575 if name not in fdroidserver.metadata.srclibs:
576 raise VCSException("Missing srclib " + name)
577 return fdroidserver.metadata.srclibs[name]['Repo Type']
582 def __init__(self, remote, local):
584 # svn, git-svn and bzr may require auth
586 if self.repotype() in ('git-svn', 'bzr'):
588 if self.repotype == 'git-svn':
589 raise VCSException("Authentication is not supported for git-svn")
590 self.username, remote = remote.split('@')
591 if ':' not in self.username:
592 raise VCSException("Password required with username")
593 self.username, self.password = self.username.split(':')
597 self.clone_failed = False
598 self.refreshed = False
604 # Take the local repository to a clean version of the given revision, which
605 # is specificed in the VCS's native format. Beforehand, the repository can
606 # be dirty, or even non-existent. If the repository does already exist
607 # locally, it will be updated from the origin, but only once in the
608 # lifetime of the vcs object.
609 # None is acceptable for 'rev' if you know you are cloning a clean copy of
610 # the repo - otherwise it must specify a valid revision.
611 def gotorevision(self, rev, refresh=True):
613 if self.clone_failed:
614 raise VCSException("Downloading the repository already failed once, not trying again.")
616 # The .fdroidvcs-id file for a repo tells us what VCS type
617 # and remote that directory was created from, allowing us to drop it
618 # automatically if either of those things changes.
619 fdpath = os.path.join(self.local, '..',
620 '.fdroidvcs-' + os.path.basename(self.local))
621 fdpath = os.path.normpath(fdpath)
622 cdata = self.repotype() + ' ' + self.remote
625 if os.path.exists(self.local):
626 if os.path.exists(fdpath):
627 with open(fdpath, 'r') as f:
628 fsdata = f.read().strip()
633 logging.info("Repository details for %s changed - deleting" % (
637 logging.info("Repository details for %s missing - deleting" % (
640 shutil.rmtree(self.local)
644 self.refreshed = True
647 self.gotorevisionx(rev)
648 except FDroidException as e:
651 # If necessary, write the .fdroidvcs file.
652 if writeback and not self.clone_failed:
653 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
654 with open(fdpath, 'w+') as f:
660 # Derived classes need to implement this. It's called once basic checking
661 # has been performend.
662 def gotorevisionx(self, rev): # pylint: disable=unused-argument
663 raise VCSException("This VCS type doesn't define gotorevisionx")
665 # Initialise and update submodules
666 def initsubmodules(self):
667 raise VCSException('Submodules not supported for this vcs type')
669 # Get a list of all known tags
671 if not self._gettags:
672 raise VCSException('gettags not supported for this vcs type')
674 for tag in self._gettags():
675 if re.match('[-A-Za-z0-9_. /]+$', tag):
679 # Get a list of all the known tags, sorted from newest to oldest
680 def latesttags(self):
681 raise VCSException('latesttags not supported for this vcs type')
683 # Get current commit reference (hash, revision, etc)
685 raise VCSException('getref not supported for this vcs type')
687 # Returns the srclib (name, path) used in setting up the current
698 # If the local directory exists, but is somehow not a git repository, git
699 # will traverse up the directory tree until it finds one that is (i.e.
700 # fdroidserver) and then we'll proceed to destroy it! This is called as
703 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
704 result = p.output.rstrip()
705 if not result.endswith(self.local):
706 raise VCSException('Repository mismatch')
708 def gotorevisionx(self, rev):
709 if not os.path.exists(self.local):
711 p = FDroidPopen(['git', 'clone', self.remote, self.local])
712 if p.returncode != 0:
713 self.clone_failed = True
714 raise VCSException("Git clone failed", p.output)
718 # Discard any working tree changes
719 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
720 'git', 'reset', '--hard'], cwd=self.local, output=False)
721 if p.returncode != 0:
722 raise VCSException("Git reset failed", p.output)
723 # Remove untracked files now, in case they're tracked in the target
724 # revision (it happens!)
725 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
726 'git', 'clean', '-dffx'], cwd=self.local, output=False)
727 if p.returncode != 0:
728 raise VCSException("Git clean failed", p.output)
729 if not self.refreshed:
730 # Get latest commits and tags from remote
731 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
732 if p.returncode != 0:
733 raise VCSException("Git fetch failed", p.output)
734 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
735 if p.returncode != 0:
736 raise VCSException("Git fetch failed", p.output)
737 # Recreate origin/HEAD as git clone would do it, in case it disappeared
738 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
739 if p.returncode != 0:
740 lines = p.output.splitlines()
741 if 'Multiple remote HEAD branches' not in lines[0]:
742 raise VCSException("Git remote set-head failed", p.output)
743 branch = lines[1].split(' ')[-1]
744 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
745 if p2.returncode != 0:
746 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
747 self.refreshed = True
748 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
749 # a github repo. Most of the time this is the same as origin/master.
750 rev = rev or 'origin/HEAD'
751 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
752 if p.returncode != 0:
753 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
754 # Get rid of any uncontrolled files left behind
755 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
756 if p.returncode != 0:
757 raise VCSException("Git clean failed", p.output)
759 def initsubmodules(self):
761 submfile = os.path.join(self.local, '.gitmodules')
762 if not os.path.isfile(submfile):
763 raise VCSException("No git submodules available")
765 # fix submodules not accessible without an account and public key auth
766 with open(submfile, 'r') as f:
767 lines = f.readlines()
768 with open(submfile, 'w') as f:
770 if 'git@github.com' in line:
771 line = line.replace('git@github.com:', 'https://github.com/')
772 if 'git@gitlab.com' in line:
773 line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
776 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
777 if p.returncode != 0:
778 raise VCSException("Git submodule sync failed", p.output)
779 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
780 if p.returncode != 0:
781 raise VCSException("Git submodule update failed", p.output)
785 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
786 return p.output.splitlines()
788 tag_format = re.compile(r'tag: ([^),]*)')
790 def latesttags(self):
792 p = FDroidPopen(['git', 'log', '--tags',
793 '--simplify-by-decoration', '--pretty=format:%d'],
794 cwd=self.local, output=False)
796 for line in p.output.splitlines():
797 for tag in self.tag_format.findall(line):
802 class vcs_gitsvn(vcs):
807 # If the local directory exists, but is somehow not a git repository, git
808 # will traverse up the directory tree until it finds one that is (i.e.
809 # fdroidserver) and then we'll proceed to destory it! This is called as
812 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
813 result = p.output.rstrip()
814 if not result.endswith(self.local):
815 raise VCSException('Repository mismatch')
817 def gotorevisionx(self, rev):
818 if not os.path.exists(self.local):
820 gitsvn_args = ['git', 'svn', 'clone']
821 if ';' in self.remote:
822 remote_split = self.remote.split(';')
823 for i in remote_split[1:]:
824 if i.startswith('trunk='):
825 gitsvn_args.extend(['-T', i[6:]])
826 elif i.startswith('tags='):
827 gitsvn_args.extend(['-t', i[5:]])
828 elif i.startswith('branches='):
829 gitsvn_args.extend(['-b', i[9:]])
830 gitsvn_args.extend([remote_split[0], self.local])
831 p = FDroidPopen(gitsvn_args, output=False)
832 if p.returncode != 0:
833 self.clone_failed = True
834 raise VCSException("Git svn clone failed", p.output)
836 gitsvn_args.extend([self.remote, self.local])
837 p = FDroidPopen(gitsvn_args, output=False)
838 if p.returncode != 0:
839 self.clone_failed = True
840 raise VCSException("Git svn clone failed", p.output)
844 # Discard any working tree changes
845 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
846 if p.returncode != 0:
847 raise VCSException("Git reset failed", p.output)
848 # Remove untracked files now, in case they're tracked in the target
849 # revision (it happens!)
850 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
851 if p.returncode != 0:
852 raise VCSException("Git clean failed", p.output)
853 if not self.refreshed:
854 # Get new commits, branches and tags from repo
855 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
856 if p.returncode != 0:
857 raise VCSException("Git svn fetch failed")
858 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
859 if p.returncode != 0:
860 raise VCSException("Git svn rebase failed", p.output)
861 self.refreshed = True
863 rev = rev or 'master'
865 nospaces_rev = rev.replace(' ', '%20')
866 # Try finding a svn tag
867 for treeish in ['origin/', '']:
868 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
869 if p.returncode == 0:
871 if p.returncode != 0:
872 # No tag found, normal svn rev translation
873 # Translate svn rev into git format
874 rev_split = rev.split('/')
877 for treeish in ['origin/', '']:
878 if len(rev_split) > 1:
879 treeish += rev_split[0]
880 svn_rev = rev_split[1]
883 # if no branch is specified, then assume trunk (i.e. 'master' branch):
887 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
889 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
890 git_rev = p.output.rstrip()
892 if p.returncode == 0 and git_rev:
895 if p.returncode != 0 or not git_rev:
896 # Try a plain git checkout as a last resort
897 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
898 if p.returncode != 0:
899 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
901 # Check out the git rev equivalent to the svn rev
902 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
903 if p.returncode != 0:
904 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
906 # Get rid of any uncontrolled files left behind
907 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
908 if p.returncode != 0:
909 raise VCSException("Git clean failed", p.output)
913 for treeish in ['origin/', '']:
914 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
920 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
921 if p.returncode != 0:
923 return p.output.strip()
931 def gotorevisionx(self, rev):
932 if not os.path.exists(self.local):
933 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
934 if p.returncode != 0:
935 self.clone_failed = True
936 raise VCSException("Hg clone failed", p.output)
938 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
939 if p.returncode != 0:
940 raise VCSException("Hg status failed", p.output)
941 for line in p.output.splitlines():
942 if not line.startswith('? '):
943 raise VCSException("Unexpected output from hg status -uS: " + line)
944 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
945 if not self.refreshed:
946 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
947 if p.returncode != 0:
948 raise VCSException("Hg pull failed", p.output)
949 self.refreshed = True
951 rev = rev or 'default'
954 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
955 if p.returncode != 0:
956 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
957 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
958 # Also delete untracked files, we have to enable purge extension for that:
959 if "'purge' is provided by the following extension" in p.output:
960 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
961 myfile.write("\n[extensions]\nhgext.purge=\n")
962 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
963 if p.returncode != 0:
964 raise VCSException("HG purge failed", p.output)
965 elif p.returncode != 0:
966 raise VCSException("HG purge failed", p.output)
969 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
970 return p.output.splitlines()[1:]
978 def gotorevisionx(self, rev):
979 if not os.path.exists(self.local):
980 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
981 if p.returncode != 0:
982 self.clone_failed = True
983 raise VCSException("Bzr branch failed", p.output)
985 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
986 if p.returncode != 0:
987 raise VCSException("Bzr revert failed", p.output)
988 if not self.refreshed:
989 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
990 if p.returncode != 0:
991 raise VCSException("Bzr update failed", p.output)
992 self.refreshed = True
994 revargs = list(['-r', rev] if rev else [])
995 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
996 if p.returncode != 0:
997 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1000 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
1001 return [tag.split(' ')[0].strip() for tag in
1002 p.output.splitlines()]
1005 def unescape_string(string):
1008 if string[0] == '"' and string[-1] == '"':
1011 return string.replace("\\'", "'")
1014 def retrieve_string(app_dir, string, xmlfiles=None):
1016 if not string.startswith('@string/'):
1017 return unescape_string(string)
1019 if xmlfiles is None:
1022 os.path.join(app_dir, 'res'),
1023 os.path.join(app_dir, 'src', 'main', 'res'),
1025 for r, d, f in os.walk(res_dir):
1026 if os.path.basename(r) == 'values':
1027 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
1029 name = string[len('@string/'):]
1031 def element_content(element):
1032 if element.text is None:
1034 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1035 return s.decode('utf-8').strip()
1037 for path in xmlfiles:
1038 if not os.path.isfile(path):
1040 xml = parse_xml(path)
1041 element = xml.find('string[@name="' + name + '"]')
1042 if element is not None:
1043 content = element_content(element)
1044 return retrieve_string(app_dir, content, xmlfiles)
1049 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1050 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1053 def manifest_paths(app_dir, flavours):
1054 '''Return list of existing files that will be used to find the highest vercode'''
1056 possible_manifests = \
1057 [os.path.join(app_dir, 'AndroidManifest.xml'),
1058 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1059 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1060 os.path.join(app_dir, 'build.gradle')]
1062 for flavour in flavours:
1063 if flavour == 'yes':
1065 possible_manifests.append(
1066 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1068 return [path for path in possible_manifests if os.path.isfile(path)]
1071 def fetch_real_name(app_dir, flavours):
1072 '''Retrieve the package name. Returns the name, or None if not found.'''
1073 for path in manifest_paths(app_dir, flavours):
1074 if not has_extension(path, 'xml') or not os.path.isfile(path):
1076 logging.debug("fetch_real_name: Checking manifest at " + path)
1077 xml = parse_xml(path)
1078 app = xml.find('application')
1081 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1083 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1084 result = retrieve_string_singleline(app_dir, label)
1086 result = result.strip()
1091 def get_library_references(root_dir):
1093 proppath = os.path.join(root_dir, 'project.properties')
1094 if not os.path.isfile(proppath):
1096 with open(proppath, 'r', encoding='iso-8859-1') as f:
1098 if not line.startswith('android.library.reference.'):
1100 path = line.split('=')[1].strip()
1101 relpath = os.path.join(root_dir, path)
1102 if not os.path.isdir(relpath):
1104 logging.debug("Found subproject at %s" % path)
1105 libraries.append(path)
1109 def ant_subprojects(root_dir):
1110 subprojects = get_library_references(root_dir)
1111 for subpath in subprojects:
1112 subrelpath = os.path.join(root_dir, subpath)
1113 for p in get_library_references(subrelpath):
1114 relp = os.path.normpath(os.path.join(subpath, p))
1115 if relp not in subprojects:
1116 subprojects.insert(0, relp)
1120 def remove_debuggable_flags(root_dir):
1121 # Remove forced debuggable flags
1122 logging.debug("Removing debuggable flags from %s" % root_dir)
1123 for root, dirs, files in os.walk(root_dir):
1124 if 'AndroidManifest.xml' in files:
1125 regsub_file(r'android:debuggable="[^"]*"',
1127 os.path.join(root, 'AndroidManifest.xml'))
1130 vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search
1131 vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1132 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1135 def app_matches_packagename(app, package):
1138 appid = app.UpdateCheckName or app.id
1139 if appid is None or appid == "Ignore":
1141 return appid == package
1144 def parse_androidmanifests(paths, app):
1146 Extract some information from the AndroidManifest.xml at the given path.
1147 Returns (version, vercode, package), any or all of which might be None.
1148 All values returned are strings.
1151 ignoreversions = app.UpdateCheckIgnore
1152 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1155 return (None, None, None)
1163 if not os.path.isfile(path):
1166 logging.debug("Parsing manifest at {0}".format(path))
1171 if has_extension(path, 'gradle'):
1172 with open(path, 'r') as f:
1174 if gradle_comment.match(line):
1176 # Grab first occurence of each to avoid running into
1177 # alternative flavours and builds.
1179 matches = psearch_g(line)
1181 s = matches.group(2)
1182 if app_matches_packagename(app, s):
1185 matches = vnsearch_g(line)
1187 version = matches.group(2)
1189 matches = vcsearch_g(line)
1191 vercode = matches.group(1)
1194 xml = parse_xml(path)
1195 if "package" in xml.attrib:
1196 s = xml.attrib["package"]
1197 if app_matches_packagename(app, s):
1199 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1200 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1201 base_dir = os.path.dirname(path)
1202 version = retrieve_string_singleline(base_dir, version)
1203 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1204 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1205 if string_is_integer(a):
1208 logging.warning("Problem with xml at {0}".format(path))
1210 # Remember package name, may be defined separately from version+vercode
1212 package = max_package
1214 logging.debug("..got package={0}, version={1}, vercode={2}"
1215 .format(package, version, vercode))
1217 # Always grab the package name and version name in case they are not
1218 # together with the highest version code
1219 if max_package is None and package is not None:
1220 max_package = package
1221 if max_version is None and version is not None:
1222 max_version = version
1224 if vercode is not None \
1225 and (max_vercode is None or vercode > max_vercode):
1226 if not ignoresearch or not ignoresearch(version):
1227 if version is not None:
1228 max_version = version
1229 if vercode is not None:
1230 max_vercode = vercode
1231 if package is not None:
1232 max_package = package
1234 max_version = "Ignore"
1236 if max_version is None:
1237 max_version = "Unknown"
1239 if max_package and not is_valid_package_name(max_package):
1240 raise FDroidException("Invalid package name {0}".format(max_package))
1242 return (max_version, max_vercode, max_package)
1245 def is_valid_package_name(name):
1246 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1249 # Get the specified source library.
1250 # Returns the path to it. Normally this is the path to be used when referencing
1251 # it, which may be a subdirectory of the actual project. If you want the base
1252 # directory of the project, pass 'basepath=True'.
1253 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1254 raw=False, prepare=True, preponly=False, refresh=True,
1263 name, ref = spec.split('@')
1265 number, name = name.split(':', 1)
1267 name, subdir = name.split('/', 1)
1269 if name not in fdroidserver.metadata.srclibs:
1270 raise VCSException('srclib ' + name + ' not found.')
1272 srclib = fdroidserver.metadata.srclibs[name]
1274 sdir = os.path.join(srclib_dir, name)
1277 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1278 vcs.srclib = (name, number, sdir)
1280 vcs.gotorevision(ref, refresh)
1287 libdir = os.path.join(sdir, subdir)
1288 elif srclib["Subdir"]:
1289 for subdir in srclib["Subdir"]:
1290 libdir_candidate = os.path.join(sdir, subdir)
1291 if os.path.exists(libdir_candidate):
1292 libdir = libdir_candidate
1298 remove_signing_keys(sdir)
1299 remove_debuggable_flags(sdir)
1303 if srclib["Prepare"]:
1304 cmd = replace_config_vars(srclib["Prepare"], build)
1306 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1307 if p.returncode != 0:
1308 raise BuildException("Error running prepare command for srclib %s"
1314 return (name, number, libdir)
1317 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1320 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1321 """ Prepare the source code for a particular build
1323 :param vcs: the appropriate vcs object for the application
1324 :param app: the application details from the metadata
1325 :param build: the build details from the metadata
1326 :param build_dir: the path to the build directory, usually 'build/app.id'
1327 :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
1328 :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
1330 Returns the (root, srclibpaths) where:
1331 :param root: is the root directory, which may be the same as 'build_dir' or may
1332 be a subdirectory of it.
1333 :param srclibpaths: is information on the srclibs being used
1336 # Optionally, the actual app source can be in a subdirectory
1338 root_dir = os.path.join(build_dir, build.subdir)
1340 root_dir = build_dir
1342 # Get a working copy of the right revision
1343 logging.info("Getting source for revision " + build.commit)
1344 vcs.gotorevision(build.commit, refresh)
1346 # Initialise submodules if required
1347 if build.submodules:
1348 logging.info("Initialising submodules")
1349 vcs.initsubmodules()
1351 # Check that a subdir (if we're using one) exists. This has to happen
1352 # after the checkout, since it might not exist elsewhere
1353 if not os.path.exists(root_dir):
1354 raise BuildException('Missing subdir ' + root_dir)
1356 # Run an init command if one is required
1358 cmd = replace_config_vars(build.init, build)
1359 logging.info("Running 'init' commands in %s" % root_dir)
1361 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1362 if p.returncode != 0:
1363 raise BuildException("Error running init command for %s:%s" %
1364 (app.id, build.versionName), p.output)
1366 # Apply patches if any
1368 logging.info("Applying patches")
1369 for patch in build.patch:
1370 patch = patch.strip()
1371 logging.info("Applying " + patch)
1372 patch_path = os.path.join('metadata', app.id, patch)
1373 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1374 if p.returncode != 0:
1375 raise BuildException("Failed to apply patch %s" % patch_path)
1377 # Get required source libraries
1380 logging.info("Collecting source libraries")
1381 for lib in build.srclibs:
1382 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1383 refresh=refresh, build=build))
1385 for name, number, libpath in srclibpaths:
1386 place_srclib(root_dir, int(number) if number else None, libpath)
1388 basesrclib = vcs.getsrclib()
1389 # If one was used for the main source, add that too.
1391 srclibpaths.append(basesrclib)
1393 # Update the local.properties file
1394 localprops = [os.path.join(build_dir, 'local.properties')]
1396 parts = build.subdir.split(os.sep)
1399 cur = os.path.join(cur, d)
1400 localprops += [os.path.join(cur, 'local.properties')]
1401 for path in localprops:
1403 if os.path.isfile(path):
1404 logging.info("Updating local.properties file at %s" % path)
1405 with open(path, 'r', encoding='iso-8859-1') as f:
1409 logging.info("Creating local.properties file at %s" % path)
1410 # Fix old-fashioned 'sdk-location' by copying
1411 # from sdk.dir, if necessary
1413 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1414 re.S | re.M).group(1)
1415 props += "sdk-location=%s\n" % sdkloc
1417 props += "sdk.dir=%s\n" % config['sdk_path']
1418 props += "sdk-location=%s\n" % config['sdk_path']
1419 ndk_path = build.ndk_path()
1420 # if for any reason the path isn't valid or the directory
1421 # doesn't exist, some versions of Gradle will error with a
1422 # cryptic message (even if the NDK is not even necessary).
1423 # https://gitlab.com/fdroid/fdroidserver/issues/171
1424 if ndk_path and os.path.exists(ndk_path):
1426 props += "ndk.dir=%s\n" % ndk_path
1427 props += "ndk-location=%s\n" % ndk_path
1428 # Add java.encoding if necessary
1430 props += "java.encoding=%s\n" % build.encoding
1431 with open(path, 'w', encoding='iso-8859-1') as f:
1435 if build.build_method() == 'gradle':
1436 flavours = build.gradle
1439 n = build.target.split('-')[1]
1440 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1441 r'compileSdkVersion %s' % n,
1442 os.path.join(root_dir, 'build.gradle'))
1444 # Remove forced debuggable flags
1445 remove_debuggable_flags(root_dir)
1447 # Insert version code and number into the manifest if necessary
1448 if build.forceversion:
1449 logging.info("Changing the version name")
1450 for path in manifest_paths(root_dir, flavours):
1451 if not os.path.isfile(path):
1453 if has_extension(path, 'xml'):
1454 regsub_file(r'android:versionName="[^"]*"',
1455 r'android:versionName="%s"' % build.versionName,
1457 elif has_extension(path, 'gradle'):
1458 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1459 r"""\1versionName '%s'""" % build.versionName,
1462 if build.forcevercode:
1463 logging.info("Changing the version code")
1464 for path in manifest_paths(root_dir, flavours):
1465 if not os.path.isfile(path):
1467 if has_extension(path, 'xml'):
1468 regsub_file(r'android:versionCode="[^"]*"',
1469 r'android:versionCode="%s"' % build.versionCode,
1471 elif has_extension(path, 'gradle'):
1472 regsub_file(r'versionCode[ =]+[0-9]+',
1473 r'versionCode %s' % build.versionCode,
1476 # Delete unwanted files
1478 logging.info("Removing specified files")
1479 for part in getpaths(build_dir, build.rm):
1480 dest = os.path.join(build_dir, part)
1481 logging.info("Removing {0}".format(part))
1482 if os.path.lexists(dest):
1483 if os.path.islink(dest):
1484 FDroidPopen(['unlink', dest], output=False)
1486 FDroidPopen(['rm', '-rf', dest], output=False)
1488 logging.info("...but it didn't exist")
1490 remove_signing_keys(build_dir)
1492 # Add required external libraries
1494 logging.info("Collecting prebuilt libraries")
1495 libsdir = os.path.join(root_dir, 'libs')
1496 if not os.path.exists(libsdir):
1498 for lib in build.extlibs:
1500 logging.info("...installing extlib {0}".format(lib))
1501 libf = os.path.basename(lib)
1502 libsrc = os.path.join(extlib_dir, lib)
1503 if not os.path.exists(libsrc):
1504 raise BuildException("Missing extlib file {0}".format(libsrc))
1505 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1507 # Run a pre-build command if one is required
1509 logging.info("Running 'prebuild' commands in %s" % root_dir)
1511 cmd = replace_config_vars(build.prebuild, build)
1513 # Substitute source library paths into prebuild commands
1514 for name, number, libpath in srclibpaths:
1515 libpath = os.path.relpath(libpath, root_dir)
1516 cmd = cmd.replace('$$' + name + '$$', libpath)
1518 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1519 if p.returncode != 0:
1520 raise BuildException("Error running prebuild command for %s:%s" %
1521 (app.id, build.versionName), p.output)
1523 # Generate (or update) the ant build file, build.xml...
1524 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1525 parms = ['android', 'update', 'lib-project']
1526 lparms = ['android', 'update', 'project']
1529 parms += ['-t', build.target]
1530 lparms += ['-t', build.target]
1531 if build.androidupdate:
1532 update_dirs = build.androidupdate
1534 update_dirs = ant_subprojects(root_dir) + ['.']
1536 for d in update_dirs:
1537 subdir = os.path.join(root_dir, d)
1539 logging.debug("Updating main project")
1540 cmd = parms + ['-p', d]
1542 logging.debug("Updating subproject %s" % d)
1543 cmd = lparms + ['-p', d]
1544 p = SdkToolsPopen(cmd, cwd=root_dir)
1545 # Check to see whether an error was returned without a proper exit
1546 # code (this is the case for the 'no target set or target invalid'
1548 if p.returncode != 0 or p.output.startswith("Error: "):
1549 raise BuildException("Failed to update project at %s" % d, p.output)
1550 # Clean update dirs via ant
1552 logging.info("Cleaning subproject %s" % d)
1553 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1555 return (root_dir, srclibpaths)
1558 # Extend via globbing the paths from a field and return them as a map from
1559 # original path to resulting paths
1560 def getpaths_map(build_dir, globpaths):
1564 full_path = os.path.join(build_dir, p)
1565 full_path = os.path.normpath(full_path)
1566 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1568 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1572 # Extend via globbing the paths from a field and return them as a set
1573 def getpaths(build_dir, globpaths):
1574 paths_map = getpaths_map(build_dir, globpaths)
1576 for k, v in paths_map.items():
1583 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1587 """permanent store of existing APKs with the date they were added
1589 This is currently the only way to permanently store the "updated"
1594 self.path = os.path.join('stats', 'known_apks.txt')
1596 if os.path.isfile(self.path):
1597 with open(self.path, 'r', encoding='utf8') as f:
1599 t = line.rstrip().split(' ')
1601 self.apks[t[0]] = (t[1], None)
1603 self.apks[t[0]] = (t[1], datetime.strptime(t[2], '%Y-%m-%d'))
1604 self.changed = False
1606 def writeifchanged(self):
1607 if not self.changed:
1610 if not os.path.exists('stats'):
1614 for apk, app in self.apks.items():
1616 line = apk + ' ' + appid
1618 line += ' ' + added.strftime('%Y-%m-%d')
1621 with open(self.path, 'w', encoding='utf8') as f:
1622 for line in sorted(lst, key=natural_key):
1623 f.write(line + '\n')
1625 def recordapk(self, apkName, app, default_date=None):
1627 Record an apk (if it's new, otherwise does nothing)
1628 Returns the date it was added as a datetime instance
1630 if apkName not in self.apks:
1631 if default_date is None:
1632 default_date = datetime.utcnow()
1633 self.apks[apkName] = (app, default_date)
1635 _, added = self.apks[apkName]
1638 # Look up information - given the 'apkname', returns (app id, date added/None).
1639 # Or returns None for an unknown apk.
1640 def getapp(self, apkname):
1641 if apkname in self.apks:
1642 return self.apks[apkname]
1645 # Get the most recent 'num' apps added to the repo, as a list of package ids
1646 # with the most recent first.
1647 def getlatest(self, num):
1649 for apk, app in self.apks.items():
1653 if apps[appid] > added:
1657 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1658 lst = [app for app, _ in sortedapps]
1663 def get_file_extension(filename):
1664 """get the normalized file extension, can be blank string but never None"""
1665 if isinstance(filename, bytes):
1666 filename = filename.decode('utf-8')
1667 return os.path.splitext(filename)[1].lower()[1:]
1670 def get_apk_debuggable_aapt(apkfile):
1671 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1673 if p.returncode != 0:
1674 raise FDroidException("Failed to get apk manifest information")
1675 for line in p.output.splitlines():
1676 if 'android:debuggable' in line and not line.endswith('0x0'):
1681 def get_apk_debuggable_androguard(apkfile):
1683 from androguard.core.bytecodes.apk import APK
1685 raise FDroidException("androguard library is not installed and aapt not present")
1687 apkobject = APK(apkfile)
1688 if apkobject.is_valid_APK():
1689 debuggable = apkobject.get_element("application", "debuggable")
1690 if debuggable is not None:
1691 return bool(strtobool(debuggable))
1695 def isApkAndDebuggable(apkfile):
1696 """Returns True if the given file is an APK and is debuggable
1698 :param apkfile: full path to the apk to check"""
1700 if get_file_extension(apkfile) != 'apk':
1703 if SdkToolsPopen(['aapt', 'version'], output=False):
1704 return get_apk_debuggable_aapt(apkfile)
1706 return get_apk_debuggable_androguard(apkfile)
1709 def get_apk_id_aapt(apkfile):
1710 """Extrat identification information from APK using aapt.
1712 :param apkfile: path to an APK file.
1713 :returns: triplet (appid, version code, version name)
1715 r = re.compile("package: name='(?P<appid>.*)' versionCode='(?P<vercode>.*)' versionName='(?P<vername>.*)' platformBuildVersionName='.*'")
1716 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1717 for line in p.output.splitlines():
1720 return m.group('appid'), m.group('vercode'), m.group('vername')
1721 raise FDroidException("reading identification failed, APK invalid: '{}'".format(apkfile))
1726 self.returncode = None
1730 def SdkToolsPopen(commands, cwd=None, output=True):
1732 if cmd not in config:
1733 config[cmd] = find_sdk_tools_cmd(commands[0])
1734 abscmd = config[cmd]
1736 raise FDroidException("Could not find '%s' on your system" % cmd)
1738 test_aapt_version(config['aapt'])
1739 return FDroidPopen([abscmd] + commands[1:],
1740 cwd=cwd, output=output)
1743 def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
1745 Run a command and capture the possibly huge output as bytes.
1747 :param commands: command and argument list like in subprocess.Popen
1748 :param cwd: optionally specifies a working directory
1749 :param envs: a optional dictionary of environment variables and their values
1750 :returns: A PopenResult.
1755 set_FDroidPopen_env()
1757 process_env = env.copy()
1758 if envs is not None and len(envs) > 0:
1759 process_env.update(envs)
1762 cwd = os.path.normpath(cwd)
1763 logging.debug("Directory: %s" % cwd)
1764 logging.debug("> %s" % ' '.join(commands))
1766 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
1767 result = PopenResult()
1770 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
1771 stdout=subprocess.PIPE, stderr=stderr_param)
1772 except OSError as e:
1773 raise BuildException("OSError while trying to execute " +
1774 ' '.join(commands) + ': ' + str(e))
1776 if not stderr_to_stdout and options.verbose:
1777 stderr_queue = Queue()
1778 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
1780 while not stderr_reader.eof():
1781 while not stderr_queue.empty():
1782 line = stderr_queue.get()
1783 sys.stderr.buffer.write(line)
1788 stdout_queue = Queue()
1789 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1792 # Check the queue for output (until there is no more to get)
1793 while not stdout_reader.eof():
1794 while not stdout_queue.empty():
1795 line = stdout_queue.get()
1796 if output and options.verbose:
1797 # Output directly to console
1798 sys.stderr.buffer.write(line)
1804 result.returncode = p.wait()
1805 result.output = buf.getvalue()
1807 # make sure all filestreams of the subprocess are closed
1808 for streamvar in ['stdin', 'stdout', 'stderr']:
1809 if hasattr(p, streamvar):
1810 stream = getattr(p, streamvar)
1816 def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
1818 Run a command and capture the possibly huge output as a str.
1820 :param commands: command and argument list like in subprocess.Popen
1821 :param cwd: optionally specifies a working directory
1822 :param envs: a optional dictionary of environment variables and their values
1823 :returns: A PopenResult.
1825 result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
1826 result.output = result.output.decode('utf-8', 'ignore')
1830 gradle_comment = re.compile(r'[ ]*//')
1831 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1832 gradle_line_matches = [
1833 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1834 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1835 re.compile(r'.*\.readLine\(.*'),
1839 def remove_signing_keys(build_dir):
1840 for root, dirs, files in os.walk(build_dir):
1841 if 'build.gradle' in files:
1842 path = os.path.join(root, 'build.gradle')
1844 with open(path, "r", encoding='utf8') as o:
1845 lines = o.readlines()
1851 with open(path, "w", encoding='utf8') as o:
1852 while i < len(lines):
1855 while line.endswith('\\\n'):
1856 line = line.rstrip('\\\n') + lines[i]
1859 if gradle_comment.match(line):
1864 opened += line.count('{')
1865 opened -= line.count('}')
1868 if gradle_signing_configs.match(line):
1873 if any(s.match(line) for s in gradle_line_matches):
1881 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1884 'project.properties',
1886 'default.properties',
1887 'ant.properties', ]:
1888 if propfile in files:
1889 path = os.path.join(root, propfile)
1891 with open(path, "r", encoding='iso-8859-1') as o:
1892 lines = o.readlines()
1896 with open(path, "w", encoding='iso-8859-1') as o:
1898 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1905 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1908 def set_FDroidPopen_env(build=None):
1910 set up the environment variables for the build environment
1912 There is only a weak standard, the variables used by gradle, so also set
1913 up the most commonly used environment variables for SDK and NDK. Also, if
1914 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
1916 global env, orig_path
1920 orig_path = env['PATH']
1921 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
1922 env[n] = config['sdk_path']
1923 for k, v in config['java_paths'].items():
1924 env['JAVA%s_HOME' % k] = v
1926 missinglocale = True
1927 for k, v in env.items():
1928 if k == 'LANG' and v != 'C':
1929 missinglocale = False
1931 missinglocale = False
1933 env['LANG'] = 'en_US.UTF-8'
1935 if build is not None:
1936 path = build.ndk_path()
1937 paths = orig_path.split(os.pathsep)
1938 if path not in paths:
1939 paths = [path] + paths
1940 env['PATH'] = os.pathsep.join(paths)
1941 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
1942 env[n] = build.ndk_path()
1945 def replace_build_vars(cmd, build):
1946 cmd = cmd.replace('$$COMMIT$$', build.commit)
1947 cmd = cmd.replace('$$VERSION$$', build.versionName)
1948 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
1952 def replace_config_vars(cmd, build):
1953 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1954 cmd = cmd.replace('$$NDK$$', build.ndk_path())
1955 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1956 cmd = cmd.replace('$$QT$$', config['qt_sdk_path'] or '')
1957 if build is not None:
1958 cmd = replace_build_vars(cmd, build)
1962 def place_srclib(root_dir, number, libpath):
1965 relpath = os.path.relpath(libpath, root_dir)
1966 proppath = os.path.join(root_dir, 'project.properties')
1969 if os.path.isfile(proppath):
1970 with open(proppath, "r", encoding='iso-8859-1') as o:
1971 lines = o.readlines()
1973 with open(proppath, "w", encoding='iso-8859-1') as o:
1976 if line.startswith('android.library.reference.%d=' % number):
1977 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1982 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1985 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
1988 def metadata_get_sigdir(appid, vercode=None):
1989 """Get signature directory for app"""
1991 return os.path.join('metadata', appid, 'signatures', vercode)
1993 return os.path.join('metadata', appid, 'signatures')
1996 def apk_extract_signatures(apkpath, outdir, manifest=True):
1997 """Extracts a signature files from APK and puts them into target directory.
1999 :param apkpath: location of the apk
2000 :param outdir: folder where the extracted signature files will be stored
2001 :param manifest: (optionally) disable extracting manifest file
2003 with ZipFile(apkpath, 'r') as in_apk:
2004 for f in in_apk.infolist():
2005 if apk_sigfile.match(f.filename) or \
2006 (manifest and f.filename == 'META-INF/MANIFEST.MF'):
2007 newpath = os.path.join(outdir, os.path.basename(f.filename))
2008 with open(newpath, 'wb') as out_file:
2009 out_file.write(in_apk.read(f.filename))
2012 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
2013 """Verify that two apks are the same
2015 One of the inputs is signed, the other is unsigned. The signature metadata
2016 is transferred from the signed to the unsigned apk, and then jarsigner is
2017 used to verify that the signature from the signed apk is also varlid for
2018 the unsigned one. If the APK given as unsigned actually does have a
2019 signature, it will be stripped out and ignored.
2021 There are two SHA1 git commit IDs that fdroidserver includes in the builds
2022 it makes: fdroidserverid and buildserverid. Originally, these were inserted
2023 into AndroidManifest.xml, but that makes the build not reproducible. So
2024 instead they are included as separate files in the APK's META-INF/ folder.
2025 If those files exist in the signed APK, they will be part of the signature
2026 and need to also be included in the unsigned APK for it to validate.
2028 :param signed_apk: Path to a signed apk file
2029 :param unsigned_apk: Path to an unsigned apk file expected to match it
2030 :param tmp_dir: Path to directory for temporary files
2031 :returns: None if the verification is successful, otherwise a string
2032 describing what went wrong.
2035 signed = ZipFile(signed_apk, 'r')
2036 meta_inf_files = ['META-INF/MANIFEST.MF']
2037 for f in signed.namelist():
2038 if apk_sigfile.match(f) \
2039 or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
2040 meta_inf_files.append(f)
2041 if len(meta_inf_files) < 3:
2042 return "Signature files missing from {0}".format(signed_apk)
2044 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2045 unsigned = ZipFile(unsigned_apk, 'r')
2046 # only read the signature from the signed APK, everything else from unsigned
2047 with ZipFile(tmp_apk, 'w') as tmp:
2048 for filename in meta_inf_files:
2049 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2050 for info in unsigned.infolist():
2051 if info.filename in meta_inf_files:
2052 logging.warning('Ignoring ' + info.filename + ' from ' + unsigned_apk)
2054 if info.filename in tmp.namelist():
2055 return "duplicate filename found: " + info.filename
2056 tmp.writestr(info, unsigned.read(info.filename))
2060 verified = verify_apk_signature(tmp_apk)
2063 logging.info("...NOT verified - {0}".format(tmp_apk))
2064 return compare_apks(signed_apk, tmp_apk, tmp_dir,
2065 os.path.dirname(unsigned_apk))
2067 logging.info("...successfully verified")
2071 def verify_apk_signature(apk, jar=False):
2072 """verify the signature on an APK
2074 Try to use apksigner whenever possible since jarsigner is very
2075 shitty: unsigned APKs pass as "verified"! So this has to turn on
2076 -strict then check for result 4.
2078 You can set :param: jar to True if you want to use this method
2079 to verify jar signatures.
2081 if set_command_in_config('apksigner'):
2082 args = [config['apksigner'], 'verify']
2084 args += ['--min-sdk-version=1']
2085 return subprocess.call(args + [apk]) == 0
2087 logging.warning("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner")
2088 return subprocess.call([config['jarsigner'], '-strict', '-verify', apk]) == 4
2091 def verify_old_apk_signature(apk):
2092 """verify the signature on an archived APK, supporting deprecated algorithms
2094 F-Droid aims to keep every single binary that it ever published. Therefore,
2095 it needs to be able to verify APK signatures that include deprecated/removed
2096 algorithms. For example, jarsigner treats an MD5 signature as unsigned.
2098 jarsigner passes unsigned APKs as "verified"! So this has to turn
2099 on -strict then check for result 4.
2103 _java_security = os.path.join(os.getcwd(), '.java.security')
2104 with open(_java_security, 'w') as fp:
2105 fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
2107 return subprocess.call([config['jarsigner'], '-J-Djava.security.properties=' + _java_security,
2108 '-strict', '-verify', apk]) == 4
2111 apk_badchars = re.compile('''[/ :;'"]''')
2114 def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
2117 Returns None if the apk content is the same (apart from the signing key),
2118 otherwise a string describing what's different, or what went wrong when
2119 trying to do the comparison.
2125 absapk1 = os.path.abspath(apk1)
2126 absapk2 = os.path.abspath(apk2)
2128 if set_command_in_config('diffoscope'):
2129 logfilename = os.path.join(log_dir, os.path.basename(absapk1))
2130 htmlfile = logfilename + '.diffoscope.html'
2131 textfile = logfilename + '.diffoscope.txt'
2132 if subprocess.call([config['diffoscope'],
2133 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2134 '--html', htmlfile, '--text', textfile,
2135 absapk1, absapk2]) != 0:
2136 return("Failed to unpack " + apk1)
2138 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2139 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2140 for d in [apk1dir, apk2dir]:
2141 if os.path.exists(d):
2144 os.mkdir(os.path.join(d, 'jar-xf'))
2146 if subprocess.call(['jar', 'xf',
2147 os.path.abspath(apk1)],
2148 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2149 return("Failed to unpack " + apk1)
2150 if subprocess.call(['jar', 'xf',
2151 os.path.abspath(apk2)],
2152 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2153 return("Failed to unpack " + apk2)
2155 if set_command_in_config('apktool'):
2156 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2158 return("Failed to unpack " + apk1)
2159 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2161 return("Failed to unpack " + apk2)
2163 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2164 lines = p.output.splitlines()
2165 if len(lines) != 1 or 'META-INF' not in lines[0]:
2166 if set_command_in_config('meld'):
2167 p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
2168 return("Unexpected diff output - " + p.output)
2170 # since everything verifies, delete the comparison to keep cruft down
2171 shutil.rmtree(apk1dir)
2172 shutil.rmtree(apk2dir)
2174 # If we get here, it seems like they're the same!
2178 def set_command_in_config(command):
2179 '''Try to find specified command in the path, if it hasn't been
2180 manually set in config.py. If found, it is added to the config
2181 dict. The return value says whether the command is available.
2184 if command in config:
2187 tmp = find_command(command)
2189 config[command] = tmp
2194 def find_command(command):
2195 '''find the full path of a command, or None if it can't be found in the PATH'''
2198 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2200 fpath, fname = os.path.split(command)
2205 for path in os.environ["PATH"].split(os.pathsep):
2206 path = path.strip('"')
2207 exe_file = os.path.join(path, command)
2208 if is_exe(exe_file):
2215 '''generate a random password for when generating keys'''
2216 h = hashlib.sha256()
2217 h.update(os.urandom(16)) # salt
2218 h.update(socket.getfqdn().encode('utf-8'))
2219 passwd = base64.b64encode(h.digest()).strip()
2220 return passwd.decode('utf-8')
2223 def genkeystore(localconfig):
2225 Generate a new key with password provided in :param localconfig and add it to new keystore
2226 :return: hexed public key, public key fingerprint
2228 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2229 keystoredir = os.path.dirname(localconfig['keystore'])
2230 if keystoredir is None or keystoredir == '':
2231 keystoredir = os.path.join(os.getcwd(), keystoredir)
2232 if not os.path.exists(keystoredir):
2233 os.makedirs(keystoredir, mode=0o700)
2236 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
2237 'FDROID_KEY_PASS': localconfig['keypass'],
2239 p = FDroidPopen([config['keytool'], '-genkey',
2240 '-keystore', localconfig['keystore'],
2241 '-alias', localconfig['repo_keyalias'],
2242 '-keyalg', 'RSA', '-keysize', '4096',
2243 '-sigalg', 'SHA256withRSA',
2244 '-validity', '10000',
2245 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2246 '-keypass:env', 'FDROID_KEY_PASS',
2247 '-dname', localconfig['keydname']], envs=env_vars)
2248 if p.returncode != 0:
2249 raise BuildException("Failed to generate key", p.output)
2250 os.chmod(localconfig['keystore'], 0o0600)
2251 if not options.quiet:
2252 # now show the lovely key that was just generated
2253 p = FDroidPopen([config['keytool'], '-list', '-v',
2254 '-keystore', localconfig['keystore'],
2255 '-alias', localconfig['repo_keyalias'],
2256 '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
2257 logging.info(p.output.strip() + '\n\n')
2258 # get the public key
2259 p = FDroidPopenBytes([config['keytool'], '-exportcert',
2260 '-keystore', localconfig['keystore'],
2261 '-alias', localconfig['repo_keyalias'],
2262 '-storepass:env', 'FDROID_KEY_STORE_PASS']
2263 + config['smartcardoptions'],
2264 envs=env_vars, output=False, stderr_to_stdout=False)
2265 if p.returncode != 0 or len(p.output) < 20:
2266 raise BuildException("Failed to get public key", p.output)
2268 fingerprint = get_cert_fingerprint(pubkey)
2269 return hexlify(pubkey), fingerprint
2272 def get_cert_fingerprint(pubkey):
2274 Generate a certificate fingerprint the same way keytool does it
2275 (but with slightly different formatting)
2277 digest = hashlib.sha256(pubkey).digest()
2278 ret = [' '.join("%02X" % b for b in bytearray(digest))]
2279 return " ".join(ret)
2282 def get_certificate(certificate_file):
2284 Extracts a certificate from the given file.
2285 :param certificate_file: file bytes (as string) representing the certificate
2286 :return: A binary representation of the certificate's public key, or None in case of error
2288 content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
2289 if content.getComponentByName('contentType') != rfc2315.signedData:
2291 content = decoder.decode(content.getComponentByName('content'),
2292 asn1Spec=rfc2315.SignedData())[0]
2294 certificates = content.getComponentByName('certificates')
2295 cert = certificates[0].getComponentByName('certificate')
2297 logging.error("Certificates not found.")
2299 return encoder.encode(cert)
2302 def write_to_config(thisconfig, key, value=None, config_file=None):
2303 '''write a key/value to the local config.py
2305 NOTE: only supports writing string variables.
2307 :param thisconfig: config dictionary
2308 :param key: variable name in config.py to be overwritten/added
2309 :param value: optional value to be written, instead of fetched
2310 from 'thisconfig' dictionary.
2313 origkey = key + '_orig'
2314 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2315 cfg = config_file if config_file else 'config.py'
2318 with open(cfg, 'r', encoding="utf-8") as f:
2319 lines = f.readlines()
2321 # make sure the file ends with a carraige return
2323 if not lines[-1].endswith('\n'):
2326 # regex for finding and replacing python string variable
2327 # definitions/initializations
2328 pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
2329 repl = key + ' = "' + value + '"'
2330 pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
2331 repl2 = key + " = '" + value + "'"
2333 # If we replaced this line once, we make sure won't be a
2334 # second instance of this line for this key in the document.
2337 with open(cfg, 'w', encoding="utf-8") as f:
2339 if pattern.match(line) or pattern2.match(line):
2341 line = pattern.sub(repl, line)
2342 line = pattern2.sub(repl2, line)
2353 def parse_xml(path):
2354 return XMLElementTree.parse(path).getroot()
2357 def string_is_integer(string):
2365 def get_per_app_repos():
2366 '''per-app repos are dirs named with the packageName of a single app'''
2368 # Android packageNames are Java packages, they may contain uppercase or
2369 # lowercase letters ('A' through 'Z'), numbers, and underscores
2370 # ('_'). However, individual package name parts may only start with
2371 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2372 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2375 for root, dirs, files in os.walk(os.getcwd()):
2377 print('checking', root, 'for', d)
2378 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2379 # standard parts of an fdroid repo, so never packageNames
2382 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
2388 def is_repo_file(filename):
2389 '''Whether the file in a repo is a build product to be delivered to users'''
2390 if isinstance(filename, str):
2391 filename = filename.encode('utf-8', errors="surrogateescape")
2392 return os.path.isfile(filename) \
2393 and not filename.endswith(b'.asc') \
2394 and not filename.endswith(b'.sig') \
2395 and os.path.basename(filename) not in [
2397 b'index_unsigned.jar',