3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
37 import xml.etree.ElementTree as XMLElementTree
39 from binascii import hexlify
40 from datetime import datetime
41 from distutils.version import LooseVersion
42 from queue import Queue
43 from zipfile import ZipFile
45 from pyasn1.codec.der import decoder, encoder
46 from pyasn1_modules import rfc2315
47 from pyasn1.error import PyAsn1Error
49 from distutils.util import strtobool
51 import fdroidserver.metadata
52 from fdroidserver.exception import FDroidException, VCSException, BuildException
53 from .asynchronousfilereader import AsynchronousFileReader
56 # A signature block file with a .DSA, .RSA, or .EC extension
57 CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
58 APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
59 STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
61 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
70 'sdk_path': "$ANDROID_HOME",
75 'r12b': "$ANDROID_NDK",
81 'build_tools': "25.0.2",
82 'force_build_tools': False,
87 'accepted_formats': ['txt', 'yml'],
88 'sync_from_local_copy_dir': False,
89 'allow_disabled_algorithms': False,
90 'per_app_repos': False,
91 'make_current_version_link': True,
92 'current_version_name_source': 'Name',
93 'update_stats': False,
97 'stats_to_carbon': False,
99 'build_server_always': False,
100 'keystore': 'keystore.jks',
101 'smartcardoptions': [],
111 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
112 'repo_name': "My First FDroid Repo Demo",
113 'repo_icon': "fdroid-icon.png",
114 'repo_description': '''
115 This is a repository of apps to be used with FDroid. Applications in this
116 repository are either official binaries built by the original application
117 developers, or are binaries built from source by the admin of f-droid.org
118 using the tools on https://gitlab.com/u/fdroid.
124 def setup_global_opts(parser):
125 parser.add_argument("-v", "--verbose", action="store_true", default=False,
126 help="Spew out even more information than normal")
127 parser.add_argument("-q", "--quiet", action="store_true", default=False,
128 help="Restrict output to warnings and errors")
131 def fill_config_defaults(thisconfig):
132 for k, v in default_config.items():
133 if k not in thisconfig:
136 # Expand paths (~users and $vars)
137 def expand_path(path):
141 path = os.path.expanduser(path)
142 path = os.path.expandvars(path)
147 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
152 thisconfig[k + '_orig'] = v
154 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
155 if thisconfig['java_paths'] is None:
156 thisconfig['java_paths'] = dict()
158 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
159 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
160 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
161 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
162 if os.getenv('JAVA_HOME') is not None:
163 pathlist.append(os.getenv('JAVA_HOME'))
164 if os.getenv('PROGRAMFILES') is not None:
165 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
166 for d in sorted(pathlist):
167 if os.path.islink(d):
169 j = os.path.basename(d)
170 # the last one found will be the canonical one, so order appropriately
172 r'^1\.([6-9])\.0\.jdk$', # OSX
173 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
174 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
175 r'^jdk([6-9])-openjdk$', # Arch
176 r'^java-([6-9])-openjdk$', # Arch
177 r'^java-([6-9])-jdk$', # Arch (oracle)
178 r'^java-1\.([6-9])\.0-.*$', # RedHat
179 r'^java-([6-9])-oracle$', # Debian WebUpd8
180 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
181 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
183 m = re.match(regex, j)
186 for p in [d, os.path.join(d, 'Contents', 'Home')]:
187 if os.path.exists(os.path.join(p, 'bin', 'javac')):
188 thisconfig['java_paths'][m.group(1)] = p
190 for java_version in ('7', '8', '9'):
191 if java_version not in thisconfig['java_paths']:
193 java_home = thisconfig['java_paths'][java_version]
194 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
195 if os.path.exists(jarsigner):
196 thisconfig['jarsigner'] = jarsigner
197 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
198 break # Java7 is preferred, so quit if found
200 for k in ['ndk_paths', 'java_paths']:
206 thisconfig[k][k2] = exp
207 thisconfig[k][k2 + '_orig'] = v
210 def regsub_file(pattern, repl, path):
211 with open(path, 'rb') as f:
213 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
214 with open(path, 'wb') as f:
218 def read_config(opts, config_file='config.py'):
219 """Read the repository config
221 The config is read from config_file, which is in the current
222 directory when any of the repo management commands are used. If
223 there is a local metadata file in the git repo, then config.py is
224 not required, just use defaults.
227 global config, options
229 if config is not None:
236 if os.path.isfile(config_file):
237 logging.debug("Reading %s" % config_file)
238 with io.open(config_file, "rb") as f:
239 code = compile(f.read(), config_file, 'exec')
240 exec(code, None, config)
241 elif len(get_local_metadata_files()) == 0:
242 raise FDroidException("Missing config file - is this a repo directory?")
244 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
246 if not type(config[k]) in (str, list, tuple):
247 logging.warn('"' + k + '" will be in random order!'
248 + ' Use () or [] brackets if order is important!')
250 # smartcardoptions must be a list since its command line args for Popen
251 if 'smartcardoptions' in config:
252 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
253 elif 'keystore' in config and config['keystore'] == 'NONE':
254 # keystore='NONE' means use smartcard, these are required defaults
255 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
256 'SunPKCS11-OpenSC', '-providerClass',
257 'sun.security.pkcs11.SunPKCS11',
258 '-providerArg', 'opensc-fdroid.cfg']
260 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
261 st = os.stat(config_file)
262 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
263 logging.warning("unsafe permissions on {0} (should be 0600)!".format(config_file))
265 fill_config_defaults(config)
267 for k in ["repo_description", "archive_description"]:
269 config[k] = clean_description(config[k])
271 if 'serverwebroot' in config:
272 if isinstance(config['serverwebroot'], str):
273 roots = [config['serverwebroot']]
274 elif all(isinstance(item, str) for item in config['serverwebroot']):
275 roots = config['serverwebroot']
277 raise TypeError('only accepts strings, lists, and tuples')
279 for rootstr in roots:
280 # since this is used with rsync, where trailing slashes have
281 # meaning, ensure there is always a trailing slash
282 if rootstr[-1] != '/':
284 rootlist.append(rootstr.replace('//', '/'))
285 config['serverwebroot'] = rootlist
287 if 'servergitmirrors' in config:
288 if isinstance(config['servergitmirrors'], str):
289 roots = [config['servergitmirrors']]
290 elif all(isinstance(item, str) for item in config['servergitmirrors']):
291 roots = config['servergitmirrors']
293 raise TypeError('only accepts strings, lists, and tuples')
294 config['servergitmirrors'] = roots
299 def find_sdk_tools_cmd(cmd):
300 '''find a working path to a tool from the Android SDK'''
303 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
304 # try to find a working path to this command, in all the recent possible paths
305 if 'build_tools' in config:
306 build_tools = os.path.join(config['sdk_path'], 'build-tools')
307 # if 'build_tools' was manually set and exists, check only that one
308 configed_build_tools = os.path.join(build_tools, config['build_tools'])
309 if os.path.exists(configed_build_tools):
310 tooldirs.append(configed_build_tools)
312 # no configed version, so hunt known paths for it
313 for f in sorted(os.listdir(build_tools), reverse=True):
314 if os.path.isdir(os.path.join(build_tools, f)):
315 tooldirs.append(os.path.join(build_tools, f))
316 tooldirs.append(build_tools)
317 sdk_tools = os.path.join(config['sdk_path'], 'tools')
318 if os.path.exists(sdk_tools):
319 tooldirs.append(sdk_tools)
320 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
321 if os.path.exists(sdk_platform_tools):
322 tooldirs.append(sdk_platform_tools)
323 tooldirs.append('/usr/bin')
325 path = os.path.join(d, cmd)
326 if os.path.isfile(path):
328 test_aapt_version(path)
330 # did not find the command, exit with error message
331 ensure_build_tools_exists(config)
334 def test_aapt_version(aapt):
335 '''Check whether the version of aapt is new enough'''
336 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
337 if output is None or output == '':
338 logging.error(aapt + ' failed to execute!')
340 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
345 # the Debian package has the version string like "v0.2-23.0.2"
346 if '.' not in bugfix and LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.2166767'):
347 logging.warning(aapt + ' is too old, fdroid requires build-tools-23.0.0 or newer!')
349 logging.warning('Unknown version of aapt, might cause problems: ' + output)
352 def test_sdk_exists(thisconfig):
353 if 'sdk_path' not in thisconfig:
354 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
355 test_aapt_version(thisconfig['aapt'])
358 logging.error("'sdk_path' not set in config.py!")
360 if thisconfig['sdk_path'] == default_config['sdk_path']:
361 logging.error('No Android SDK found!')
362 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
363 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
365 if not os.path.exists(thisconfig['sdk_path']):
366 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
368 if not os.path.isdir(thisconfig['sdk_path']):
369 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
371 for d in ['build-tools', 'platform-tools', 'tools']:
372 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
373 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
374 thisconfig['sdk_path'], d))
379 def ensure_build_tools_exists(thisconfig):
380 if not test_sdk_exists(thisconfig):
381 raise FDroidException("Android SDK not found.")
382 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
383 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
384 if not os.path.isdir(versioned_build_tools):
385 raise FDroidException(
386 'Android Build Tools path "' + versioned_build_tools + '" does not exist!')
389 def get_local_metadata_files():
390 '''get any metadata files local to an app's source repo
392 This tries to ignore anything that does not count as app metdata,
393 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
396 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
399 def read_pkg_args(args, allow_vercodes=False):
401 :param args: arguments in the form of multiple appid:[vc] strings
402 :returns: a dictionary with the set of vercodes specified for each package
410 if allow_vercodes and ':' in p:
411 package, vercode = p.split(':')
413 package, vercode = p, None
414 if package not in vercodes:
415 vercodes[package] = [vercode] if vercode else []
417 elif vercode and vercode not in vercodes[package]:
418 vercodes[package] += [vercode] if vercode else []
423 def read_app_args(args, allapps, allow_vercodes=False):
425 On top of what read_pkg_args does, this returns the whole app metadata, but
426 limiting the builds list to the builds matching the vercodes specified.
429 vercodes = read_pkg_args(args, allow_vercodes)
435 for appid, app in allapps.items():
436 if appid in vercodes:
439 if len(apps) != len(vercodes):
442 logging.critical("No such package: %s" % p)
443 raise FDroidException("Found invalid app ids in arguments")
445 raise FDroidException("No packages specified")
448 for appid, app in apps.items():
452 app.builds = [b for b in app.builds if b.versionCode in vc]
453 if len(app.builds) != len(vercodes[appid]):
455 allvcs = [b.versionCode for b in app.builds]
456 for v in vercodes[appid]:
458 logging.critical("No such vercode %s for app %s" % (v, appid))
461 raise FDroidException("Found invalid vercodes for some apps")
466 def get_extension(filename):
467 base, ext = os.path.splitext(filename)
470 return base, ext.lower()[1:]
473 def has_extension(filename, ext):
474 _, f_ext = get_extension(filename)
478 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
481 def clean_description(description):
482 'Remove unneeded newlines and spaces from a block of description text'
484 # this is split up by paragraph to make removing the newlines easier
485 for paragraph in re.split(r'\n\n', description):
486 paragraph = re.sub('\r', '', paragraph)
487 paragraph = re.sub('\n', ' ', paragraph)
488 paragraph = re.sub(' {2,}', ' ', paragraph)
489 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
490 returnstring += paragraph + '\n\n'
491 return returnstring.rstrip('\n')
494 def publishednameinfo(filename):
495 filename = os.path.basename(filename)
496 m = publish_name_regex.match(filename)
498 result = (m.group(1), m.group(2))
499 except AttributeError:
500 raise FDroidException("Invalid name for published file: %s" % filename)
504 def get_release_filename(app, build):
506 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
508 return "%s_%s.apk" % (app.id, build.versionCode)
511 def get_toolsversion_logname(app, build):
512 return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
515 def getsrcname(app, build):
516 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
528 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
531 def get_build_dir(app):
532 '''get the dir that this app will be built in'''
534 if app.RepoType == 'srclib':
535 return os.path.join('build', 'srclib', app.Repo)
537 return os.path.join('build', app.id)
541 '''checkout code from VCS and return instance of vcs and the build dir'''
542 build_dir = get_build_dir(app)
544 # Set up vcs interface and make sure we have the latest code...
545 logging.debug("Getting {0} vcs interface for {1}"
546 .format(app.RepoType, app.Repo))
547 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
551 vcs = getvcs(app.RepoType, remote, build_dir)
553 return vcs, build_dir
556 def getvcs(vcstype, remote, local):
558 return vcs_git(remote, local)
559 if vcstype == 'git-svn':
560 return vcs_gitsvn(remote, local)
562 return vcs_hg(remote, local)
564 return vcs_bzr(remote, local)
565 if vcstype == 'srclib':
566 if local != os.path.join('build', 'srclib', remote):
567 raise VCSException("Error: srclib paths are hard-coded!")
568 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
570 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
571 raise VCSException("Invalid vcs type " + vcstype)
574 def getsrclibvcs(name):
575 if name not in fdroidserver.metadata.srclibs:
576 raise VCSException("Missing srclib " + name)
577 return fdroidserver.metadata.srclibs[name]['Repo Type']
582 def __init__(self, remote, local):
584 # svn, git-svn and bzr may require auth
586 if self.repotype() in ('git-svn', 'bzr'):
588 if self.repotype == 'git-svn':
589 raise VCSException("Authentication is not supported for git-svn")
590 self.username, remote = remote.split('@')
591 if ':' not in self.username:
592 raise VCSException("Password required with username")
593 self.username, self.password = self.username.split(':')
597 self.clone_failed = False
598 self.refreshed = False
604 # Take the local repository to a clean version of the given revision, which
605 # is specificed in the VCS's native format. Beforehand, the repository can
606 # be dirty, or even non-existent. If the repository does already exist
607 # locally, it will be updated from the origin, but only once in the
608 # lifetime of the vcs object.
609 # None is acceptable for 'rev' if you know you are cloning a clean copy of
610 # the repo - otherwise it must specify a valid revision.
611 def gotorevision(self, rev, refresh=True):
613 if self.clone_failed:
614 raise VCSException("Downloading the repository already failed once, not trying again.")
616 # The .fdroidvcs-id file for a repo tells us what VCS type
617 # and remote that directory was created from, allowing us to drop it
618 # automatically if either of those things changes.
619 fdpath = os.path.join(self.local, '..',
620 '.fdroidvcs-' + os.path.basename(self.local))
621 fdpath = os.path.normpath(fdpath)
622 cdata = self.repotype() + ' ' + self.remote
625 if os.path.exists(self.local):
626 if os.path.exists(fdpath):
627 with open(fdpath, 'r') as f:
628 fsdata = f.read().strip()
633 logging.info("Repository details for %s changed - deleting" % (
637 logging.info("Repository details for %s missing - deleting" % (
640 shutil.rmtree(self.local)
644 self.refreshed = True
647 self.gotorevisionx(rev)
648 except FDroidException as e:
651 # If necessary, write the .fdroidvcs file.
652 if writeback and not self.clone_failed:
653 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
654 with open(fdpath, 'w+') as f:
660 # Derived classes need to implement this. It's called once basic checking
661 # has been performend.
662 def gotorevisionx(self, rev): # pylint: disable=unused-argument
663 raise VCSException("This VCS type doesn't define gotorevisionx")
665 # Initialise and update submodules
666 def initsubmodules(self):
667 raise VCSException('Submodules not supported for this vcs type')
669 # Get a list of all known tags
671 if not self._gettags:
672 raise VCSException('gettags not supported for this vcs type')
674 for tag in self._gettags():
675 if re.match('[-A-Za-z0-9_. /]+$', tag):
679 # Get a list of all the known tags, sorted from newest to oldest
680 def latesttags(self):
681 raise VCSException('latesttags not supported for this vcs type')
683 # Get current commit reference (hash, revision, etc)
685 raise VCSException('getref not supported for this vcs type')
687 # Returns the srclib (name, path) used in setting up the current
698 # If the local directory exists, but is somehow not a git repository, git
699 # will traverse up the directory tree until it finds one that is (i.e.
700 # fdroidserver) and then we'll proceed to destroy it! This is called as
703 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
704 result = p.output.rstrip()
705 if not result.endswith(self.local):
706 raise VCSException('Repository mismatch')
708 def gotorevisionx(self, rev):
709 if not os.path.exists(self.local):
711 p = FDroidPopen(['git', 'clone', self.remote, self.local])
712 if p.returncode != 0:
713 self.clone_failed = True
714 raise VCSException("Git clone failed", p.output)
718 # Discard any working tree changes
719 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
720 'git', 'reset', '--hard'], cwd=self.local, output=False)
721 if p.returncode != 0:
722 raise VCSException("Git reset failed", p.output)
723 # Remove untracked files now, in case they're tracked in the target
724 # revision (it happens!)
725 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
726 'git', 'clean', '-dffx'], cwd=self.local, output=False)
727 if p.returncode != 0:
728 raise VCSException("Git clean failed", p.output)
729 if not self.refreshed:
730 # Get latest commits and tags from remote
731 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
732 if p.returncode != 0:
733 raise VCSException("Git fetch failed", p.output)
734 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
735 if p.returncode != 0:
736 raise VCSException("Git fetch failed", p.output)
737 # Recreate origin/HEAD as git clone would do it, in case it disappeared
738 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
739 if p.returncode != 0:
740 lines = p.output.splitlines()
741 if 'Multiple remote HEAD branches' not in lines[0]:
742 raise VCSException("Git remote set-head failed", p.output)
743 branch = lines[1].split(' ')[-1]
744 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
745 if p2.returncode != 0:
746 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
747 self.refreshed = True
748 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
749 # a github repo. Most of the time this is the same as origin/master.
750 rev = rev or 'origin/HEAD'
751 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
752 if p.returncode != 0:
753 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
754 # Get rid of any uncontrolled files left behind
755 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
756 if p.returncode != 0:
757 raise VCSException("Git clean failed", p.output)
759 def initsubmodules(self):
761 submfile = os.path.join(self.local, '.gitmodules')
762 if not os.path.isfile(submfile):
763 raise VCSException("No git submodules available")
765 # fix submodules not accessible without an account and public key auth
766 with open(submfile, 'r') as f:
767 lines = f.readlines()
768 with open(submfile, 'w') as f:
770 if 'git@github.com' in line:
771 line = line.replace('git@github.com:', 'https://github.com/')
772 if 'git@gitlab.com' in line:
773 line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
776 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
777 if p.returncode != 0:
778 raise VCSException("Git submodule sync failed", p.output)
779 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
780 if p.returncode != 0:
781 raise VCSException("Git submodule update failed", p.output)
785 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
786 return p.output.splitlines()
788 tag_format = re.compile(r'tag: ([^),]*)')
790 def latesttags(self):
792 p = FDroidPopen(['git', 'log', '--tags',
793 '--simplify-by-decoration', '--pretty=format:%d'],
794 cwd=self.local, output=False)
796 for line in p.output.splitlines():
797 for tag in self.tag_format.findall(line):
802 class vcs_gitsvn(vcs):
807 # If the local directory exists, but is somehow not a git repository, git
808 # will traverse up the directory tree until it finds one that is (i.e.
809 # fdroidserver) and then we'll proceed to destory it! This is called as
812 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
813 result = p.output.rstrip()
814 if not result.endswith(self.local):
815 raise VCSException('Repository mismatch')
817 def gotorevisionx(self, rev):
818 if not os.path.exists(self.local):
820 gitsvn_args = ['git', 'svn', 'clone']
821 if ';' in self.remote:
822 remote_split = self.remote.split(';')
823 for i in remote_split[1:]:
824 if i.startswith('trunk='):
825 gitsvn_args.extend(['-T', i[6:]])
826 elif i.startswith('tags='):
827 gitsvn_args.extend(['-t', i[5:]])
828 elif i.startswith('branches='):
829 gitsvn_args.extend(['-b', i[9:]])
830 gitsvn_args.extend([remote_split[0], self.local])
831 p = FDroidPopen(gitsvn_args, output=False)
832 if p.returncode != 0:
833 self.clone_failed = True
834 raise VCSException("Git svn clone failed", p.output)
836 gitsvn_args.extend([self.remote, self.local])
837 p = FDroidPopen(gitsvn_args, output=False)
838 if p.returncode != 0:
839 self.clone_failed = True
840 raise VCSException("Git svn clone failed", p.output)
844 # Discard any working tree changes
845 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
846 if p.returncode != 0:
847 raise VCSException("Git reset failed", p.output)
848 # Remove untracked files now, in case they're tracked in the target
849 # revision (it happens!)
850 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
851 if p.returncode != 0:
852 raise VCSException("Git clean failed", p.output)
853 if not self.refreshed:
854 # Get new commits, branches and tags from repo
855 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
856 if p.returncode != 0:
857 raise VCSException("Git svn fetch failed")
858 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
859 if p.returncode != 0:
860 raise VCSException("Git svn rebase failed", p.output)
861 self.refreshed = True
863 rev = rev or 'master'
865 nospaces_rev = rev.replace(' ', '%20')
866 # Try finding a svn tag
867 for treeish in ['origin/', '']:
868 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
869 if p.returncode == 0:
871 if p.returncode != 0:
872 # No tag found, normal svn rev translation
873 # Translate svn rev into git format
874 rev_split = rev.split('/')
877 for treeish in ['origin/', '']:
878 if len(rev_split) > 1:
879 treeish += rev_split[0]
880 svn_rev = rev_split[1]
883 # if no branch is specified, then assume trunk (i.e. 'master' branch):
887 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
889 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
890 git_rev = p.output.rstrip()
892 if p.returncode == 0 and git_rev:
895 if p.returncode != 0 or not git_rev:
896 # Try a plain git checkout as a last resort
897 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
898 if p.returncode != 0:
899 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
901 # Check out the git rev equivalent to the svn rev
902 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
903 if p.returncode != 0:
904 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
906 # Get rid of any uncontrolled files left behind
907 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
908 if p.returncode != 0:
909 raise VCSException("Git clean failed", p.output)
913 for treeish in ['origin/', '']:
914 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
920 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
921 if p.returncode != 0:
923 return p.output.strip()
931 def gotorevisionx(self, rev):
932 if not os.path.exists(self.local):
933 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
934 if p.returncode != 0:
935 self.clone_failed = True
936 raise VCSException("Hg clone failed", p.output)
938 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
939 if p.returncode != 0:
940 raise VCSException("Hg status failed", p.output)
941 for line in p.output.splitlines():
942 if not line.startswith('? '):
943 raise VCSException("Unexpected output from hg status -uS: " + line)
944 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
945 if not self.refreshed:
946 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
947 if p.returncode != 0:
948 raise VCSException("Hg pull failed", p.output)
949 self.refreshed = True
951 rev = rev or 'default'
954 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
955 if p.returncode != 0:
956 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
957 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
958 # Also delete untracked files, we have to enable purge extension for that:
959 if "'purge' is provided by the following extension" in p.output:
960 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
961 myfile.write("\n[extensions]\nhgext.purge=\n")
962 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
963 if p.returncode != 0:
964 raise VCSException("HG purge failed", p.output)
965 elif p.returncode != 0:
966 raise VCSException("HG purge failed", p.output)
969 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
970 return p.output.splitlines()[1:]
978 def gotorevisionx(self, rev):
979 if not os.path.exists(self.local):
980 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
981 if p.returncode != 0:
982 self.clone_failed = True
983 raise VCSException("Bzr branch failed", p.output)
985 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
986 if p.returncode != 0:
987 raise VCSException("Bzr revert failed", p.output)
988 if not self.refreshed:
989 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
990 if p.returncode != 0:
991 raise VCSException("Bzr update failed", p.output)
992 self.refreshed = True
994 revargs = list(['-r', rev] if rev else [])
995 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
996 if p.returncode != 0:
997 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1000 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
1001 return [tag.split(' ')[0].strip() for tag in
1002 p.output.splitlines()]
1005 def unescape_string(string):
1008 if string[0] == '"' and string[-1] == '"':
1011 return string.replace("\\'", "'")
1014 def retrieve_string(app_dir, string, xmlfiles=None):
1016 if not string.startswith('@string/'):
1017 return unescape_string(string)
1019 if xmlfiles is None:
1022 os.path.join(app_dir, 'res'),
1023 os.path.join(app_dir, 'src', 'main', 'res'),
1025 for r, d, f in os.walk(res_dir):
1026 if os.path.basename(r) == 'values':
1027 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
1029 name = string[len('@string/'):]
1031 def element_content(element):
1032 if element.text is None:
1034 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1035 return s.decode('utf-8').strip()
1037 for path in xmlfiles:
1038 if not os.path.isfile(path):
1040 xml = parse_xml(path)
1041 element = xml.find('string[@name="' + name + '"]')
1042 if element is not None:
1043 content = element_content(element)
1044 return retrieve_string(app_dir, content, xmlfiles)
1049 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1050 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1053 def manifest_paths(app_dir, flavours):
1054 '''Return list of existing files that will be used to find the highest vercode'''
1056 possible_manifests = \
1057 [os.path.join(app_dir, 'AndroidManifest.xml'),
1058 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1059 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1060 os.path.join(app_dir, 'build.gradle')]
1062 for flavour in flavours:
1063 if flavour == 'yes':
1065 possible_manifests.append(
1066 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1068 return [path for path in possible_manifests if os.path.isfile(path)]
1071 def fetch_real_name(app_dir, flavours):
1072 '''Retrieve the package name. Returns the name, or None if not found.'''
1073 for path in manifest_paths(app_dir, flavours):
1074 if not has_extension(path, 'xml') or not os.path.isfile(path):
1076 logging.debug("fetch_real_name: Checking manifest at " + path)
1077 xml = parse_xml(path)
1078 app = xml.find('application')
1081 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1083 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1084 result = retrieve_string_singleline(app_dir, label)
1086 result = result.strip()
1091 def get_library_references(root_dir):
1093 proppath = os.path.join(root_dir, 'project.properties')
1094 if not os.path.isfile(proppath):
1096 with open(proppath, 'r', encoding='iso-8859-1') as f:
1098 if not line.startswith('android.library.reference.'):
1100 path = line.split('=')[1].strip()
1101 relpath = os.path.join(root_dir, path)
1102 if not os.path.isdir(relpath):
1104 logging.debug("Found subproject at %s" % path)
1105 libraries.append(path)
1109 def ant_subprojects(root_dir):
1110 subprojects = get_library_references(root_dir)
1111 for subpath in subprojects:
1112 subrelpath = os.path.join(root_dir, subpath)
1113 for p in get_library_references(subrelpath):
1114 relp = os.path.normpath(os.path.join(subpath, p))
1115 if relp not in subprojects:
1116 subprojects.insert(0, relp)
1120 def remove_debuggable_flags(root_dir):
1121 # Remove forced debuggable flags
1122 logging.debug("Removing debuggable flags from %s" % root_dir)
1123 for root, dirs, files in os.walk(root_dir):
1124 if 'AndroidManifest.xml' in files:
1125 regsub_file(r'android:debuggable="[^"]*"',
1127 os.path.join(root, 'AndroidManifest.xml'))
1130 vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search
1131 vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1132 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1135 def app_matches_packagename(app, package):
1138 appid = app.UpdateCheckName or app.id
1139 if appid is None or appid == "Ignore":
1141 return appid == package
1144 def parse_androidmanifests(paths, app):
1146 Extract some information from the AndroidManifest.xml at the given path.
1147 Returns (version, vercode, package), any or all of which might be None.
1148 All values returned are strings.
1151 ignoreversions = app.UpdateCheckIgnore
1152 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1155 return (None, None, None)
1163 if not os.path.isfile(path):
1166 logging.debug("Parsing manifest at {0}".format(path))
1171 if has_extension(path, 'gradle'):
1172 with open(path, 'r') as f:
1174 if gradle_comment.match(line):
1176 # Grab first occurence of each to avoid running into
1177 # alternative flavours and builds.
1179 matches = psearch_g(line)
1181 s = matches.group(2)
1182 if app_matches_packagename(app, s):
1185 matches = vnsearch_g(line)
1187 version = matches.group(2)
1189 matches = vcsearch_g(line)
1191 vercode = matches.group(1)
1194 xml = parse_xml(path)
1195 if "package" in xml.attrib:
1196 s = xml.attrib["package"]
1197 if app_matches_packagename(app, s):
1199 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1200 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1201 base_dir = os.path.dirname(path)
1202 version = retrieve_string_singleline(base_dir, version)
1203 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1204 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1205 if string_is_integer(a):
1208 logging.warning("Problem with xml at {0}".format(path))
1210 # Remember package name, may be defined separately from version+vercode
1212 package = max_package
1214 logging.debug("..got package={0}, version={1}, vercode={2}"
1215 .format(package, version, vercode))
1217 # Always grab the package name and version name in case they are not
1218 # together with the highest version code
1219 if max_package is None and package is not None:
1220 max_package = package
1221 if max_version is None and version is not None:
1222 max_version = version
1224 if vercode is not None \
1225 and (max_vercode is None or vercode > max_vercode):
1226 if not ignoresearch or not ignoresearch(version):
1227 if version is not None:
1228 max_version = version
1229 if vercode is not None:
1230 max_vercode = vercode
1231 if package is not None:
1232 max_package = package
1234 max_version = "Ignore"
1236 if max_version is None:
1237 max_version = "Unknown"
1239 if max_package and not is_valid_package_name(max_package):
1240 raise FDroidException("Invalid package name {0}".format(max_package))
1242 return (max_version, max_vercode, max_package)
1245 def is_valid_package_name(name):
1246 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1249 # Get the specified source library.
1250 # Returns the path to it. Normally this is the path to be used when referencing
1251 # it, which may be a subdirectory of the actual project. If you want the base
1252 # directory of the project, pass 'basepath=True'.
1253 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1254 raw=False, prepare=True, preponly=False, refresh=True,
1263 name, ref = spec.split('@')
1265 number, name = name.split(':', 1)
1267 name, subdir = name.split('/', 1)
1269 if name not in fdroidserver.metadata.srclibs:
1270 raise VCSException('srclib ' + name + ' not found.')
1272 srclib = fdroidserver.metadata.srclibs[name]
1274 sdir = os.path.join(srclib_dir, name)
1277 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1278 vcs.srclib = (name, number, sdir)
1280 vcs.gotorevision(ref, refresh)
1287 libdir = os.path.join(sdir, subdir)
1288 elif srclib["Subdir"]:
1289 for subdir in srclib["Subdir"]:
1290 libdir_candidate = os.path.join(sdir, subdir)
1291 if os.path.exists(libdir_candidate):
1292 libdir = libdir_candidate
1298 remove_signing_keys(sdir)
1299 remove_debuggable_flags(sdir)
1303 if srclib["Prepare"]:
1304 cmd = replace_config_vars(srclib["Prepare"], build)
1306 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1307 if p.returncode != 0:
1308 raise BuildException("Error running prepare command for srclib %s"
1314 return (name, number, libdir)
1317 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1320 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1321 """ Prepare the source code for a particular build
1323 :param vcs: the appropriate vcs object for the application
1324 :param app: the application details from the metadata
1325 :param build: the build details from the metadata
1326 :param build_dir: the path to the build directory, usually 'build/app.id'
1327 :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
1328 :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
1330 Returns the (root, srclibpaths) where:
1331 :param root: is the root directory, which may be the same as 'build_dir' or may
1332 be a subdirectory of it.
1333 :param srclibpaths: is information on the srclibs being used
1336 # Optionally, the actual app source can be in a subdirectory
1338 root_dir = os.path.join(build_dir, build.subdir)
1340 root_dir = build_dir
1342 # Get a working copy of the right revision
1343 logging.info("Getting source for revision " + build.commit)
1344 vcs.gotorevision(build.commit, refresh)
1346 # Initialise submodules if required
1347 if build.submodules:
1348 logging.info("Initialising submodules")
1349 vcs.initsubmodules()
1351 # Check that a subdir (if we're using one) exists. This has to happen
1352 # after the checkout, since it might not exist elsewhere
1353 if not os.path.exists(root_dir):
1354 raise BuildException('Missing subdir ' + root_dir)
1356 # Run an init command if one is required
1358 cmd = replace_config_vars(build.init, build)
1359 logging.info("Running 'init' commands in %s" % root_dir)
1361 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1362 if p.returncode != 0:
1363 raise BuildException("Error running init command for %s:%s" %
1364 (app.id, build.versionName), p.output)
1366 # Apply patches if any
1368 logging.info("Applying patches")
1369 for patch in build.patch:
1370 patch = patch.strip()
1371 logging.info("Applying " + patch)
1372 patch_path = os.path.join('metadata', app.id, patch)
1373 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1374 if p.returncode != 0:
1375 raise BuildException("Failed to apply patch %s" % patch_path)
1377 # Get required source libraries
1380 logging.info("Collecting source libraries")
1381 for lib in build.srclibs:
1382 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1383 refresh=refresh, build=build))
1385 for name, number, libpath in srclibpaths:
1386 place_srclib(root_dir, int(number) if number else None, libpath)
1388 basesrclib = vcs.getsrclib()
1389 # If one was used for the main source, add that too.
1391 srclibpaths.append(basesrclib)
1393 # Update the local.properties file
1394 localprops = [os.path.join(build_dir, 'local.properties')]
1396 parts = build.subdir.split(os.sep)
1399 cur = os.path.join(cur, d)
1400 localprops += [os.path.join(cur, 'local.properties')]
1401 for path in localprops:
1403 if os.path.isfile(path):
1404 logging.info("Updating local.properties file at %s" % path)
1405 with open(path, 'r', encoding='iso-8859-1') as f:
1409 logging.info("Creating local.properties file at %s" % path)
1410 # Fix old-fashioned 'sdk-location' by copying
1411 # from sdk.dir, if necessary
1413 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1414 re.S | re.M).group(1)
1415 props += "sdk-location=%s\n" % sdkloc
1417 props += "sdk.dir=%s\n" % config['sdk_path']
1418 props += "sdk-location=%s\n" % config['sdk_path']
1419 ndk_path = build.ndk_path()
1420 # if for any reason the path isn't valid or the directory
1421 # doesn't exist, some versions of Gradle will error with a
1422 # cryptic message (even if the NDK is not even necessary).
1423 # https://gitlab.com/fdroid/fdroidserver/issues/171
1424 if ndk_path and os.path.exists(ndk_path):
1426 props += "ndk.dir=%s\n" % ndk_path
1427 props += "ndk-location=%s\n" % ndk_path
1428 # Add java.encoding if necessary
1430 props += "java.encoding=%s\n" % build.encoding
1431 with open(path, 'w', encoding='iso-8859-1') as f:
1435 if build.build_method() == 'gradle':
1436 flavours = build.gradle
1439 n = build.target.split('-')[1]
1440 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1441 r'compileSdkVersion %s' % n,
1442 os.path.join(root_dir, 'build.gradle'))
1444 # Remove forced debuggable flags
1445 remove_debuggable_flags(root_dir)
1447 # Insert version code and number into the manifest if necessary
1448 if build.forceversion:
1449 logging.info("Changing the version name")
1450 for path in manifest_paths(root_dir, flavours):
1451 if not os.path.isfile(path):
1453 if has_extension(path, 'xml'):
1454 regsub_file(r'android:versionName="[^"]*"',
1455 r'android:versionName="%s"' % build.versionName,
1457 elif has_extension(path, 'gradle'):
1458 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1459 r"""\1versionName '%s'""" % build.versionName,
1462 if build.forcevercode:
1463 logging.info("Changing the version code")
1464 for path in manifest_paths(root_dir, flavours):
1465 if not os.path.isfile(path):
1467 if has_extension(path, 'xml'):
1468 regsub_file(r'android:versionCode="[^"]*"',
1469 r'android:versionCode="%s"' % build.versionCode,
1471 elif has_extension(path, 'gradle'):
1472 regsub_file(r'versionCode[ =]+[0-9]+',
1473 r'versionCode %s' % build.versionCode,
1476 # Delete unwanted files
1478 logging.info("Removing specified files")
1479 for part in getpaths(build_dir, build.rm):
1480 dest = os.path.join(build_dir, part)
1481 logging.info("Removing {0}".format(part))
1482 if os.path.lexists(dest):
1483 if os.path.islink(dest):
1484 FDroidPopen(['unlink', dest], output=False)
1486 FDroidPopen(['rm', '-rf', dest], output=False)
1488 logging.info("...but it didn't exist")
1490 remove_signing_keys(build_dir)
1492 # Add required external libraries
1494 logging.info("Collecting prebuilt libraries")
1495 libsdir = os.path.join(root_dir, 'libs')
1496 if not os.path.exists(libsdir):
1498 for lib in build.extlibs:
1500 logging.info("...installing extlib {0}".format(lib))
1501 libf = os.path.basename(lib)
1502 libsrc = os.path.join(extlib_dir, lib)
1503 if not os.path.exists(libsrc):
1504 raise BuildException("Missing extlib file {0}".format(libsrc))
1505 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1507 # Run a pre-build command if one is required
1509 logging.info("Running 'prebuild' commands in %s" % root_dir)
1511 cmd = replace_config_vars(build.prebuild, build)
1513 # Substitute source library paths into prebuild commands
1514 for name, number, libpath in srclibpaths:
1515 libpath = os.path.relpath(libpath, root_dir)
1516 cmd = cmd.replace('$$' + name + '$$', libpath)
1518 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1519 if p.returncode != 0:
1520 raise BuildException("Error running prebuild command for %s:%s" %
1521 (app.id, build.versionName), p.output)
1523 # Generate (or update) the ant build file, build.xml...
1524 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1525 parms = ['android', 'update', 'lib-project']
1526 lparms = ['android', 'update', 'project']
1529 parms += ['-t', build.target]
1530 lparms += ['-t', build.target]
1531 if build.androidupdate:
1532 update_dirs = build.androidupdate
1534 update_dirs = ant_subprojects(root_dir) + ['.']
1536 for d in update_dirs:
1537 subdir = os.path.join(root_dir, d)
1539 logging.debug("Updating main project")
1540 cmd = parms + ['-p', d]
1542 logging.debug("Updating subproject %s" % d)
1543 cmd = lparms + ['-p', d]
1544 p = SdkToolsPopen(cmd, cwd=root_dir)
1545 # Check to see whether an error was returned without a proper exit
1546 # code (this is the case for the 'no target set or target invalid'
1548 if p.returncode != 0 or p.output.startswith("Error: "):
1549 raise BuildException("Failed to update project at %s" % d, p.output)
1550 # Clean update dirs via ant
1552 logging.info("Cleaning subproject %s" % d)
1553 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1555 return (root_dir, srclibpaths)
1558 # Extend via globbing the paths from a field and return them as a map from
1559 # original path to resulting paths
1560 def getpaths_map(build_dir, globpaths):
1564 full_path = os.path.join(build_dir, p)
1565 full_path = os.path.normpath(full_path)
1566 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1568 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1572 # Extend via globbing the paths from a field and return them as a set
1573 def getpaths(build_dir, globpaths):
1574 paths_map = getpaths_map(build_dir, globpaths)
1576 for k, v in paths_map.items():
1583 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1587 """permanent store of existing APKs with the date they were added
1589 This is currently the only way to permanently store the "updated"
1594 self.path = os.path.join('stats', 'known_apks.txt')
1596 if os.path.isfile(self.path):
1597 with open(self.path, 'r', encoding='utf8') as f:
1599 t = line.rstrip().split(' ')
1601 self.apks[t[0]] = (t[1], None)
1603 self.apks[t[0]] = (t[1], datetime.strptime(t[2], '%Y-%m-%d'))
1604 self.changed = False
1606 def writeifchanged(self):
1607 if not self.changed:
1610 if not os.path.exists('stats'):
1614 for apk, app in self.apks.items():
1616 line = apk + ' ' + appid
1618 line += ' ' + added.strftime('%Y-%m-%d')
1621 with open(self.path, 'w', encoding='utf8') as f:
1622 for line in sorted(lst, key=natural_key):
1623 f.write(line + '\n')
1625 def recordapk(self, apkName, app, default_date=None):
1627 Record an apk (if it's new, otherwise does nothing)
1628 Returns the date it was added as a datetime instance
1630 if apkName not in self.apks:
1631 if default_date is None:
1632 default_date = datetime.utcnow()
1633 self.apks[apkName] = (app, default_date)
1635 _, added = self.apks[apkName]
1638 # Look up information - given the 'apkname', returns (app id, date added/None).
1639 # Or returns None for an unknown apk.
1640 def getapp(self, apkname):
1641 if apkname in self.apks:
1642 return self.apks[apkname]
1645 # Get the most recent 'num' apps added to the repo, as a list of package ids
1646 # with the most recent first.
1647 def getlatest(self, num):
1649 for apk, app in self.apks.items():
1653 if apps[appid] > added:
1657 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1658 lst = [app for app, _ in sortedapps]
1663 def get_file_extension(filename):
1664 """get the normalized file extension, can be blank string but never None"""
1665 if isinstance(filename, bytes):
1666 filename = filename.decode('utf-8')
1667 return os.path.splitext(filename)[1].lower()[1:]
1670 def get_apk_debuggable_aapt(apkfile):
1671 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1673 if p.returncode != 0:
1674 raise FDroidException("Failed to get apk manifest information")
1675 for line in p.output.splitlines():
1676 if 'android:debuggable' in line and not line.endswith('0x0'):
1681 def get_apk_debuggable_androguard(apkfile):
1683 from androguard.core.bytecodes.apk import APK
1685 raise FDroidException("androguard library is not installed and aapt not present")
1687 apkobject = APK(apkfile)
1688 if apkobject.is_valid_APK():
1689 debuggable = apkobject.get_element("application", "debuggable")
1690 if debuggable is not None:
1691 return bool(strtobool(debuggable))
1695 def isApkAndDebuggable(apkfile):
1696 """Returns True if the given file is an APK and is debuggable
1698 :param apkfile: full path to the apk to check"""
1700 if get_file_extension(apkfile) != 'apk':
1703 if SdkToolsPopen(['aapt', 'version'], output=False):
1704 return get_apk_debuggable_aapt(apkfile)
1706 return get_apk_debuggable_androguard(apkfile)
1709 def get_apk_id_aapt(apkfile):
1710 """Extrat identification information from APK using aapt.
1712 :param apkfile: path to an APK file.
1713 :returns: triplet (appid, version code, version name)
1715 r = re.compile("package: name='(?P<appid>.*)' versionCode='(?P<vercode>.*)' versionName='(?P<vername>.*)' platformBuildVersionName='.*'")
1716 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1717 for line in p.output.splitlines():
1720 return m.group('appid'), m.group('vercode'), m.group('vername')
1721 raise FDroidException("reading identification failed, APK invalid: '{}'".format(apkfile))
1726 self.returncode = None
1730 def SdkToolsPopen(commands, cwd=None, output=True):
1732 if cmd not in config:
1733 config[cmd] = find_sdk_tools_cmd(commands[0])
1734 abscmd = config[cmd]
1736 raise FDroidException("Could not find '%s' on your system" % cmd)
1738 test_aapt_version(config['aapt'])
1739 return FDroidPopen([abscmd] + commands[1:],
1740 cwd=cwd, output=output)
1743 def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
1745 Run a command and capture the possibly huge output as bytes.
1747 :param commands: command and argument list like in subprocess.Popen
1748 :param cwd: optionally specifies a working directory
1749 :param envs: a optional dictionary of environment variables and their values
1750 :returns: A PopenResult.
1755 set_FDroidPopen_env()
1757 process_env = env.copy()
1758 if envs is not None and len(envs) > 0:
1759 process_env.update(envs)
1762 cwd = os.path.normpath(cwd)
1763 logging.debug("Directory: %s" % cwd)
1764 logging.debug("> %s" % ' '.join(commands))
1766 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
1767 result = PopenResult()
1770 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
1771 stdout=subprocess.PIPE, stderr=stderr_param)
1772 except OSError as e:
1773 raise BuildException("OSError while trying to execute " +
1774 ' '.join(commands) + ': ' + str(e))
1776 if not stderr_to_stdout and options.verbose:
1777 stderr_queue = Queue()
1778 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
1780 while not stderr_reader.eof():
1781 while not stderr_queue.empty():
1782 line = stderr_queue.get()
1783 sys.stderr.buffer.write(line)
1788 stdout_queue = Queue()
1789 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1792 # Check the queue for output (until there is no more to get)
1793 while not stdout_reader.eof():
1794 while not stdout_queue.empty():
1795 line = stdout_queue.get()
1796 if output and options.verbose:
1797 # Output directly to console
1798 sys.stderr.buffer.write(line)
1804 result.returncode = p.wait()
1805 result.output = buf.getvalue()
1810 def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
1812 Run a command and capture the possibly huge output as a str.
1814 :param commands: command and argument list like in subprocess.Popen
1815 :param cwd: optionally specifies a working directory
1816 :param envs: a optional dictionary of environment variables and their values
1817 :returns: A PopenResult.
1819 result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
1820 result.output = result.output.decode('utf-8', 'ignore')
1824 gradle_comment = re.compile(r'[ ]*//')
1825 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1826 gradle_line_matches = [
1827 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1828 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1829 re.compile(r'.*\.readLine\(.*'),
1833 def remove_signing_keys(build_dir):
1834 for root, dirs, files in os.walk(build_dir):
1835 if 'build.gradle' in files:
1836 path = os.path.join(root, 'build.gradle')
1838 with open(path, "r", encoding='utf8') as o:
1839 lines = o.readlines()
1845 with open(path, "w", encoding='utf8') as o:
1846 while i < len(lines):
1849 while line.endswith('\\\n'):
1850 line = line.rstrip('\\\n') + lines[i]
1853 if gradle_comment.match(line):
1858 opened += line.count('{')
1859 opened -= line.count('}')
1862 if gradle_signing_configs.match(line):
1867 if any(s.match(line) for s in gradle_line_matches):
1875 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1878 'project.properties',
1880 'default.properties',
1881 'ant.properties', ]:
1882 if propfile in files:
1883 path = os.path.join(root, propfile)
1885 with open(path, "r", encoding='iso-8859-1') as o:
1886 lines = o.readlines()
1890 with open(path, "w", encoding='iso-8859-1') as o:
1892 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1899 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1902 def set_FDroidPopen_env(build=None):
1904 set up the environment variables for the build environment
1906 There is only a weak standard, the variables used by gradle, so also set
1907 up the most commonly used environment variables for SDK and NDK. Also, if
1908 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
1910 global env, orig_path
1914 orig_path = env['PATH']
1915 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
1916 env[n] = config['sdk_path']
1917 for k, v in config['java_paths'].items():
1918 env['JAVA%s_HOME' % k] = v
1920 missinglocale = True
1921 for k, v in env.items():
1922 if k == 'LANG' and v != 'C':
1923 missinglocale = False
1925 missinglocale = False
1927 env['LANG'] = 'en_US.UTF-8'
1929 if build is not None:
1930 path = build.ndk_path()
1931 paths = orig_path.split(os.pathsep)
1932 if path not in paths:
1933 paths = [path] + paths
1934 env['PATH'] = os.pathsep.join(paths)
1935 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
1936 env[n] = build.ndk_path()
1939 def replace_build_vars(cmd, build):
1940 cmd = cmd.replace('$$COMMIT$$', build.commit)
1941 cmd = cmd.replace('$$VERSION$$', build.versionName)
1942 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
1946 def replace_config_vars(cmd, build):
1947 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1948 cmd = cmd.replace('$$NDK$$', build.ndk_path())
1949 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1950 cmd = cmd.replace('$$QT$$', config['qt_sdk_path'] or '')
1951 if build is not None:
1952 cmd = replace_build_vars(cmd, build)
1956 def place_srclib(root_dir, number, libpath):
1959 relpath = os.path.relpath(libpath, root_dir)
1960 proppath = os.path.join(root_dir, 'project.properties')
1963 if os.path.isfile(proppath):
1964 with open(proppath, "r", encoding='iso-8859-1') as o:
1965 lines = o.readlines()
1967 with open(proppath, "w", encoding='iso-8859-1') as o:
1970 if line.startswith('android.library.reference.%d=' % number):
1971 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1976 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1979 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
1982 def metadata_get_sigdir(appid, vercode=None):
1983 """Get signature directory for app"""
1985 return os.path.join('metadata', appid, 'signatures', vercode)
1987 return os.path.join('metadata', appid, 'signatures')
1990 def apk_extract_signatures(apkpath, outdir, manifest=True):
1991 """Extracts a signature files from APK and puts them into target directory.
1993 :param apkpath: location of the apk
1994 :param outdir: folder where the extracted signature files will be stored
1995 :param manifest: (optionally) disable extracting manifest file
1997 with ZipFile(apkpath, 'r') as in_apk:
1998 for f in in_apk.infolist():
1999 if apk_sigfile.match(f.filename) or \
2000 (manifest and f.filename == 'META-INF/MANIFEST.MF'):
2001 newpath = os.path.join(outdir, os.path.basename(f.filename))
2002 with open(newpath, 'wb') as out_file:
2003 out_file.write(in_apk.read(f.filename))
2006 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
2007 """Verify that two apks are the same
2009 One of the inputs is signed, the other is unsigned. The signature metadata
2010 is transferred from the signed to the unsigned apk, and then jarsigner is
2011 used to verify that the signature from the signed apk is also varlid for
2012 the unsigned one. If the APK given as unsigned actually does have a
2013 signature, it will be stripped out and ignored.
2015 There are two SHA1 git commit IDs that fdroidserver includes in the builds
2016 it makes: fdroidserverid and buildserverid. Originally, these were inserted
2017 into AndroidManifest.xml, but that makes the build not reproducible. So
2018 instead they are included as separate files in the APK's META-INF/ folder.
2019 If those files exist in the signed APK, they will be part of the signature
2020 and need to also be included in the unsigned APK for it to validate.
2022 :param signed_apk: Path to a signed apk file
2023 :param unsigned_apk: Path to an unsigned apk file expected to match it
2024 :param tmp_dir: Path to directory for temporary files
2025 :returns: None if the verification is successful, otherwise a string
2026 describing what went wrong.
2029 signed = ZipFile(signed_apk, 'r')
2030 meta_inf_files = ['META-INF/MANIFEST.MF']
2031 for f in signed.namelist():
2032 if apk_sigfile.match(f) \
2033 or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
2034 meta_inf_files.append(f)
2035 if len(meta_inf_files) < 3:
2036 return "Signature files missing from {0}".format(signed_apk)
2038 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2039 unsigned = ZipFile(unsigned_apk, 'r')
2040 # only read the signature from the signed APK, everything else from unsigned
2041 with ZipFile(tmp_apk, 'w') as tmp:
2042 for filename in meta_inf_files:
2043 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2044 for info in unsigned.infolist():
2045 if info.filename in meta_inf_files:
2046 logging.warning('Ignoring ' + info.filename + ' from ' + unsigned_apk)
2048 if info.filename in tmp.namelist():
2049 return "duplicate filename found: " + info.filename
2050 tmp.writestr(info, unsigned.read(info.filename))
2054 verified = verify_apk_signature(tmp_apk)
2057 logging.info("...NOT verified - {0}".format(tmp_apk))
2058 return compare_apks(signed_apk, tmp_apk, tmp_dir,
2059 os.path.dirname(unsigned_apk))
2061 logging.info("...successfully verified")
2065 def verify_apk_signature(apk, jar=False):
2066 """verify the signature on an APK
2068 Try to use apksigner whenever possible since jarsigner is very
2069 shitty: unsigned APKs pass as "verified"! So this has to turn on
2070 -strict then check for result 4.
2072 You can set :param: jar to True if you want to use this method
2073 to verify jar signatures.
2075 if set_command_in_config('apksigner'):
2076 args = [config['apksigner'], 'verify']
2078 args += ['--min-sdk-version=1']
2079 return subprocess.call(args + [apk]) == 0
2081 logging.warning("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner")
2082 return subprocess.call([config['jarsigner'], '-strict', '-verify', apk]) == 4
2085 def verify_old_apk_signature(apk):
2086 """verify the signature on an archived APK, supporting deprecated algorithms
2088 F-Droid aims to keep every single binary that it ever published. Therefore,
2089 it needs to be able to verify APK signatures that include deprecated/removed
2090 algorithms. For example, jarsigner treats an MD5 signature as unsigned.
2092 jarsigner passes unsigned APKs as "verified"! So this has to turn
2093 on -strict then check for result 4.
2097 _java_security = os.path.join(os.getcwd(), '.java.security')
2098 with open(_java_security, 'w') as fp:
2099 fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
2101 return subprocess.call([config['jarsigner'], '-J-Djava.security.properties=' + _java_security,
2102 '-strict', '-verify', apk]) == 4
2105 apk_badchars = re.compile('''[/ :;'"]''')
2108 def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
2111 Returns None if the apk content is the same (apart from the signing key),
2112 otherwise a string describing what's different, or what went wrong when
2113 trying to do the comparison.
2119 absapk1 = os.path.abspath(apk1)
2120 absapk2 = os.path.abspath(apk2)
2122 if set_command_in_config('diffoscope'):
2123 logfilename = os.path.join(log_dir, os.path.basename(absapk1))
2124 htmlfile = logfilename + '.diffoscope.html'
2125 textfile = logfilename + '.diffoscope.txt'
2126 if subprocess.call([config['diffoscope'],
2127 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2128 '--html', htmlfile, '--text', textfile,
2129 absapk1, absapk2]) != 0:
2130 return("Failed to unpack " + apk1)
2132 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2133 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2134 for d in [apk1dir, apk2dir]:
2135 if os.path.exists(d):
2138 os.mkdir(os.path.join(d, 'jar-xf'))
2140 if subprocess.call(['jar', 'xf',
2141 os.path.abspath(apk1)],
2142 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2143 return("Failed to unpack " + apk1)
2144 if subprocess.call(['jar', 'xf',
2145 os.path.abspath(apk2)],
2146 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2147 return("Failed to unpack " + apk2)
2149 if set_command_in_config('apktool'):
2150 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2152 return("Failed to unpack " + apk1)
2153 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2155 return("Failed to unpack " + apk2)
2157 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2158 lines = p.output.splitlines()
2159 if len(lines) != 1 or 'META-INF' not in lines[0]:
2160 if set_command_in_config('meld'):
2161 p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
2162 return("Unexpected diff output - " + p.output)
2164 # since everything verifies, delete the comparison to keep cruft down
2165 shutil.rmtree(apk1dir)
2166 shutil.rmtree(apk2dir)
2168 # If we get here, it seems like they're the same!
2172 def set_command_in_config(command):
2173 '''Try to find specified command in the path, if it hasn't been
2174 manually set in config.py. If found, it is added to the config
2175 dict. The return value says whether the command is available.
2178 if command in config:
2181 tmp = find_command(command)
2183 config[command] = tmp
2188 def find_command(command):
2189 '''find the full path of a command, or None if it can't be found in the PATH'''
2192 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2194 fpath, fname = os.path.split(command)
2199 for path in os.environ["PATH"].split(os.pathsep):
2200 path = path.strip('"')
2201 exe_file = os.path.join(path, command)
2202 if is_exe(exe_file):
2209 '''generate a random password for when generating keys'''
2210 h = hashlib.sha256()
2211 h.update(os.urandom(16)) # salt
2212 h.update(socket.getfqdn().encode('utf-8'))
2213 passwd = base64.b64encode(h.digest()).strip()
2214 return passwd.decode('utf-8')
2217 def genkeystore(localconfig):
2219 Generate a new key with password provided in :param localconfig and add it to new keystore
2220 :return: hexed public key, public key fingerprint
2222 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2223 keystoredir = os.path.dirname(localconfig['keystore'])
2224 if keystoredir is None or keystoredir == '':
2225 keystoredir = os.path.join(os.getcwd(), keystoredir)
2226 if not os.path.exists(keystoredir):
2227 os.makedirs(keystoredir, mode=0o700)
2230 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
2231 'FDROID_KEY_PASS': localconfig['keypass'],
2233 p = FDroidPopen([config['keytool'], '-genkey',
2234 '-keystore', localconfig['keystore'],
2235 '-alias', localconfig['repo_keyalias'],
2236 '-keyalg', 'RSA', '-keysize', '4096',
2237 '-sigalg', 'SHA256withRSA',
2238 '-validity', '10000',
2239 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2240 '-keypass:env', 'FDROID_KEY_PASS',
2241 '-dname', localconfig['keydname']], envs=env_vars)
2242 if p.returncode != 0:
2243 raise BuildException("Failed to generate key", p.output)
2244 os.chmod(localconfig['keystore'], 0o0600)
2245 if not options.quiet:
2246 # now show the lovely key that was just generated
2247 p = FDroidPopen([config['keytool'], '-list', '-v',
2248 '-keystore', localconfig['keystore'],
2249 '-alias', localconfig['repo_keyalias'],
2250 '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
2251 logging.info(p.output.strip() + '\n\n')
2252 # get the public key
2253 p = FDroidPopenBytes([config['keytool'], '-exportcert',
2254 '-keystore', localconfig['keystore'],
2255 '-alias', localconfig['repo_keyalias'],
2256 '-storepass:env', 'FDROID_KEY_STORE_PASS']
2257 + config['smartcardoptions'],
2258 envs=env_vars, output=False, stderr_to_stdout=False)
2259 if p.returncode != 0 or len(p.output) < 20:
2260 raise BuildException("Failed to get public key", p.output)
2262 fingerprint = get_cert_fingerprint(pubkey)
2263 return hexlify(pubkey), fingerprint
2266 def get_cert_fingerprint(pubkey):
2268 Generate a certificate fingerprint the same way keytool does it
2269 (but with slightly different formatting)
2271 digest = hashlib.sha256(pubkey).digest()
2272 ret = [' '.join("%02X" % b for b in bytearray(digest))]
2273 return " ".join(ret)
2276 def get_certificate(certificate_file):
2278 Extracts a certificate from the given file.
2279 :param certificate_file: file bytes (as string) representing the certificate
2280 :return: A binary representation of the certificate's public key, or None in case of error
2282 content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
2283 if content.getComponentByName('contentType') != rfc2315.signedData:
2285 content = decoder.decode(content.getComponentByName('content'),
2286 asn1Spec=rfc2315.SignedData())[0]
2288 certificates = content.getComponentByName('certificates')
2289 cert = certificates[0].getComponentByName('certificate')
2291 logging.error("Certificates not found.")
2293 return encoder.encode(cert)
2296 def write_to_config(thisconfig, key, value=None, config_file=None):
2297 '''write a key/value to the local config.py
2299 NOTE: only supports writing string variables.
2301 :param thisconfig: config dictionary
2302 :param key: variable name in config.py to be overwritten/added
2303 :param value: optional value to be written, instead of fetched
2304 from 'thisconfig' dictionary.
2307 origkey = key + '_orig'
2308 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2309 cfg = config_file if config_file else 'config.py'
2312 with open(cfg, 'r', encoding="utf-8") as f:
2313 lines = f.readlines()
2315 # make sure the file ends with a carraige return
2317 if not lines[-1].endswith('\n'):
2320 # regex for finding and replacing python string variable
2321 # definitions/initializations
2322 pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
2323 repl = key + ' = "' + value + '"'
2324 pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
2325 repl2 = key + " = '" + value + "'"
2327 # If we replaced this line once, we make sure won't be a
2328 # second instance of this line for this key in the document.
2331 with open(cfg, 'w', encoding="utf-8") as f:
2333 if pattern.match(line) or pattern2.match(line):
2335 line = pattern.sub(repl, line)
2336 line = pattern2.sub(repl2, line)
2347 def parse_xml(path):
2348 return XMLElementTree.parse(path).getroot()
2351 def string_is_integer(string):
2359 def get_per_app_repos():
2360 '''per-app repos are dirs named with the packageName of a single app'''
2362 # Android packageNames are Java packages, they may contain uppercase or
2363 # lowercase letters ('A' through 'Z'), numbers, and underscores
2364 # ('_'). However, individual package name parts may only start with
2365 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2366 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2369 for root, dirs, files in os.walk(os.getcwd()):
2371 print('checking', root, 'for', d)
2372 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2373 # standard parts of an fdroid repo, so never packageNames
2376 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
2382 def is_repo_file(filename):
2383 '''Whether the file in a repo is a build product to be delivered to users'''
2384 if isinstance(filename, str):
2385 filename = filename.encode('utf-8', errors="surrogateescape")
2386 return os.path.isfile(filename) \
2387 and not filename.endswith(b'.asc') \
2388 and not filename.endswith(b'.sig') \
2389 and os.path.basename(filename) not in [
2391 b'index_unsigned.jar',