3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
38 import xml.etree.ElementTree as XMLElementTree
40 from datetime import datetime
41 from distutils.version import LooseVersion
42 from queue import Queue
43 from zipfile import ZipFile
45 import fdroidserver.metadata
46 from .asynchronousfilereader import AsynchronousFileReader
49 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
58 'sdk_path': "$ANDROID_HOME",
63 'r12b': "$ANDROID_NDK",
68 'build_tools': "25.0.2",
69 'force_build_tools': False,
74 'accepted_formats': ['txt', 'yml'],
75 'sync_from_local_copy_dir': False,
76 'per_app_repos': False,
77 'make_current_version_link': True,
78 'current_version_name_source': 'Name',
79 'update_stats': False,
83 'stats_to_carbon': False,
85 'build_server_always': False,
86 'keystore': 'keystore.jks',
87 'smartcardoptions': [],
93 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
94 'repo_name': "My First FDroid Repo Demo",
95 'repo_icon': "fdroid-icon.png",
96 'repo_description': '''
97 This is a repository of apps to be used with FDroid. Applications in this
98 repository are either official binaries built by the original application
99 developers, or are binaries built from source by the admin of f-droid.org
100 using the tools on https://gitlab.com/u/fdroid.
106 def setup_global_opts(parser):
107 parser.add_argument("-v", "--verbose", action="store_true", default=False,
108 help="Spew out even more information than normal")
109 parser.add_argument("-q", "--quiet", action="store_true", default=False,
110 help="Restrict output to warnings and errors")
113 def fill_config_defaults(thisconfig):
114 for k, v in default_config.items():
115 if k not in thisconfig:
118 # Expand paths (~users and $vars)
119 def expand_path(path):
123 path = os.path.expanduser(path)
124 path = os.path.expandvars(path)
129 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
134 thisconfig[k + '_orig'] = v
136 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
137 if thisconfig['java_paths'] is None:
138 thisconfig['java_paths'] = dict()
140 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
141 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
142 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
143 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
144 if os.getenv('JAVA_HOME') is not None:
145 pathlist.append(os.getenv('JAVA_HOME'))
146 if os.getenv('PROGRAMFILES') is not None:
147 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
148 for d in sorted(pathlist):
149 if os.path.islink(d):
151 j = os.path.basename(d)
152 # the last one found will be the canonical one, so order appropriately
154 r'^1\.([6-9])\.0\.jdk$', # OSX
155 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
156 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
157 r'^jdk([6-9])-openjdk$', # Arch
158 r'^java-([6-9])-openjdk$', # Arch
159 r'^java-([6-9])-jdk$', # Arch (oracle)
160 r'^java-1\.([6-9])\.0-.*$', # RedHat
161 r'^java-([6-9])-oracle$', # Debian WebUpd8
162 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
163 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
165 m = re.match(regex, j)
168 for p in [d, os.path.join(d, 'Contents', 'Home')]:
169 if os.path.exists(os.path.join(p, 'bin', 'javac')):
170 thisconfig['java_paths'][m.group(1)] = p
172 for java_version in ('7', '8', '9'):
173 if java_version not in thisconfig['java_paths']:
175 java_home = thisconfig['java_paths'][java_version]
176 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
177 if os.path.exists(jarsigner):
178 thisconfig['jarsigner'] = jarsigner
179 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
180 break # Java7 is preferred, so quit if found
182 for k in ['ndk_paths', 'java_paths']:
188 thisconfig[k][k2] = exp
189 thisconfig[k][k2 + '_orig'] = v
192 def regsub_file(pattern, repl, path):
193 with open(path, 'rb') as f:
195 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
196 with open(path, 'wb') as f:
200 def read_config(opts, config_file='config.py'):
201 """Read the repository config
203 The config is read from config_file, which is in the current
204 directory when any of the repo management commands are used. If
205 there is a local metadata file in the git repo, then config.py is
206 not required, just use defaults.
209 global config, options
211 if config is not None:
218 if os.path.isfile(config_file):
219 logging.debug("Reading %s" % config_file)
220 with io.open(config_file, "rb") as f:
221 code = compile(f.read(), config_file, 'exec')
222 exec(code, None, config)
223 elif len(get_local_metadata_files()) == 0:
224 logging.critical("Missing config file - is this a repo directory?")
227 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
229 if not type(config[k]) in (str, list, tuple):
230 logging.warn('"' + k + '" will be in random order!'
231 + ' Use () or [] brackets if order is important!')
233 # smartcardoptions must be a list since its command line args for Popen
234 if 'smartcardoptions' in config:
235 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
236 elif 'keystore' in config and config['keystore'] == 'NONE':
237 # keystore='NONE' means use smartcard, these are required defaults
238 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
239 'SunPKCS11-OpenSC', '-providerClass',
240 'sun.security.pkcs11.SunPKCS11',
241 '-providerArg', 'opensc-fdroid.cfg']
243 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
244 st = os.stat(config_file)
245 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
246 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
248 fill_config_defaults(config)
250 for k in ["keystorepass", "keypass"]:
252 write_password_file(k)
254 for k in ["repo_description", "archive_description"]:
256 config[k] = clean_description(config[k])
258 if 'serverwebroot' in config:
259 if isinstance(config['serverwebroot'], str):
260 roots = [config['serverwebroot']]
261 elif all(isinstance(item, str) for item in config['serverwebroot']):
262 roots = config['serverwebroot']
264 raise TypeError('only accepts strings, lists, and tuples')
266 for rootstr in roots:
267 # since this is used with rsync, where trailing slashes have
268 # meaning, ensure there is always a trailing slash
269 if rootstr[-1] != '/':
271 rootlist.append(rootstr.replace('//', '/'))
272 config['serverwebroot'] = rootlist
274 if 'servergitmirrors' in config:
275 if isinstance(config['servergitmirrors'], str):
276 roots = [config['servergitmirrors']]
277 elif all(isinstance(item, str) for item in config['servergitmirrors']):
278 roots = config['servergitmirrors']
280 raise TypeError('only accepts strings, lists, and tuples')
281 config['servergitmirrors'] = roots
286 def find_sdk_tools_cmd(cmd):
287 '''find a working path to a tool from the Android SDK'''
290 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
291 # try to find a working path to this command, in all the recent possible paths
292 if 'build_tools' in config:
293 build_tools = os.path.join(config['sdk_path'], 'build-tools')
294 # if 'build_tools' was manually set and exists, check only that one
295 configed_build_tools = os.path.join(build_tools, config['build_tools'])
296 if os.path.exists(configed_build_tools):
297 tooldirs.append(configed_build_tools)
299 # no configed version, so hunt known paths for it
300 for f in sorted(os.listdir(build_tools), reverse=True):
301 if os.path.isdir(os.path.join(build_tools, f)):
302 tooldirs.append(os.path.join(build_tools, f))
303 tooldirs.append(build_tools)
304 sdk_tools = os.path.join(config['sdk_path'], 'tools')
305 if os.path.exists(sdk_tools):
306 tooldirs.append(sdk_tools)
307 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
308 if os.path.exists(sdk_platform_tools):
309 tooldirs.append(sdk_platform_tools)
310 tooldirs.append('/usr/bin')
312 path = os.path.join(d, cmd)
313 if os.path.isfile(path):
315 test_aapt_version(path)
317 # did not find the command, exit with error message
318 ensure_build_tools_exists(config)
321 def test_aapt_version(aapt):
322 '''Check whether the version of aapt is new enough'''
323 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
324 if output is None or output == '':
325 logging.error(aapt + ' failed to execute!')
327 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
332 # the Debian package has the version string like "v0.2-23.0.2"
333 if '.' not in bugfix and LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.2166767'):
334 logging.warning(aapt + ' is too old, fdroid requires build-tools-23.0.0 or newer!')
336 logging.warning('Unknown version of aapt, might cause problems: ' + output)
339 def test_sdk_exists(thisconfig):
340 if 'sdk_path' not in thisconfig:
341 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
342 test_aapt_version(thisconfig['aapt'])
345 logging.error("'sdk_path' not set in config.py!")
347 if thisconfig['sdk_path'] == default_config['sdk_path']:
348 logging.error('No Android SDK found!')
349 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
350 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
352 if not os.path.exists(thisconfig['sdk_path']):
353 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
355 if not os.path.isdir(thisconfig['sdk_path']):
356 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
358 for d in ['build-tools', 'platform-tools', 'tools']:
359 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
360 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
361 thisconfig['sdk_path'], d))
366 def ensure_build_tools_exists(thisconfig):
367 if not test_sdk_exists(thisconfig):
369 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
370 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
371 if not os.path.isdir(versioned_build_tools):
372 logging.critical('Android Build Tools path "'
373 + versioned_build_tools + '" does not exist!')
377 def write_password_file(pwtype, password=None):
379 writes out passwords to a protected file instead of passing passwords as
380 command line argments
382 filename = '.fdroid.' + pwtype + '.txt'
383 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
385 os.write(fd, config[pwtype].encode('utf-8'))
387 os.write(fd, password.encode('utf-8'))
389 config[pwtype + 'file'] = filename
394 sign a JAR file with Java's jarsigner.
396 This does use old hashing algorithms, i.e. SHA1, but that's not
397 broken yet for file verification. This could be set to SHA256,
398 but then Android < 4.3 would not be able to verify it.
399 https://code.google.com/p/android/issues/detail?id=38321
401 args = [config['jarsigner'], '-keystore', config['keystore'],
402 '-storepass:file', config['keystorepassfile'],
403 '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
404 jar, config['repo_keyalias']]
405 if config['keystore'] == 'NONE':
406 args += config['smartcardoptions']
407 else: # smardcards never use -keypass
408 args += ['-keypass:file', config['keypassfile']]
409 p = FDroidPopen(args)
410 if p.returncode != 0:
411 logging.critical("Failed to sign %s!" % jar)
415 def sign_index_v1(repodir, json_name):
417 sign index-v1.json to make index-v1.jar
419 This is a bit different than index.jar: instead of their being index.xml
420 and index_unsigned.jar, the presense of index-v1.json means that there is
421 unsigned data. That file is then stuck into a jar and signed by the
422 signing process. index-v1.json is never published to the repo. It is
423 included in the binary transparency log, if that is enabled.
425 name, ext = get_extension(json_name)
426 index_file = os.path.join(repodir, json_name)
427 jar_file = os.path.join(repodir, name + '.jar')
428 with zipfile.ZipFile(jar_file, 'w', zipfile.ZIP_DEFLATED) as jar:
429 jar.write(index_file, json_name)
433 def get_local_metadata_files():
434 '''get any metadata files local to an app's source repo
436 This tries to ignore anything that does not count as app metdata,
437 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
440 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
443 def read_pkg_args(args, allow_vercodes=False):
445 Given the arguments in the form of multiple appid:[vc] strings, this returns
446 a dictionary with the set of vercodes specified for each package.
454 if allow_vercodes and ':' in p:
455 package, vercode = p.split(':')
457 package, vercode = p, None
458 if package not in vercodes:
459 vercodes[package] = [vercode] if vercode else []
461 elif vercode and vercode not in vercodes[package]:
462 vercodes[package] += [vercode] if vercode else []
467 def read_app_args(args, allapps, allow_vercodes=False):
469 On top of what read_pkg_args does, this returns the whole app metadata, but
470 limiting the builds list to the builds matching the vercodes specified.
473 vercodes = read_pkg_args(args, allow_vercodes)
479 for appid, app in allapps.items():
480 if appid in vercodes:
483 if len(apps) != len(vercodes):
486 logging.critical("No such package: %s" % p)
487 raise FDroidException("Found invalid app ids in arguments")
489 raise FDroidException("No packages specified")
492 for appid, app in apps.items():
496 app.builds = [b for b in app.builds if b.versionCode in vc]
497 if len(app.builds) != len(vercodes[appid]):
499 allvcs = [b.versionCode for b in app.builds]
500 for v in vercodes[appid]:
502 logging.critical("No such vercode %s for app %s" % (v, appid))
505 raise FDroidException("Found invalid vercodes for some apps")
510 def get_extension(filename):
511 base, ext = os.path.splitext(filename)
514 return base, ext.lower()[1:]
517 def has_extension(filename, ext):
518 _, f_ext = get_extension(filename)
522 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
525 def clean_description(description):
526 'Remove unneeded newlines and spaces from a block of description text'
528 # this is split up by paragraph to make removing the newlines easier
529 for paragraph in re.split(r'\n\n', description):
530 paragraph = re.sub('\r', '', paragraph)
531 paragraph = re.sub('\n', ' ', paragraph)
532 paragraph = re.sub(' {2,}', ' ', paragraph)
533 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
534 returnstring += paragraph + '\n\n'
535 return returnstring.rstrip('\n')
538 def publishednameinfo(filename):
539 filename = os.path.basename(filename)
540 m = publish_name_regex.match(filename)
542 result = (m.group(1), m.group(2))
543 except AttributeError:
544 raise FDroidException("Invalid name for published file: %s" % filename)
548 def get_release_filename(app, build):
550 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
552 return "%s_%s.apk" % (app.id, build.versionCode)
555 def getsrcname(app, build):
556 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
568 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
571 def get_build_dir(app):
572 '''get the dir that this app will be built in'''
574 if app.RepoType == 'srclib':
575 return os.path.join('build', 'srclib', app.Repo)
577 return os.path.join('build', app.id)
581 '''checkout code from VCS and return instance of vcs and the build dir'''
582 build_dir = get_build_dir(app)
584 # Set up vcs interface and make sure we have the latest code...
585 logging.debug("Getting {0} vcs interface for {1}"
586 .format(app.RepoType, app.Repo))
587 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
591 vcs = getvcs(app.RepoType, remote, build_dir)
593 return vcs, build_dir
596 def getvcs(vcstype, remote, local):
598 return vcs_git(remote, local)
599 if vcstype == 'git-svn':
600 return vcs_gitsvn(remote, local)
602 return vcs_hg(remote, local)
604 return vcs_bzr(remote, local)
605 if vcstype == 'srclib':
606 if local != os.path.join('build', 'srclib', remote):
607 raise VCSException("Error: srclib paths are hard-coded!")
608 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
610 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
611 raise VCSException("Invalid vcs type " + vcstype)
614 def getsrclibvcs(name):
615 if name not in fdroidserver.metadata.srclibs:
616 raise VCSException("Missing srclib " + name)
617 return fdroidserver.metadata.srclibs[name]['Repo Type']
622 def __init__(self, remote, local):
624 # svn, git-svn and bzr may require auth
626 if self.repotype() in ('git-svn', 'bzr'):
628 if self.repotype == 'git-svn':
629 raise VCSException("Authentication is not supported for git-svn")
630 self.username, remote = remote.split('@')
631 if ':' not in self.username:
632 raise VCSException("Password required with username")
633 self.username, self.password = self.username.split(':')
637 self.clone_failed = False
638 self.refreshed = False
644 # Take the local repository to a clean version of the given revision, which
645 # is specificed in the VCS's native format. Beforehand, the repository can
646 # be dirty, or even non-existent. If the repository does already exist
647 # locally, it will be updated from the origin, but only once in the
648 # lifetime of the vcs object.
649 # None is acceptable for 'rev' if you know you are cloning a clean copy of
650 # the repo - otherwise it must specify a valid revision.
651 def gotorevision(self, rev, refresh=True):
653 if self.clone_failed:
654 raise VCSException("Downloading the repository already failed once, not trying again.")
656 # The .fdroidvcs-id file for a repo tells us what VCS type
657 # and remote that directory was created from, allowing us to drop it
658 # automatically if either of those things changes.
659 fdpath = os.path.join(self.local, '..',
660 '.fdroidvcs-' + os.path.basename(self.local))
661 fdpath = os.path.normpath(fdpath)
662 cdata = self.repotype() + ' ' + self.remote
665 if os.path.exists(self.local):
666 if os.path.exists(fdpath):
667 with open(fdpath, 'r') as f:
668 fsdata = f.read().strip()
673 logging.info("Repository details for %s changed - deleting" % (
677 logging.info("Repository details for %s missing - deleting" % (
680 shutil.rmtree(self.local)
684 self.refreshed = True
687 self.gotorevisionx(rev)
688 except FDroidException as e:
691 # If necessary, write the .fdroidvcs file.
692 if writeback and not self.clone_failed:
693 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
694 with open(fdpath, 'w+') as f:
700 # Derived classes need to implement this. It's called once basic checking
701 # has been performend.
702 def gotorevisionx(self, rev):
703 raise VCSException("This VCS type doesn't define gotorevisionx")
705 # Initialise and update submodules
706 def initsubmodules(self):
707 raise VCSException('Submodules not supported for this vcs type')
709 # Get a list of all known tags
711 if not self._gettags:
712 raise VCSException('gettags not supported for this vcs type')
714 for tag in self._gettags():
715 if re.match('[-A-Za-z0-9_. /]+$', tag):
719 # Get a list of all the known tags, sorted from newest to oldest
720 def latesttags(self):
721 raise VCSException('latesttags not supported for this vcs type')
723 # Get current commit reference (hash, revision, etc)
725 raise VCSException('getref not supported for this vcs type')
727 # Returns the srclib (name, path) used in setting up the current
738 # If the local directory exists, but is somehow not a git repository, git
739 # will traverse up the directory tree until it finds one that is (i.e.
740 # fdroidserver) and then we'll proceed to destroy it! This is called as
743 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
744 result = p.output.rstrip()
745 if not result.endswith(self.local):
746 raise VCSException('Repository mismatch')
748 def gotorevisionx(self, rev):
749 if not os.path.exists(self.local):
751 p = FDroidPopen(['git', 'clone', self.remote, self.local])
752 if p.returncode != 0:
753 self.clone_failed = True
754 raise VCSException("Git clone failed", p.output)
758 # Discard any working tree changes
759 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
760 'git', 'reset', '--hard'], cwd=self.local, output=False)
761 if p.returncode != 0:
762 raise VCSException("Git reset failed", p.output)
763 # Remove untracked files now, in case they're tracked in the target
764 # revision (it happens!)
765 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
766 'git', 'clean', '-dffx'], cwd=self.local, output=False)
767 if p.returncode != 0:
768 raise VCSException("Git clean failed", p.output)
769 if not self.refreshed:
770 # Get latest commits and tags from remote
771 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
772 if p.returncode != 0:
773 raise VCSException("Git fetch failed", p.output)
774 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
775 if p.returncode != 0:
776 raise VCSException("Git fetch failed", p.output)
777 # Recreate origin/HEAD as git clone would do it, in case it disappeared
778 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
779 if p.returncode != 0:
780 lines = p.output.splitlines()
781 if 'Multiple remote HEAD branches' not in lines[0]:
782 raise VCSException("Git remote set-head failed", p.output)
783 branch = lines[1].split(' ')[-1]
784 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
785 if p2.returncode != 0:
786 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
787 self.refreshed = True
788 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
789 # a github repo. Most of the time this is the same as origin/master.
790 rev = rev or 'origin/HEAD'
791 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
792 if p.returncode != 0:
793 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
794 # Get rid of any uncontrolled files left behind
795 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
796 if p.returncode != 0:
797 raise VCSException("Git clean failed", p.output)
799 def initsubmodules(self):
801 submfile = os.path.join(self.local, '.gitmodules')
802 if not os.path.isfile(submfile):
803 raise VCSException("No git submodules available")
805 # fix submodules not accessible without an account and public key auth
806 with open(submfile, 'r') as f:
807 lines = f.readlines()
808 with open(submfile, 'w') as f:
810 if 'git@github.com' in line:
811 line = line.replace('git@github.com:', 'https://github.com/')
812 if 'git@gitlab.com' in line:
813 line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
816 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
817 if p.returncode != 0:
818 raise VCSException("Git submodule sync failed", p.output)
819 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
820 if p.returncode != 0:
821 raise VCSException("Git submodule update failed", p.output)
825 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
826 return p.output.splitlines()
828 tag_format = re.compile(r'tag: ([^),]*)')
830 def latesttags(self):
832 p = FDroidPopen(['git', 'log', '--tags',
833 '--simplify-by-decoration', '--pretty=format:%d'],
834 cwd=self.local, output=False)
836 for line in p.output.splitlines():
837 for tag in self.tag_format.findall(line):
842 class vcs_gitsvn(vcs):
847 # If the local directory exists, but is somehow not a git repository, git
848 # will traverse up the directory tree until it finds one that is (i.e.
849 # fdroidserver) and then we'll proceed to destory it! This is called as
852 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
853 result = p.output.rstrip()
854 if not result.endswith(self.local):
855 raise VCSException('Repository mismatch')
857 def gotorevisionx(self, rev):
858 if not os.path.exists(self.local):
860 gitsvn_args = ['git', 'svn', 'clone']
861 if ';' in self.remote:
862 remote_split = self.remote.split(';')
863 for i in remote_split[1:]:
864 if i.startswith('trunk='):
865 gitsvn_args.extend(['-T', i[6:]])
866 elif i.startswith('tags='):
867 gitsvn_args.extend(['-t', i[5:]])
868 elif i.startswith('branches='):
869 gitsvn_args.extend(['-b', i[9:]])
870 gitsvn_args.extend([remote_split[0], self.local])
871 p = FDroidPopen(gitsvn_args, output=False)
872 if p.returncode != 0:
873 self.clone_failed = True
874 raise VCSException("Git svn clone failed", p.output)
876 gitsvn_args.extend([self.remote, self.local])
877 p = FDroidPopen(gitsvn_args, output=False)
878 if p.returncode != 0:
879 self.clone_failed = True
880 raise VCSException("Git svn clone failed", p.output)
884 # Discard any working tree changes
885 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
886 if p.returncode != 0:
887 raise VCSException("Git reset failed", p.output)
888 # Remove untracked files now, in case they're tracked in the target
889 # revision (it happens!)
890 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
891 if p.returncode != 0:
892 raise VCSException("Git clean failed", p.output)
893 if not self.refreshed:
894 # Get new commits, branches and tags from repo
895 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
896 if p.returncode != 0:
897 raise VCSException("Git svn fetch failed")
898 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
899 if p.returncode != 0:
900 raise VCSException("Git svn rebase failed", p.output)
901 self.refreshed = True
903 rev = rev or 'master'
905 nospaces_rev = rev.replace(' ', '%20')
906 # Try finding a svn tag
907 for treeish in ['origin/', '']:
908 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
909 if p.returncode == 0:
911 if p.returncode != 0:
912 # No tag found, normal svn rev translation
913 # Translate svn rev into git format
914 rev_split = rev.split('/')
917 for treeish in ['origin/', '']:
918 if len(rev_split) > 1:
919 treeish += rev_split[0]
920 svn_rev = rev_split[1]
923 # if no branch is specified, then assume trunk (i.e. 'master' branch):
927 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
929 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
930 git_rev = p.output.rstrip()
932 if p.returncode == 0 and git_rev:
935 if p.returncode != 0 or not git_rev:
936 # Try a plain git checkout as a last resort
937 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
938 if p.returncode != 0:
939 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
941 # Check out the git rev equivalent to the svn rev
942 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
943 if p.returncode != 0:
944 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
946 # Get rid of any uncontrolled files left behind
947 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
948 if p.returncode != 0:
949 raise VCSException("Git clean failed", p.output)
953 for treeish in ['origin/', '']:
954 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
960 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
961 if p.returncode != 0:
963 return p.output.strip()
971 def gotorevisionx(self, rev):
972 if not os.path.exists(self.local):
973 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
974 if p.returncode != 0:
975 self.clone_failed = True
976 raise VCSException("Hg clone failed", p.output)
978 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
979 if p.returncode != 0:
980 raise VCSException("Hg status failed", p.output)
981 for line in p.output.splitlines():
982 if not line.startswith('? '):
983 raise VCSException("Unexpected output from hg status -uS: " + line)
984 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
985 if not self.refreshed:
986 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
987 if p.returncode != 0:
988 raise VCSException("Hg pull failed", p.output)
989 self.refreshed = True
991 rev = rev or 'default'
994 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
995 if p.returncode != 0:
996 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
997 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
998 # Also delete untracked files, we have to enable purge extension for that:
999 if "'purge' is provided by the following extension" in p.output:
1000 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
1001 myfile.write("\n[extensions]\nhgext.purge=\n")
1002 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1003 if p.returncode != 0:
1004 raise VCSException("HG purge failed", p.output)
1005 elif p.returncode != 0:
1006 raise VCSException("HG purge failed", p.output)
1009 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
1010 return p.output.splitlines()[1:]
1018 def gotorevisionx(self, rev):
1019 if not os.path.exists(self.local):
1020 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
1021 if p.returncode != 0:
1022 self.clone_failed = True
1023 raise VCSException("Bzr branch failed", p.output)
1025 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
1026 if p.returncode != 0:
1027 raise VCSException("Bzr revert failed", p.output)
1028 if not self.refreshed:
1029 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
1030 if p.returncode != 0:
1031 raise VCSException("Bzr update failed", p.output)
1032 self.refreshed = True
1034 revargs = list(['-r', rev] if rev else [])
1035 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
1036 if p.returncode != 0:
1037 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1040 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
1041 return [tag.split(' ')[0].strip() for tag in
1042 p.output.splitlines()]
1045 def unescape_string(string):
1048 if string[0] == '"' and string[-1] == '"':
1051 return string.replace("\\'", "'")
1054 def retrieve_string(app_dir, string, xmlfiles=None):
1056 if not string.startswith('@string/'):
1057 return unescape_string(string)
1059 if xmlfiles is None:
1062 os.path.join(app_dir, 'res'),
1063 os.path.join(app_dir, 'src', 'main', 'res'),
1065 for r, d, f in os.walk(res_dir):
1066 if os.path.basename(r) == 'values':
1067 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
1069 name = string[len('@string/'):]
1071 def element_content(element):
1072 if element.text is None:
1074 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1075 return s.decode('utf-8').strip()
1077 for path in xmlfiles:
1078 if not os.path.isfile(path):
1080 xml = parse_xml(path)
1081 element = xml.find('string[@name="' + name + '"]')
1082 if element is not None:
1083 content = element_content(element)
1084 return retrieve_string(app_dir, content, xmlfiles)
1089 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1090 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1093 def manifest_paths(app_dir, flavours):
1094 '''Return list of existing files that will be used to find the highest vercode'''
1096 possible_manifests = \
1097 [os.path.join(app_dir, 'AndroidManifest.xml'),
1098 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1099 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1100 os.path.join(app_dir, 'build.gradle')]
1102 for flavour in flavours:
1103 if flavour == 'yes':
1105 possible_manifests.append(
1106 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1108 return [path for path in possible_manifests if os.path.isfile(path)]
1111 def fetch_real_name(app_dir, flavours):
1112 '''Retrieve the package name. Returns the name, or None if not found.'''
1113 for path in manifest_paths(app_dir, flavours):
1114 if not has_extension(path, 'xml') or not os.path.isfile(path):
1116 logging.debug("fetch_real_name: Checking manifest at " + path)
1117 xml = parse_xml(path)
1118 app = xml.find('application')
1121 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1123 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1124 result = retrieve_string_singleline(app_dir, label)
1126 result = result.strip()
1131 def get_library_references(root_dir):
1133 proppath = os.path.join(root_dir, 'project.properties')
1134 if not os.path.isfile(proppath):
1136 with open(proppath, 'r', encoding='iso-8859-1') as f:
1138 if not line.startswith('android.library.reference.'):
1140 path = line.split('=')[1].strip()
1141 relpath = os.path.join(root_dir, path)
1142 if not os.path.isdir(relpath):
1144 logging.debug("Found subproject at %s" % path)
1145 libraries.append(path)
1149 def ant_subprojects(root_dir):
1150 subprojects = get_library_references(root_dir)
1151 for subpath in subprojects:
1152 subrelpath = os.path.join(root_dir, subpath)
1153 for p in get_library_references(subrelpath):
1154 relp = os.path.normpath(os.path.join(subpath, p))
1155 if relp not in subprojects:
1156 subprojects.insert(0, relp)
1160 def remove_debuggable_flags(root_dir):
1161 # Remove forced debuggable flags
1162 logging.debug("Removing debuggable flags from %s" % root_dir)
1163 for root, dirs, files in os.walk(root_dir):
1164 if 'AndroidManifest.xml' in files:
1165 regsub_file(r'android:debuggable="[^"]*"',
1167 os.path.join(root, 'AndroidManifest.xml'))
1170 vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search
1171 vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1172 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1175 def app_matches_packagename(app, package):
1178 appid = app.UpdateCheckName or app.id
1179 if appid is None or appid == "Ignore":
1181 return appid == package
1184 def parse_androidmanifests(paths, app):
1186 Extract some information from the AndroidManifest.xml at the given path.
1187 Returns (version, vercode, package), any or all of which might be None.
1188 All values returned are strings.
1191 ignoreversions = app.UpdateCheckIgnore
1192 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1195 return (None, None, None)
1203 if not os.path.isfile(path):
1206 logging.debug("Parsing manifest at {0}".format(path))
1211 if has_extension(path, 'gradle'):
1212 with open(path, 'r') as f:
1214 if gradle_comment.match(line):
1216 # Grab first occurence of each to avoid running into
1217 # alternative flavours and builds.
1219 matches = psearch_g(line)
1221 s = matches.group(2)
1222 if app_matches_packagename(app, s):
1225 matches = vnsearch_g(line)
1227 version = matches.group(2)
1229 matches = vcsearch_g(line)
1231 vercode = matches.group(1)
1234 xml = parse_xml(path)
1235 if "package" in xml.attrib:
1236 s = xml.attrib["package"]
1237 if app_matches_packagename(app, s):
1239 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1240 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1241 base_dir = os.path.dirname(path)
1242 version = retrieve_string_singleline(base_dir, version)
1243 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1244 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1245 if string_is_integer(a):
1248 logging.warning("Problem with xml at {0}".format(path))
1250 # Remember package name, may be defined separately from version+vercode
1252 package = max_package
1254 logging.debug("..got package={0}, version={1}, vercode={2}"
1255 .format(package, version, vercode))
1257 # Always grab the package name and version name in case they are not
1258 # together with the highest version code
1259 if max_package is None and package is not None:
1260 max_package = package
1261 if max_version is None and version is not None:
1262 max_version = version
1264 if vercode is not None \
1265 and (max_vercode is None or vercode > max_vercode):
1266 if not ignoresearch or not ignoresearch(version):
1267 if version is not None:
1268 max_version = version
1269 if vercode is not None:
1270 max_vercode = vercode
1271 if package is not None:
1272 max_package = package
1274 max_version = "Ignore"
1276 if max_version is None:
1277 max_version = "Unknown"
1279 if max_package and not is_valid_package_name(max_package):
1280 raise FDroidException("Invalid package name {0}".format(max_package))
1282 return (max_version, max_vercode, max_package)
1285 def is_valid_package_name(name):
1286 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1289 class FDroidException(Exception):
1291 def __init__(self, value, detail=None):
1293 self.detail = detail
1295 def shortened_detail(self):
1296 if len(self.detail) < 16000:
1298 return '[...]\n' + self.detail[-16000:]
1300 def get_wikitext(self):
1301 ret = repr(self.value) + "\n"
1304 ret += "<pre>\n" + self.shortened_detail() + "</pre>\n"
1310 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1314 class VCSException(FDroidException):
1318 class BuildException(FDroidException):
1322 # Get the specified source library.
1323 # Returns the path to it. Normally this is the path to be used when referencing
1324 # it, which may be a subdirectory of the actual project. If you want the base
1325 # directory of the project, pass 'basepath=True'.
1326 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1327 raw=False, prepare=True, preponly=False, refresh=True,
1336 name, ref = spec.split('@')
1338 number, name = name.split(':', 1)
1340 name, subdir = name.split('/', 1)
1342 if name not in fdroidserver.metadata.srclibs:
1343 raise VCSException('srclib ' + name + ' not found.')
1345 srclib = fdroidserver.metadata.srclibs[name]
1347 sdir = os.path.join(srclib_dir, name)
1350 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1351 vcs.srclib = (name, number, sdir)
1353 vcs.gotorevision(ref, refresh)
1360 libdir = os.path.join(sdir, subdir)
1361 elif srclib["Subdir"]:
1362 for subdir in srclib["Subdir"]:
1363 libdir_candidate = os.path.join(sdir, subdir)
1364 if os.path.exists(libdir_candidate):
1365 libdir = libdir_candidate
1371 remove_signing_keys(sdir)
1372 remove_debuggable_flags(sdir)
1376 if srclib["Prepare"]:
1377 cmd = replace_config_vars(srclib["Prepare"], build)
1379 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1380 if p.returncode != 0:
1381 raise BuildException("Error running prepare command for srclib %s"
1387 return (name, number, libdir)
1390 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1393 # Prepare the source code for a particular build
1394 # 'vcs' - the appropriate vcs object for the application
1395 # 'app' - the application details from the metadata
1396 # 'build' - the build details from the metadata
1397 # 'build_dir' - the path to the build directory, usually
1399 # 'srclib_dir' - the path to the source libraries directory, usually
1401 # 'extlib_dir' - the path to the external libraries directory, usually
1403 # Returns the (root, srclibpaths) where:
1404 # 'root' is the root directory, which may be the same as 'build_dir' or may
1405 # be a subdirectory of it.
1406 # 'srclibpaths' is information on the srclibs being used
1407 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1409 # Optionally, the actual app source can be in a subdirectory
1411 root_dir = os.path.join(build_dir, build.subdir)
1413 root_dir = build_dir
1415 # Get a working copy of the right revision
1416 logging.info("Getting source for revision " + build.commit)
1417 vcs.gotorevision(build.commit, refresh)
1419 # Initialise submodules if required
1420 if build.submodules:
1421 logging.info("Initialising submodules")
1422 vcs.initsubmodules()
1424 # Check that a subdir (if we're using one) exists. This has to happen
1425 # after the checkout, since it might not exist elsewhere
1426 if not os.path.exists(root_dir):
1427 raise BuildException('Missing subdir ' + root_dir)
1429 # Run an init command if one is required
1431 cmd = replace_config_vars(build.init, build)
1432 logging.info("Running 'init' commands in %s" % root_dir)
1434 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1435 if p.returncode != 0:
1436 raise BuildException("Error running init command for %s:%s" %
1437 (app.id, build.versionName), p.output)
1439 # Apply patches if any
1441 logging.info("Applying patches")
1442 for patch in build.patch:
1443 patch = patch.strip()
1444 logging.info("Applying " + patch)
1445 patch_path = os.path.join('metadata', app.id, patch)
1446 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1447 if p.returncode != 0:
1448 raise BuildException("Failed to apply patch %s" % patch_path)
1450 # Get required source libraries
1453 logging.info("Collecting source libraries")
1454 for lib in build.srclibs:
1455 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1456 refresh=refresh, build=build))
1458 for name, number, libpath in srclibpaths:
1459 place_srclib(root_dir, int(number) if number else None, libpath)
1461 basesrclib = vcs.getsrclib()
1462 # If one was used for the main source, add that too.
1464 srclibpaths.append(basesrclib)
1466 # Update the local.properties file
1467 localprops = [os.path.join(build_dir, 'local.properties')]
1469 parts = build.subdir.split(os.sep)
1472 cur = os.path.join(cur, d)
1473 localprops += [os.path.join(cur, 'local.properties')]
1474 for path in localprops:
1476 if os.path.isfile(path):
1477 logging.info("Updating local.properties file at %s" % path)
1478 with open(path, 'r', encoding='iso-8859-1') as f:
1482 logging.info("Creating local.properties file at %s" % path)
1483 # Fix old-fashioned 'sdk-location' by copying
1484 # from sdk.dir, if necessary
1486 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1487 re.S | re.M).group(1)
1488 props += "sdk-location=%s\n" % sdkloc
1490 props += "sdk.dir=%s\n" % config['sdk_path']
1491 props += "sdk-location=%s\n" % config['sdk_path']
1492 ndk_path = build.ndk_path()
1493 # if for any reason the path isn't valid or the directory
1494 # doesn't exist, some versions of Gradle will error with a
1495 # cryptic message (even if the NDK is not even necessary).
1496 # https://gitlab.com/fdroid/fdroidserver/issues/171
1497 if ndk_path and os.path.exists(ndk_path):
1499 props += "ndk.dir=%s\n" % ndk_path
1500 props += "ndk-location=%s\n" % ndk_path
1501 # Add java.encoding if necessary
1503 props += "java.encoding=%s\n" % build.encoding
1504 with open(path, 'w', encoding='iso-8859-1') as f:
1508 if build.build_method() == 'gradle':
1509 flavours = build.gradle
1512 n = build.target.split('-')[1]
1513 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1514 r'compileSdkVersion %s' % n,
1515 os.path.join(root_dir, 'build.gradle'))
1517 # Remove forced debuggable flags
1518 remove_debuggable_flags(root_dir)
1520 # Insert version code and number into the manifest if necessary
1521 if build.forceversion:
1522 logging.info("Changing the version name")
1523 for path in manifest_paths(root_dir, flavours):
1524 if not os.path.isfile(path):
1526 if has_extension(path, 'xml'):
1527 regsub_file(r'android:versionName="[^"]*"',
1528 r'android:versionName="%s"' % build.versionName,
1530 elif has_extension(path, 'gradle'):
1531 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1532 r"""\1versionName '%s'""" % build.versionName,
1535 if build.forcevercode:
1536 logging.info("Changing the version code")
1537 for path in manifest_paths(root_dir, flavours):
1538 if not os.path.isfile(path):
1540 if has_extension(path, 'xml'):
1541 regsub_file(r'android:versionCode="[^"]*"',
1542 r'android:versionCode="%s"' % build.versionCode,
1544 elif has_extension(path, 'gradle'):
1545 regsub_file(r'versionCode[ =]+[0-9]+',
1546 r'versionCode %s' % build.versionCode,
1549 # Delete unwanted files
1551 logging.info("Removing specified files")
1552 for part in getpaths(build_dir, build.rm):
1553 dest = os.path.join(build_dir, part)
1554 logging.info("Removing {0}".format(part))
1555 if os.path.lexists(dest):
1556 if os.path.islink(dest):
1557 FDroidPopen(['unlink', dest], output=False)
1559 FDroidPopen(['rm', '-rf', dest], output=False)
1561 logging.info("...but it didn't exist")
1563 remove_signing_keys(build_dir)
1565 # Add required external libraries
1567 logging.info("Collecting prebuilt libraries")
1568 libsdir = os.path.join(root_dir, 'libs')
1569 if not os.path.exists(libsdir):
1571 for lib in build.extlibs:
1573 logging.info("...installing extlib {0}".format(lib))
1574 libf = os.path.basename(lib)
1575 libsrc = os.path.join(extlib_dir, lib)
1576 if not os.path.exists(libsrc):
1577 raise BuildException("Missing extlib file {0}".format(libsrc))
1578 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1580 # Run a pre-build command if one is required
1582 logging.info("Running 'prebuild' commands in %s" % root_dir)
1584 cmd = replace_config_vars(build.prebuild, build)
1586 # Substitute source library paths into prebuild commands
1587 for name, number, libpath in srclibpaths:
1588 libpath = os.path.relpath(libpath, root_dir)
1589 cmd = cmd.replace('$$' + name + '$$', libpath)
1591 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1592 if p.returncode != 0:
1593 raise BuildException("Error running prebuild command for %s:%s" %
1594 (app.id, build.versionName), p.output)
1596 # Generate (or update) the ant build file, build.xml...
1597 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1598 parms = ['android', 'update', 'lib-project']
1599 lparms = ['android', 'update', 'project']
1602 parms += ['-t', build.target]
1603 lparms += ['-t', build.target]
1604 if build.androidupdate:
1605 update_dirs = build.androidupdate
1607 update_dirs = ant_subprojects(root_dir) + ['.']
1609 for d in update_dirs:
1610 subdir = os.path.join(root_dir, d)
1612 logging.debug("Updating main project")
1613 cmd = parms + ['-p', d]
1615 logging.debug("Updating subproject %s" % d)
1616 cmd = lparms + ['-p', d]
1617 p = SdkToolsPopen(cmd, cwd=root_dir)
1618 # Check to see whether an error was returned without a proper exit
1619 # code (this is the case for the 'no target set or target invalid'
1621 if p.returncode != 0 or p.output.startswith("Error: "):
1622 raise BuildException("Failed to update project at %s" % d, p.output)
1623 # Clean update dirs via ant
1625 logging.info("Cleaning subproject %s" % d)
1626 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1628 return (root_dir, srclibpaths)
1631 # Extend via globbing the paths from a field and return them as a map from
1632 # original path to resulting paths
1633 def getpaths_map(build_dir, globpaths):
1637 full_path = os.path.join(build_dir, p)
1638 full_path = os.path.normpath(full_path)
1639 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1641 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1645 # Extend via globbing the paths from a field and return them as a set
1646 def getpaths(build_dir, globpaths):
1647 paths_map = getpaths_map(build_dir, globpaths)
1649 for k, v in paths_map.items():
1656 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1662 self.path = os.path.join('stats', 'known_apks.txt')
1664 if os.path.isfile(self.path):
1665 with open(self.path, 'r', encoding='utf8') as f:
1667 t = line.rstrip().split(' ')
1669 self.apks[t[0]] = (t[1], None)
1671 self.apks[t[0]] = (t[1], datetime.strptime(t[2], '%Y-%m-%d'))
1672 self.changed = False
1674 def writeifchanged(self):
1675 if not self.changed:
1678 if not os.path.exists('stats'):
1682 for apk, app in self.apks.items():
1684 line = apk + ' ' + appid
1686 line += ' ' + added.strftime('%Y-%m-%d')
1689 with open(self.path, 'w', encoding='utf8') as f:
1690 for line in sorted(lst, key=natural_key):
1691 f.write(line + '\n')
1693 def recordapk(self, apk, app, default_date=None):
1695 Record an apk (if it's new, otherwise does nothing)
1696 Returns the date it was added as a datetime instance
1698 if apk not in self.apks:
1699 if default_date is None:
1700 default_date = datetime.utcnow()
1701 self.apks[apk] = (app, default_date)
1703 _, added = self.apks[apk]
1706 # Look up information - given the 'apkname', returns (app id, date added/None).
1707 # Or returns None for an unknown apk.
1708 def getapp(self, apkname):
1709 if apkname in self.apks:
1710 return self.apks[apkname]
1713 # Get the most recent 'num' apps added to the repo, as a list of package ids
1714 # with the most recent first.
1715 def getlatest(self, num):
1717 for apk, app in self.apks.items():
1721 if apps[appid] > added:
1725 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1726 lst = [app for app, _ in sortedapps]
1731 def get_file_extension(filename):
1732 """get the normalized file extension, can be blank string but never None"""
1734 return os.path.splitext(filename)[1].lower()[1:]
1737 def isApkAndDebuggable(apkfile, config):
1738 """Returns True if the given file is an APK and is debuggable
1740 :param apkfile: full path to the apk to check"""
1742 if get_file_extension(apkfile) != 'apk':
1745 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1747 if p.returncode != 0:
1748 logging.critical("Failed to get apk manifest information")
1750 for line in p.output.splitlines():
1751 if 'android:debuggable' in line and not line.endswith('0x0'):
1758 self.returncode = None
1762 def SdkToolsPopen(commands, cwd=None, output=True):
1764 if cmd not in config:
1765 config[cmd] = find_sdk_tools_cmd(commands[0])
1766 abscmd = config[cmd]
1768 logging.critical("Could not find '%s' on your system" % cmd)
1771 test_aapt_version(config['aapt'])
1772 return FDroidPopen([abscmd] + commands[1:],
1773 cwd=cwd, output=output)
1776 def FDroidPopenBytes(commands, cwd=None, output=True, stderr_to_stdout=True):
1778 Run a command and capture the possibly huge output as bytes.
1780 :param commands: command and argument list like in subprocess.Popen
1781 :param cwd: optionally specifies a working directory
1782 :returns: A PopenResult.
1787 set_FDroidPopen_env()
1790 cwd = os.path.normpath(cwd)
1791 logging.debug("Directory: %s" % cwd)
1792 logging.debug("> %s" % ' '.join(commands))
1794 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
1795 result = PopenResult()
1798 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
1799 stdout=subprocess.PIPE, stderr=stderr_param)
1800 except OSError as e:
1801 raise BuildException("OSError while trying to execute " +
1802 ' '.join(commands) + ': ' + str(e))
1804 if not stderr_to_stdout and options.verbose:
1805 stderr_queue = Queue()
1806 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
1808 while not stderr_reader.eof():
1809 while not stderr_queue.empty():
1810 line = stderr_queue.get()
1811 sys.stderr.buffer.write(line)
1816 stdout_queue = Queue()
1817 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1820 # Check the queue for output (until there is no more to get)
1821 while not stdout_reader.eof():
1822 while not stdout_queue.empty():
1823 line = stdout_queue.get()
1824 if output and options.verbose:
1825 # Output directly to console
1826 sys.stderr.buffer.write(line)
1832 result.returncode = p.wait()
1833 result.output = buf.getvalue()
1838 def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
1840 Run a command and capture the possibly huge output as a str.
1842 :param commands: command and argument list like in subprocess.Popen
1843 :param cwd: optionally specifies a working directory
1844 :returns: A PopenResult.
1846 result = FDroidPopenBytes(commands, cwd, output, stderr_to_stdout)
1847 result.output = result.output.decode('utf-8', 'ignore')
1851 gradle_comment = re.compile(r'[ ]*//')
1852 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1853 gradle_line_matches = [
1854 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1855 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1856 re.compile(r'.*\.readLine\(.*'),
1860 def remove_signing_keys(build_dir):
1861 for root, dirs, files in os.walk(build_dir):
1862 if 'build.gradle' in files:
1863 path = os.path.join(root, 'build.gradle')
1865 with open(path, "r", encoding='utf8') as o:
1866 lines = o.readlines()
1872 with open(path, "w", encoding='utf8') as o:
1873 while i < len(lines):
1876 while line.endswith('\\\n'):
1877 line = line.rstrip('\\\n') + lines[i]
1880 if gradle_comment.match(line):
1885 opened += line.count('{')
1886 opened -= line.count('}')
1889 if gradle_signing_configs.match(line):
1894 if any(s.match(line) for s in gradle_line_matches):
1902 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1905 'project.properties',
1907 'default.properties',
1908 'ant.properties', ]:
1909 if propfile in files:
1910 path = os.path.join(root, propfile)
1912 with open(path, "r", encoding='iso-8859-1') as o:
1913 lines = o.readlines()
1917 with open(path, "w", encoding='iso-8859-1') as o:
1919 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1926 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1929 def set_FDroidPopen_env(build=None):
1931 set up the environment variables for the build environment
1933 There is only a weak standard, the variables used by gradle, so also set
1934 up the most commonly used environment variables for SDK and NDK. Also, if
1935 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
1937 global env, orig_path
1941 orig_path = env['PATH']
1942 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
1943 env[n] = config['sdk_path']
1944 for k, v in config['java_paths'].items():
1945 env['JAVA%s_HOME' % k] = v
1947 missinglocale = True
1948 for k, v in env.items():
1949 if k == 'LANG' and v != 'C':
1950 missinglocale = False
1952 missinglocale = False
1954 env['LANG'] = 'en_US.UTF-8'
1956 if build is not None:
1957 path = build.ndk_path()
1958 paths = orig_path.split(os.pathsep)
1959 if path not in paths:
1960 paths = [path] + paths
1961 env['PATH'] = os.pathsep.join(paths)
1962 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
1963 env[n] = build.ndk_path()
1966 def replace_build_vars(cmd, build):
1967 cmd = cmd.replace('$$COMMIT$$', build.commit)
1968 cmd = cmd.replace('$$VERSION$$', build.versionName)
1969 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
1973 def replace_config_vars(cmd, build):
1974 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1975 cmd = cmd.replace('$$NDK$$', build.ndk_path())
1976 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1977 cmd = cmd.replace('$$QT$$', config['qt_sdk_path'] or '')
1978 if build is not None:
1979 cmd = replace_build_vars(cmd, build)
1983 def place_srclib(root_dir, number, libpath):
1986 relpath = os.path.relpath(libpath, root_dir)
1987 proppath = os.path.join(root_dir, 'project.properties')
1990 if os.path.isfile(proppath):
1991 with open(proppath, "r", encoding='iso-8859-1') as o:
1992 lines = o.readlines()
1994 with open(proppath, "w", encoding='iso-8859-1') as o:
1997 if line.startswith('android.library.reference.%d=' % number):
1998 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2003 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2006 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
2009 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
2010 """Verify that two apks are the same
2012 One of the inputs is signed, the other is unsigned. The signature metadata
2013 is transferred from the signed to the unsigned apk, and then jarsigner is
2014 used to verify that the signature from the signed apk is also varlid for
2015 the unsigned one. If the APK given as unsigned actually does have a
2016 signature, it will be stripped out and ignored.
2017 :param signed_apk: Path to a signed apk file
2018 :param unsigned_apk: Path to an unsigned apk file expected to match it
2019 :param tmp_dir: Path to directory for temporary files
2020 :returns: None if the verification is successful, otherwise a string
2021 describing what went wrong.
2024 signed = ZipFile(signed_apk, 'r')
2025 meta_inf_files = ['META-INF/MANIFEST.MF']
2026 for f in signed.namelist():
2027 if apk_sigfile.match(f):
2028 meta_inf_files.append(f)
2029 if len(meta_inf_files) < 3:
2030 return "Signature files missing from {0}".format(signed_apk)
2032 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2033 unsigned = ZipFile(unsigned_apk, 'r')
2034 # only read the signature from the signed APK, everything else from unsigned
2035 with ZipFile(tmp_apk, 'w') as tmp:
2036 for filename in meta_inf_files:
2037 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2038 for info in unsigned.infolist():
2039 if info.filename in meta_inf_files:
2040 logging.warning('Ignoring ' + info.filename + ' from ' + unsigned_apk)
2042 if info.filename in tmp.namelist():
2043 return "duplicate filename found: " + info.filename
2044 tmp.writestr(info, unsigned.read(info.filename))
2048 if subprocess.call([config['jarsigner'], '-verify', tmp_apk]) != 0:
2049 logging.info("...NOT verified - {0}".format(unsigned_apk))
2050 return compare_apks(signed_apk, tmp_apk, tmp_dir)
2052 logging.info("...successfully verified")
2056 apk_badchars = re.compile('''[/ :;'"]''')
2059 def compare_apks(apk1, apk2, tmp_dir):
2062 Returns None if the apk content is the same (apart from the signing key),
2063 otherwise a string describing what's different, or what went wrong when
2064 trying to do the comparison.
2067 absapk1 = os.path.abspath(apk1)
2068 absapk2 = os.path.abspath(apk2)
2070 # try to find diffoscope in the path, if it hasn't been manually configed
2071 if 'diffoscope' not in config:
2072 tmp = find_command('diffoscope')
2074 config['diffoscope'] = tmp
2075 if 'diffoscope' in config:
2076 htmlfile = absapk1 + '.diffoscope.html'
2077 textfile = absapk1 + '.diffoscope.txt'
2078 if subprocess.call([config['diffoscope'],
2079 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2080 '--html', htmlfile, '--text', textfile,
2081 absapk1, absapk2]) != 0:
2082 return("Failed to unpack " + apk1)
2084 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2085 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2086 for d in [apk1dir, apk2dir]:
2087 if os.path.exists(d):
2090 os.mkdir(os.path.join(d, 'jar-xf'))
2092 if subprocess.call(['jar', 'xf',
2093 os.path.abspath(apk1)],
2094 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2095 return("Failed to unpack " + apk1)
2096 if subprocess.call(['jar', 'xf',
2097 os.path.abspath(apk2)],
2098 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2099 return("Failed to unpack " + apk2)
2101 # try to find apktool in the path, if it hasn't been manually configed
2102 if 'apktool' not in config:
2103 tmp = find_command('apktool')
2105 config['apktool'] = tmp
2106 if 'apktool' in config:
2107 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2109 return("Failed to unpack " + apk1)
2110 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2112 return("Failed to unpack " + apk2)
2114 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2115 lines = p.output.splitlines()
2116 if len(lines) != 1 or 'META-INF' not in lines[0]:
2117 meld = find_command('meld')
2118 if meld is not None:
2119 p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
2120 return("Unexpected diff output - " + p.output)
2122 # since everything verifies, delete the comparison to keep cruft down
2123 shutil.rmtree(apk1dir)
2124 shutil.rmtree(apk2dir)
2126 # If we get here, it seems like they're the same!
2130 def find_command(command):
2131 '''find the full path of a command, or None if it can't be found in the PATH'''
2134 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2136 fpath, fname = os.path.split(command)
2141 for path in os.environ["PATH"].split(os.pathsep):
2142 path = path.strip('"')
2143 exe_file = os.path.join(path, command)
2144 if is_exe(exe_file):
2151 '''generate a random password for when generating keys'''
2152 h = hashlib.sha256()
2153 h.update(os.urandom(16)) # salt
2154 h.update(socket.getfqdn().encode('utf-8'))
2155 passwd = base64.b64encode(h.digest()).strip()
2156 return passwd.decode('utf-8')
2159 def genkeystore(localconfig):
2160 '''Generate a new key with random passwords and add it to new keystore'''
2161 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2162 keystoredir = os.path.dirname(localconfig['keystore'])
2163 if keystoredir is None or keystoredir == '':
2164 keystoredir = os.path.join(os.getcwd(), keystoredir)
2165 if not os.path.exists(keystoredir):
2166 os.makedirs(keystoredir, mode=0o700)
2168 write_password_file("keystorepass", localconfig['keystorepass'])
2169 write_password_file("keypass", localconfig['keypass'])
2170 p = FDroidPopen([config['keytool'], '-genkey',
2171 '-keystore', localconfig['keystore'],
2172 '-alias', localconfig['repo_keyalias'],
2173 '-keyalg', 'RSA', '-keysize', '4096',
2174 '-sigalg', 'SHA256withRSA',
2175 '-validity', '10000',
2176 '-storepass:file', config['keystorepassfile'],
2177 '-keypass:file', config['keypassfile'],
2178 '-dname', localconfig['keydname']])
2179 # TODO keypass should be sent via stdin
2180 if p.returncode != 0:
2181 raise BuildException("Failed to generate key", p.output)
2182 os.chmod(localconfig['keystore'], 0o0600)
2183 # now show the lovely key that was just generated
2184 p = FDroidPopen([config['keytool'], '-list', '-v',
2185 '-keystore', localconfig['keystore'],
2186 '-alias', localconfig['repo_keyalias'],
2187 '-storepass:file', config['keystorepassfile']])
2188 logging.info(p.output.strip() + '\n\n')
2191 def write_to_config(thisconfig, key, value=None):
2192 '''write a key/value to the local config.py'''
2194 origkey = key + '_orig'
2195 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2196 with open('config.py', 'r', encoding='utf8') as f:
2198 pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
2199 repl = '\n' + key + ' = "' + value + '"'
2200 data = re.sub(pattern, repl, data)
2201 # if this key is not in the file, append it
2202 if not re.match('\s*' + key + '\s*=\s*"', data):
2204 # make sure the file ends with a carraige return
2205 if not re.match('\n$', data):
2207 with open('config.py', 'w', encoding='utf8') as f:
2211 def parse_xml(path):
2212 return XMLElementTree.parse(path).getroot()
2215 def string_is_integer(string):
2223 def get_per_app_repos():
2224 '''per-app repos are dirs named with the packageName of a single app'''
2226 # Android packageNames are Java packages, they may contain uppercase or
2227 # lowercase letters ('A' through 'Z'), numbers, and underscores
2228 # ('_'). However, individual package name parts may only start with
2229 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2230 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2233 for root, dirs, files in os.walk(os.getcwd()):
2235 print('checking', root, 'for', d)
2236 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2237 # standard parts of an fdroid repo, so never packageNames
2240 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
2246 def is_repo_file(filename):
2247 '''Whether the file in a repo is a build product to be delivered to users'''
2248 return os.path.isfile(filename) \
2249 and not filename.endswith('.asc') \
2250 and not filename.endswith('.sig') \
2251 and os.path.basename(filename) not in [
2253 'index_unsigned.jar',