3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
37 import xml.etree.ElementTree as XMLElementTree
39 from binascii import hexlify
40 from datetime import datetime
41 from distutils.version import LooseVersion
42 from queue import Queue
43 from zipfile import ZipFile
45 from pyasn1.codec.der import decoder, encoder
46 from pyasn1_modules import rfc2315
47 from pyasn1.error import PyAsn1Error
49 import fdroidserver.metadata
50 from .asynchronousfilereader import AsynchronousFileReader
53 # A signature block file with a .DSA, .RSA, or .EC extension
54 CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
56 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
65 'sdk_path': "$ANDROID_HOME",
70 'r12b': "$ANDROID_NDK",
75 'build_tools': "25.0.2",
76 'force_build_tools': False,
81 'accepted_formats': ['txt', 'yml'],
82 'sync_from_local_copy_dir': False,
83 'per_app_repos': False,
84 'make_current_version_link': True,
85 'current_version_name_source': 'Name',
86 'update_stats': False,
90 'stats_to_carbon': False,
92 'build_server_always': False,
93 'keystore': 'keystore.jks',
94 'smartcardoptions': [],
100 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
101 'repo_name': "My First FDroid Repo Demo",
102 'repo_icon': "fdroid-icon.png",
103 'repo_description': '''
104 This is a repository of apps to be used with FDroid. Applications in this
105 repository are either official binaries built by the original application
106 developers, or are binaries built from source by the admin of f-droid.org
107 using the tools on https://gitlab.com/u/fdroid.
113 def setup_global_opts(parser):
114 parser.add_argument("-v", "--verbose", action="store_true", default=False,
115 help="Spew out even more information than normal")
116 parser.add_argument("-q", "--quiet", action="store_true", default=False,
117 help="Restrict output to warnings and errors")
120 def fill_config_defaults(thisconfig):
121 for k, v in default_config.items():
122 if k not in thisconfig:
125 # Expand paths (~users and $vars)
126 def expand_path(path):
130 path = os.path.expanduser(path)
131 path = os.path.expandvars(path)
136 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
141 thisconfig[k + '_orig'] = v
143 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
144 if thisconfig['java_paths'] is None:
145 thisconfig['java_paths'] = dict()
147 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
148 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
149 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
150 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
151 if os.getenv('JAVA_HOME') is not None:
152 pathlist.append(os.getenv('JAVA_HOME'))
153 if os.getenv('PROGRAMFILES') is not None:
154 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
155 for d in sorted(pathlist):
156 if os.path.islink(d):
158 j = os.path.basename(d)
159 # the last one found will be the canonical one, so order appropriately
161 r'^1\.([6-9])\.0\.jdk$', # OSX
162 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
163 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
164 r'^jdk([6-9])-openjdk$', # Arch
165 r'^java-([6-9])-openjdk$', # Arch
166 r'^java-([6-9])-jdk$', # Arch (oracle)
167 r'^java-1\.([6-9])\.0-.*$', # RedHat
168 r'^java-([6-9])-oracle$', # Debian WebUpd8
169 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
170 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
172 m = re.match(regex, j)
175 for p in [d, os.path.join(d, 'Contents', 'Home')]:
176 if os.path.exists(os.path.join(p, 'bin', 'javac')):
177 thisconfig['java_paths'][m.group(1)] = p
179 for java_version in ('7', '8', '9'):
180 if java_version not in thisconfig['java_paths']:
182 java_home = thisconfig['java_paths'][java_version]
183 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
184 if os.path.exists(jarsigner):
185 thisconfig['jarsigner'] = jarsigner
186 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
187 break # Java7 is preferred, so quit if found
189 for k in ['ndk_paths', 'java_paths']:
195 thisconfig[k][k2] = exp
196 thisconfig[k][k2 + '_orig'] = v
199 def regsub_file(pattern, repl, path):
200 with open(path, 'rb') as f:
202 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
203 with open(path, 'wb') as f:
207 def read_config(opts, config_file='config.py'):
208 """Read the repository config
210 The config is read from config_file, which is in the current
211 directory when any of the repo management commands are used. If
212 there is a local metadata file in the git repo, then config.py is
213 not required, just use defaults.
216 global config, options
218 if config is not None:
225 if os.path.isfile(config_file):
226 logging.debug("Reading %s" % config_file)
227 with io.open(config_file, "rb") as f:
228 code = compile(f.read(), config_file, 'exec')
229 exec(code, None, config)
230 elif len(get_local_metadata_files()) == 0:
231 logging.critical("Missing config file - is this a repo directory?")
234 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
236 if not type(config[k]) in (str, list, tuple):
237 logging.warn('"' + k + '" will be in random order!'
238 + ' Use () or [] brackets if order is important!')
240 # smartcardoptions must be a list since its command line args for Popen
241 if 'smartcardoptions' in config:
242 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
243 elif 'keystore' in config and config['keystore'] == 'NONE':
244 # keystore='NONE' means use smartcard, these are required defaults
245 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
246 'SunPKCS11-OpenSC', '-providerClass',
247 'sun.security.pkcs11.SunPKCS11',
248 '-providerArg', 'opensc-fdroid.cfg']
250 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
251 st = os.stat(config_file)
252 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
253 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
255 fill_config_defaults(config)
257 for k in ["keystorepass", "keypass"]:
259 write_password_file(k)
261 for k in ["repo_description", "archive_description"]:
263 config[k] = clean_description(config[k])
265 if 'serverwebroot' in config:
266 if isinstance(config['serverwebroot'], str):
267 roots = [config['serverwebroot']]
268 elif all(isinstance(item, str) for item in config['serverwebroot']):
269 roots = config['serverwebroot']
271 raise TypeError('only accepts strings, lists, and tuples')
273 for rootstr in roots:
274 # since this is used with rsync, where trailing slashes have
275 # meaning, ensure there is always a trailing slash
276 if rootstr[-1] != '/':
278 rootlist.append(rootstr.replace('//', '/'))
279 config['serverwebroot'] = rootlist
281 if 'servergitmirrors' in config:
282 if isinstance(config['servergitmirrors'], str):
283 roots = [config['servergitmirrors']]
284 elif all(isinstance(item, str) for item in config['servergitmirrors']):
285 roots = config['servergitmirrors']
287 raise TypeError('only accepts strings, lists, and tuples')
288 config['servergitmirrors'] = roots
293 def find_sdk_tools_cmd(cmd):
294 '''find a working path to a tool from the Android SDK'''
297 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
298 # try to find a working path to this command, in all the recent possible paths
299 if 'build_tools' in config:
300 build_tools = os.path.join(config['sdk_path'], 'build-tools')
301 # if 'build_tools' was manually set and exists, check only that one
302 configed_build_tools = os.path.join(build_tools, config['build_tools'])
303 if os.path.exists(configed_build_tools):
304 tooldirs.append(configed_build_tools)
306 # no configed version, so hunt known paths for it
307 for f in sorted(os.listdir(build_tools), reverse=True):
308 if os.path.isdir(os.path.join(build_tools, f)):
309 tooldirs.append(os.path.join(build_tools, f))
310 tooldirs.append(build_tools)
311 sdk_tools = os.path.join(config['sdk_path'], 'tools')
312 if os.path.exists(sdk_tools):
313 tooldirs.append(sdk_tools)
314 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
315 if os.path.exists(sdk_platform_tools):
316 tooldirs.append(sdk_platform_tools)
317 tooldirs.append('/usr/bin')
319 path = os.path.join(d, cmd)
320 if os.path.isfile(path):
322 test_aapt_version(path)
324 # did not find the command, exit with error message
325 ensure_build_tools_exists(config)
328 def test_aapt_version(aapt):
329 '''Check whether the version of aapt is new enough'''
330 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
331 if output is None or output == '':
332 logging.error(aapt + ' failed to execute!')
334 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
339 # the Debian package has the version string like "v0.2-23.0.2"
340 if '.' not in bugfix and LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.2166767'):
341 logging.warning(aapt + ' is too old, fdroid requires build-tools-23.0.0 or newer!')
343 logging.warning('Unknown version of aapt, might cause problems: ' + output)
346 def test_sdk_exists(thisconfig):
347 if 'sdk_path' not in thisconfig:
348 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
349 test_aapt_version(thisconfig['aapt'])
352 logging.error("'sdk_path' not set in config.py!")
354 if thisconfig['sdk_path'] == default_config['sdk_path']:
355 logging.error('No Android SDK found!')
356 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
357 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
359 if not os.path.exists(thisconfig['sdk_path']):
360 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
362 if not os.path.isdir(thisconfig['sdk_path']):
363 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
365 for d in ['build-tools', 'platform-tools', 'tools']:
366 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
367 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
368 thisconfig['sdk_path'], d))
373 def ensure_build_tools_exists(thisconfig):
374 if not test_sdk_exists(thisconfig):
376 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
377 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
378 if not os.path.isdir(versioned_build_tools):
379 logging.critical('Android Build Tools path "'
380 + versioned_build_tools + '" does not exist!')
384 def write_password_file(pwtype, password=None):
386 writes out passwords to a protected file instead of passing passwords as
387 command line argments
389 filename = '.fdroid.' + pwtype + '.txt'
390 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
392 os.write(fd, config[pwtype].encode('utf-8'))
394 os.write(fd, password.encode('utf-8'))
396 config[pwtype + 'file'] = filename
399 def get_local_metadata_files():
400 '''get any metadata files local to an app's source repo
402 This tries to ignore anything that does not count as app metdata,
403 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
406 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
409 def read_pkg_args(args, allow_vercodes=False):
411 Given the arguments in the form of multiple appid:[vc] strings, this returns
412 a dictionary with the set of vercodes specified for each package.
420 if allow_vercodes and ':' in p:
421 package, vercode = p.split(':')
423 package, vercode = p, None
424 if package not in vercodes:
425 vercodes[package] = [vercode] if vercode else []
427 elif vercode and vercode not in vercodes[package]:
428 vercodes[package] += [vercode] if vercode else []
433 def read_app_args(args, allapps, allow_vercodes=False):
435 On top of what read_pkg_args does, this returns the whole app metadata, but
436 limiting the builds list to the builds matching the vercodes specified.
439 vercodes = read_pkg_args(args, allow_vercodes)
445 for appid, app in allapps.items():
446 if appid in vercodes:
449 if len(apps) != len(vercodes):
452 logging.critical("No such package: %s" % p)
453 raise FDroidException("Found invalid app ids in arguments")
455 raise FDroidException("No packages specified")
458 for appid, app in apps.items():
462 app.builds = [b for b in app.builds if b.versionCode in vc]
463 if len(app.builds) != len(vercodes[appid]):
465 allvcs = [b.versionCode for b in app.builds]
466 for v in vercodes[appid]:
468 logging.critical("No such vercode %s for app %s" % (v, appid))
471 raise FDroidException("Found invalid vercodes for some apps")
476 def get_extension(filename):
477 base, ext = os.path.splitext(filename)
480 return base, ext.lower()[1:]
483 def has_extension(filename, ext):
484 _, f_ext = get_extension(filename)
488 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
491 def clean_description(description):
492 'Remove unneeded newlines and spaces from a block of description text'
494 # this is split up by paragraph to make removing the newlines easier
495 for paragraph in re.split(r'\n\n', description):
496 paragraph = re.sub('\r', '', paragraph)
497 paragraph = re.sub('\n', ' ', paragraph)
498 paragraph = re.sub(' {2,}', ' ', paragraph)
499 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
500 returnstring += paragraph + '\n\n'
501 return returnstring.rstrip('\n')
504 def publishednameinfo(filename):
505 filename = os.path.basename(filename)
506 m = publish_name_regex.match(filename)
508 result = (m.group(1), m.group(2))
509 except AttributeError:
510 raise FDroidException("Invalid name for published file: %s" % filename)
514 def get_release_filename(app, build):
516 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
518 return "%s_%s.apk" % (app.id, build.versionCode)
521 def get_toolsversion_logname(app, build):
522 return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
525 def getsrcname(app, build):
526 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
538 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
541 def get_build_dir(app):
542 '''get the dir that this app will be built in'''
544 if app.RepoType == 'srclib':
545 return os.path.join('build', 'srclib', app.Repo)
547 return os.path.join('build', app.id)
551 '''checkout code from VCS and return instance of vcs and the build dir'''
552 build_dir = get_build_dir(app)
554 # Set up vcs interface and make sure we have the latest code...
555 logging.debug("Getting {0} vcs interface for {1}"
556 .format(app.RepoType, app.Repo))
557 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
561 vcs = getvcs(app.RepoType, remote, build_dir)
563 return vcs, build_dir
566 def getvcs(vcstype, remote, local):
568 return vcs_git(remote, local)
569 if vcstype == 'git-svn':
570 return vcs_gitsvn(remote, local)
572 return vcs_hg(remote, local)
574 return vcs_bzr(remote, local)
575 if vcstype == 'srclib':
576 if local != os.path.join('build', 'srclib', remote):
577 raise VCSException("Error: srclib paths are hard-coded!")
578 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
580 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
581 raise VCSException("Invalid vcs type " + vcstype)
584 def getsrclibvcs(name):
585 if name not in fdroidserver.metadata.srclibs:
586 raise VCSException("Missing srclib " + name)
587 return fdroidserver.metadata.srclibs[name]['Repo Type']
592 def __init__(self, remote, local):
594 # svn, git-svn and bzr may require auth
596 if self.repotype() in ('git-svn', 'bzr'):
598 if self.repotype == 'git-svn':
599 raise VCSException("Authentication is not supported for git-svn")
600 self.username, remote = remote.split('@')
601 if ':' not in self.username:
602 raise VCSException("Password required with username")
603 self.username, self.password = self.username.split(':')
607 self.clone_failed = False
608 self.refreshed = False
614 # Take the local repository to a clean version of the given revision, which
615 # is specificed in the VCS's native format. Beforehand, the repository can
616 # be dirty, or even non-existent. If the repository does already exist
617 # locally, it will be updated from the origin, but only once in the
618 # lifetime of the vcs object.
619 # None is acceptable for 'rev' if you know you are cloning a clean copy of
620 # the repo - otherwise it must specify a valid revision.
621 def gotorevision(self, rev, refresh=True):
623 if self.clone_failed:
624 raise VCSException("Downloading the repository already failed once, not trying again.")
626 # The .fdroidvcs-id file for a repo tells us what VCS type
627 # and remote that directory was created from, allowing us to drop it
628 # automatically if either of those things changes.
629 fdpath = os.path.join(self.local, '..',
630 '.fdroidvcs-' + os.path.basename(self.local))
631 fdpath = os.path.normpath(fdpath)
632 cdata = self.repotype() + ' ' + self.remote
635 if os.path.exists(self.local):
636 if os.path.exists(fdpath):
637 with open(fdpath, 'r') as f:
638 fsdata = f.read().strip()
643 logging.info("Repository details for %s changed - deleting" % (
647 logging.info("Repository details for %s missing - deleting" % (
650 shutil.rmtree(self.local)
654 self.refreshed = True
657 self.gotorevisionx(rev)
658 except FDroidException as e:
661 # If necessary, write the .fdroidvcs file.
662 if writeback and not self.clone_failed:
663 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
664 with open(fdpath, 'w+') as f:
670 # Derived classes need to implement this. It's called once basic checking
671 # has been performend.
672 def gotorevisionx(self, rev):
673 raise VCSException("This VCS type doesn't define gotorevisionx")
675 # Initialise and update submodules
676 def initsubmodules(self):
677 raise VCSException('Submodules not supported for this vcs type')
679 # Get a list of all known tags
681 if not self._gettags:
682 raise VCSException('gettags not supported for this vcs type')
684 for tag in self._gettags():
685 if re.match('[-A-Za-z0-9_. /]+$', tag):
689 # Get a list of all the known tags, sorted from newest to oldest
690 def latesttags(self):
691 raise VCSException('latesttags not supported for this vcs type')
693 # Get current commit reference (hash, revision, etc)
695 raise VCSException('getref not supported for this vcs type')
697 # Returns the srclib (name, path) used in setting up the current
708 # If the local directory exists, but is somehow not a git repository, git
709 # will traverse up the directory tree until it finds one that is (i.e.
710 # fdroidserver) and then we'll proceed to destroy it! This is called as
713 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
714 result = p.output.rstrip()
715 if not result.endswith(self.local):
716 raise VCSException('Repository mismatch')
718 def gotorevisionx(self, rev):
719 if not os.path.exists(self.local):
721 p = FDroidPopen(['git', 'clone', self.remote, self.local])
722 if p.returncode != 0:
723 self.clone_failed = True
724 raise VCSException("Git clone failed", p.output)
728 # Discard any working tree changes
729 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
730 'git', 'reset', '--hard'], cwd=self.local, output=False)
731 if p.returncode != 0:
732 raise VCSException("Git reset failed", p.output)
733 # Remove untracked files now, in case they're tracked in the target
734 # revision (it happens!)
735 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
736 'git', 'clean', '-dffx'], cwd=self.local, output=False)
737 if p.returncode != 0:
738 raise VCSException("Git clean failed", p.output)
739 if not self.refreshed:
740 # Get latest commits and tags from remote
741 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
742 if p.returncode != 0:
743 raise VCSException("Git fetch failed", p.output)
744 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
745 if p.returncode != 0:
746 raise VCSException("Git fetch failed", p.output)
747 # Recreate origin/HEAD as git clone would do it, in case it disappeared
748 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
749 if p.returncode != 0:
750 lines = p.output.splitlines()
751 if 'Multiple remote HEAD branches' not in lines[0]:
752 raise VCSException("Git remote set-head failed", p.output)
753 branch = lines[1].split(' ')[-1]
754 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
755 if p2.returncode != 0:
756 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
757 self.refreshed = True
758 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
759 # a github repo. Most of the time this is the same as origin/master.
760 rev = rev or 'origin/HEAD'
761 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
762 if p.returncode != 0:
763 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
764 # Get rid of any uncontrolled files left behind
765 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
766 if p.returncode != 0:
767 raise VCSException("Git clean failed", p.output)
769 def initsubmodules(self):
771 submfile = os.path.join(self.local, '.gitmodules')
772 if not os.path.isfile(submfile):
773 raise VCSException("No git submodules available")
775 # fix submodules not accessible without an account and public key auth
776 with open(submfile, 'r') as f:
777 lines = f.readlines()
778 with open(submfile, 'w') as f:
780 if 'git@github.com' in line:
781 line = line.replace('git@github.com:', 'https://github.com/')
782 if 'git@gitlab.com' in line:
783 line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
786 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
787 if p.returncode != 0:
788 raise VCSException("Git submodule sync failed", p.output)
789 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
790 if p.returncode != 0:
791 raise VCSException("Git submodule update failed", p.output)
795 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
796 return p.output.splitlines()
798 tag_format = re.compile(r'tag: ([^),]*)')
800 def latesttags(self):
802 p = FDroidPopen(['git', 'log', '--tags',
803 '--simplify-by-decoration', '--pretty=format:%d'],
804 cwd=self.local, output=False)
806 for line in p.output.splitlines():
807 for tag in self.tag_format.findall(line):
812 class vcs_gitsvn(vcs):
817 # If the local directory exists, but is somehow not a git repository, git
818 # will traverse up the directory tree until it finds one that is (i.e.
819 # fdroidserver) and then we'll proceed to destory it! This is called as
822 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
823 result = p.output.rstrip()
824 if not result.endswith(self.local):
825 raise VCSException('Repository mismatch')
827 def gotorevisionx(self, rev):
828 if not os.path.exists(self.local):
830 gitsvn_args = ['git', 'svn', 'clone']
831 if ';' in self.remote:
832 remote_split = self.remote.split(';')
833 for i in remote_split[1:]:
834 if i.startswith('trunk='):
835 gitsvn_args.extend(['-T', i[6:]])
836 elif i.startswith('tags='):
837 gitsvn_args.extend(['-t', i[5:]])
838 elif i.startswith('branches='):
839 gitsvn_args.extend(['-b', i[9:]])
840 gitsvn_args.extend([remote_split[0], self.local])
841 p = FDroidPopen(gitsvn_args, output=False)
842 if p.returncode != 0:
843 self.clone_failed = True
844 raise VCSException("Git svn clone failed", p.output)
846 gitsvn_args.extend([self.remote, self.local])
847 p = FDroidPopen(gitsvn_args, output=False)
848 if p.returncode != 0:
849 self.clone_failed = True
850 raise VCSException("Git svn clone failed", p.output)
854 # Discard any working tree changes
855 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
856 if p.returncode != 0:
857 raise VCSException("Git reset failed", p.output)
858 # Remove untracked files now, in case they're tracked in the target
859 # revision (it happens!)
860 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
861 if p.returncode != 0:
862 raise VCSException("Git clean failed", p.output)
863 if not self.refreshed:
864 # Get new commits, branches and tags from repo
865 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
866 if p.returncode != 0:
867 raise VCSException("Git svn fetch failed")
868 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
869 if p.returncode != 0:
870 raise VCSException("Git svn rebase failed", p.output)
871 self.refreshed = True
873 rev = rev or 'master'
875 nospaces_rev = rev.replace(' ', '%20')
876 # Try finding a svn tag
877 for treeish in ['origin/', '']:
878 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
879 if p.returncode == 0:
881 if p.returncode != 0:
882 # No tag found, normal svn rev translation
883 # Translate svn rev into git format
884 rev_split = rev.split('/')
887 for treeish in ['origin/', '']:
888 if len(rev_split) > 1:
889 treeish += rev_split[0]
890 svn_rev = rev_split[1]
893 # if no branch is specified, then assume trunk (i.e. 'master' branch):
897 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
899 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
900 git_rev = p.output.rstrip()
902 if p.returncode == 0 and git_rev:
905 if p.returncode != 0 or not git_rev:
906 # Try a plain git checkout as a last resort
907 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
908 if p.returncode != 0:
909 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
911 # Check out the git rev equivalent to the svn rev
912 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
913 if p.returncode != 0:
914 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
916 # Get rid of any uncontrolled files left behind
917 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
918 if p.returncode != 0:
919 raise VCSException("Git clean failed", p.output)
923 for treeish in ['origin/', '']:
924 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
930 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
931 if p.returncode != 0:
933 return p.output.strip()
941 def gotorevisionx(self, rev):
942 if not os.path.exists(self.local):
943 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
944 if p.returncode != 0:
945 self.clone_failed = True
946 raise VCSException("Hg clone failed", p.output)
948 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
949 if p.returncode != 0:
950 raise VCSException("Hg status failed", p.output)
951 for line in p.output.splitlines():
952 if not line.startswith('? '):
953 raise VCSException("Unexpected output from hg status -uS: " + line)
954 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
955 if not self.refreshed:
956 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
957 if p.returncode != 0:
958 raise VCSException("Hg pull failed", p.output)
959 self.refreshed = True
961 rev = rev or 'default'
964 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
965 if p.returncode != 0:
966 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
967 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
968 # Also delete untracked files, we have to enable purge extension for that:
969 if "'purge' is provided by the following extension" in p.output:
970 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
971 myfile.write("\n[extensions]\nhgext.purge=\n")
972 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
973 if p.returncode != 0:
974 raise VCSException("HG purge failed", p.output)
975 elif p.returncode != 0:
976 raise VCSException("HG purge failed", p.output)
979 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
980 return p.output.splitlines()[1:]
988 def gotorevisionx(self, rev):
989 if not os.path.exists(self.local):
990 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
991 if p.returncode != 0:
992 self.clone_failed = True
993 raise VCSException("Bzr branch failed", p.output)
995 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
996 if p.returncode != 0:
997 raise VCSException("Bzr revert failed", p.output)
998 if not self.refreshed:
999 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
1000 if p.returncode != 0:
1001 raise VCSException("Bzr update failed", p.output)
1002 self.refreshed = True
1004 revargs = list(['-r', rev] if rev else [])
1005 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
1006 if p.returncode != 0:
1007 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1010 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
1011 return [tag.split(' ')[0].strip() for tag in
1012 p.output.splitlines()]
1015 def unescape_string(string):
1018 if string[0] == '"' and string[-1] == '"':
1021 return string.replace("\\'", "'")
1024 def retrieve_string(app_dir, string, xmlfiles=None):
1026 if not string.startswith('@string/'):
1027 return unescape_string(string)
1029 if xmlfiles is None:
1032 os.path.join(app_dir, 'res'),
1033 os.path.join(app_dir, 'src', 'main', 'res'),
1035 for r, d, f in os.walk(res_dir):
1036 if os.path.basename(r) == 'values':
1037 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
1039 name = string[len('@string/'):]
1041 def element_content(element):
1042 if element.text is None:
1044 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1045 return s.decode('utf-8').strip()
1047 for path in xmlfiles:
1048 if not os.path.isfile(path):
1050 xml = parse_xml(path)
1051 element = xml.find('string[@name="' + name + '"]')
1052 if element is not None:
1053 content = element_content(element)
1054 return retrieve_string(app_dir, content, xmlfiles)
1059 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1060 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1063 def manifest_paths(app_dir, flavours):
1064 '''Return list of existing files that will be used to find the highest vercode'''
1066 possible_manifests = \
1067 [os.path.join(app_dir, 'AndroidManifest.xml'),
1068 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1069 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1070 os.path.join(app_dir, 'build.gradle')]
1072 for flavour in flavours:
1073 if flavour == 'yes':
1075 possible_manifests.append(
1076 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1078 return [path for path in possible_manifests if os.path.isfile(path)]
1081 def fetch_real_name(app_dir, flavours):
1082 '''Retrieve the package name. Returns the name, or None if not found.'''
1083 for path in manifest_paths(app_dir, flavours):
1084 if not has_extension(path, 'xml') or not os.path.isfile(path):
1086 logging.debug("fetch_real_name: Checking manifest at " + path)
1087 xml = parse_xml(path)
1088 app = xml.find('application')
1091 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1093 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1094 result = retrieve_string_singleline(app_dir, label)
1096 result = result.strip()
1101 def get_library_references(root_dir):
1103 proppath = os.path.join(root_dir, 'project.properties')
1104 if not os.path.isfile(proppath):
1106 with open(proppath, 'r', encoding='iso-8859-1') as f:
1108 if not line.startswith('android.library.reference.'):
1110 path = line.split('=')[1].strip()
1111 relpath = os.path.join(root_dir, path)
1112 if not os.path.isdir(relpath):
1114 logging.debug("Found subproject at %s" % path)
1115 libraries.append(path)
1119 def ant_subprojects(root_dir):
1120 subprojects = get_library_references(root_dir)
1121 for subpath in subprojects:
1122 subrelpath = os.path.join(root_dir, subpath)
1123 for p in get_library_references(subrelpath):
1124 relp = os.path.normpath(os.path.join(subpath, p))
1125 if relp not in subprojects:
1126 subprojects.insert(0, relp)
1130 def remove_debuggable_flags(root_dir):
1131 # Remove forced debuggable flags
1132 logging.debug("Removing debuggable flags from %s" % root_dir)
1133 for root, dirs, files in os.walk(root_dir):
1134 if 'AndroidManifest.xml' in files:
1135 regsub_file(r'android:debuggable="[^"]*"',
1137 os.path.join(root, 'AndroidManifest.xml'))
1140 vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search
1141 vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1142 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1145 def app_matches_packagename(app, package):
1148 appid = app.UpdateCheckName or app.id
1149 if appid is None or appid == "Ignore":
1151 return appid == package
1154 def parse_androidmanifests(paths, app):
1156 Extract some information from the AndroidManifest.xml at the given path.
1157 Returns (version, vercode, package), any or all of which might be None.
1158 All values returned are strings.
1161 ignoreversions = app.UpdateCheckIgnore
1162 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1165 return (None, None, None)
1173 if not os.path.isfile(path):
1176 logging.debug("Parsing manifest at {0}".format(path))
1181 if has_extension(path, 'gradle'):
1182 with open(path, 'r') as f:
1184 if gradle_comment.match(line):
1186 # Grab first occurence of each to avoid running into
1187 # alternative flavours and builds.
1189 matches = psearch_g(line)
1191 s = matches.group(2)
1192 if app_matches_packagename(app, s):
1195 matches = vnsearch_g(line)
1197 version = matches.group(2)
1199 matches = vcsearch_g(line)
1201 vercode = matches.group(1)
1204 xml = parse_xml(path)
1205 if "package" in xml.attrib:
1206 s = xml.attrib["package"]
1207 if app_matches_packagename(app, s):
1209 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1210 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1211 base_dir = os.path.dirname(path)
1212 version = retrieve_string_singleline(base_dir, version)
1213 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1214 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1215 if string_is_integer(a):
1218 logging.warning("Problem with xml at {0}".format(path))
1220 # Remember package name, may be defined separately from version+vercode
1222 package = max_package
1224 logging.debug("..got package={0}, version={1}, vercode={2}"
1225 .format(package, version, vercode))
1227 # Always grab the package name and version name in case they are not
1228 # together with the highest version code
1229 if max_package is None and package is not None:
1230 max_package = package
1231 if max_version is None and version is not None:
1232 max_version = version
1234 if vercode is not None \
1235 and (max_vercode is None or vercode > max_vercode):
1236 if not ignoresearch or not ignoresearch(version):
1237 if version is not None:
1238 max_version = version
1239 if vercode is not None:
1240 max_vercode = vercode
1241 if package is not None:
1242 max_package = package
1244 max_version = "Ignore"
1246 if max_version is None:
1247 max_version = "Unknown"
1249 if max_package and not is_valid_package_name(max_package):
1250 raise FDroidException("Invalid package name {0}".format(max_package))
1252 return (max_version, max_vercode, max_package)
1255 def is_valid_package_name(name):
1256 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1259 class FDroidException(Exception):
1261 def __init__(self, value, detail=None):
1263 self.detail = detail
1265 def shortened_detail(self):
1266 if len(self.detail) < 16000:
1268 return '[...]\n' + self.detail[-16000:]
1270 def get_wikitext(self):
1271 ret = repr(self.value) + "\n"
1274 ret += "<pre>\n" + self.shortened_detail() + "</pre>\n"
1280 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1284 class VCSException(FDroidException):
1288 class BuildException(FDroidException):
1292 # Get the specified source library.
1293 # Returns the path to it. Normally this is the path to be used when referencing
1294 # it, which may be a subdirectory of the actual project. If you want the base
1295 # directory of the project, pass 'basepath=True'.
1296 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1297 raw=False, prepare=True, preponly=False, refresh=True,
1306 name, ref = spec.split('@')
1308 number, name = name.split(':', 1)
1310 name, subdir = name.split('/', 1)
1312 if name not in fdroidserver.metadata.srclibs:
1313 raise VCSException('srclib ' + name + ' not found.')
1315 srclib = fdroidserver.metadata.srclibs[name]
1317 sdir = os.path.join(srclib_dir, name)
1320 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1321 vcs.srclib = (name, number, sdir)
1323 vcs.gotorevision(ref, refresh)
1330 libdir = os.path.join(sdir, subdir)
1331 elif srclib["Subdir"]:
1332 for subdir in srclib["Subdir"]:
1333 libdir_candidate = os.path.join(sdir, subdir)
1334 if os.path.exists(libdir_candidate):
1335 libdir = libdir_candidate
1341 remove_signing_keys(sdir)
1342 remove_debuggable_flags(sdir)
1346 if srclib["Prepare"]:
1347 cmd = replace_config_vars(srclib["Prepare"], build)
1349 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1350 if p.returncode != 0:
1351 raise BuildException("Error running prepare command for srclib %s"
1357 return (name, number, libdir)
1360 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1363 # Prepare the source code for a particular build
1364 # 'vcs' - the appropriate vcs object for the application
1365 # 'app' - the application details from the metadata
1366 # 'build' - the build details from the metadata
1367 # 'build_dir' - the path to the build directory, usually
1369 # 'srclib_dir' - the path to the source libraries directory, usually
1371 # 'extlib_dir' - the path to the external libraries directory, usually
1373 # Returns the (root, srclibpaths) where:
1374 # 'root' is the root directory, which may be the same as 'build_dir' or may
1375 # be a subdirectory of it.
1376 # 'srclibpaths' is information on the srclibs being used
1377 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1379 # Optionally, the actual app source can be in a subdirectory
1381 root_dir = os.path.join(build_dir, build.subdir)
1383 root_dir = build_dir
1385 # Get a working copy of the right revision
1386 logging.info("Getting source for revision " + build.commit)
1387 vcs.gotorevision(build.commit, refresh)
1389 # Initialise submodules if required
1390 if build.submodules:
1391 logging.info("Initialising submodules")
1392 vcs.initsubmodules()
1394 # Check that a subdir (if we're using one) exists. This has to happen
1395 # after the checkout, since it might not exist elsewhere
1396 if not os.path.exists(root_dir):
1397 raise BuildException('Missing subdir ' + root_dir)
1399 # Run an init command if one is required
1401 cmd = replace_config_vars(build.init, build)
1402 logging.info("Running 'init' commands in %s" % root_dir)
1404 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1405 if p.returncode != 0:
1406 raise BuildException("Error running init command for %s:%s" %
1407 (app.id, build.versionName), p.output)
1409 # Apply patches if any
1411 logging.info("Applying patches")
1412 for patch in build.patch:
1413 patch = patch.strip()
1414 logging.info("Applying " + patch)
1415 patch_path = os.path.join('metadata', app.id, patch)
1416 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1417 if p.returncode != 0:
1418 raise BuildException("Failed to apply patch %s" % patch_path)
1420 # Get required source libraries
1423 logging.info("Collecting source libraries")
1424 for lib in build.srclibs:
1425 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1426 refresh=refresh, build=build))
1428 for name, number, libpath in srclibpaths:
1429 place_srclib(root_dir, int(number) if number else None, libpath)
1431 basesrclib = vcs.getsrclib()
1432 # If one was used for the main source, add that too.
1434 srclibpaths.append(basesrclib)
1436 # Update the local.properties file
1437 localprops = [os.path.join(build_dir, 'local.properties')]
1439 parts = build.subdir.split(os.sep)
1442 cur = os.path.join(cur, d)
1443 localprops += [os.path.join(cur, 'local.properties')]
1444 for path in localprops:
1446 if os.path.isfile(path):
1447 logging.info("Updating local.properties file at %s" % path)
1448 with open(path, 'r', encoding='iso-8859-1') as f:
1452 logging.info("Creating local.properties file at %s" % path)
1453 # Fix old-fashioned 'sdk-location' by copying
1454 # from sdk.dir, if necessary
1456 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1457 re.S | re.M).group(1)
1458 props += "sdk-location=%s\n" % sdkloc
1460 props += "sdk.dir=%s\n" % config['sdk_path']
1461 props += "sdk-location=%s\n" % config['sdk_path']
1462 ndk_path = build.ndk_path()
1463 # if for any reason the path isn't valid or the directory
1464 # doesn't exist, some versions of Gradle will error with a
1465 # cryptic message (even if the NDK is not even necessary).
1466 # https://gitlab.com/fdroid/fdroidserver/issues/171
1467 if ndk_path and os.path.exists(ndk_path):
1469 props += "ndk.dir=%s\n" % ndk_path
1470 props += "ndk-location=%s\n" % ndk_path
1471 # Add java.encoding if necessary
1473 props += "java.encoding=%s\n" % build.encoding
1474 with open(path, 'w', encoding='iso-8859-1') as f:
1478 if build.build_method() == 'gradle':
1479 flavours = build.gradle
1482 n = build.target.split('-')[1]
1483 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1484 r'compileSdkVersion %s' % n,
1485 os.path.join(root_dir, 'build.gradle'))
1487 # Remove forced debuggable flags
1488 remove_debuggable_flags(root_dir)
1490 # Insert version code and number into the manifest if necessary
1491 if build.forceversion:
1492 logging.info("Changing the version name")
1493 for path in manifest_paths(root_dir, flavours):
1494 if not os.path.isfile(path):
1496 if has_extension(path, 'xml'):
1497 regsub_file(r'android:versionName="[^"]*"',
1498 r'android:versionName="%s"' % build.versionName,
1500 elif has_extension(path, 'gradle'):
1501 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1502 r"""\1versionName '%s'""" % build.versionName,
1505 if build.forcevercode:
1506 logging.info("Changing the version code")
1507 for path in manifest_paths(root_dir, flavours):
1508 if not os.path.isfile(path):
1510 if has_extension(path, 'xml'):
1511 regsub_file(r'android:versionCode="[^"]*"',
1512 r'android:versionCode="%s"' % build.versionCode,
1514 elif has_extension(path, 'gradle'):
1515 regsub_file(r'versionCode[ =]+[0-9]+',
1516 r'versionCode %s' % build.versionCode,
1519 # Delete unwanted files
1521 logging.info("Removing specified files")
1522 for part in getpaths(build_dir, build.rm):
1523 dest = os.path.join(build_dir, part)
1524 logging.info("Removing {0}".format(part))
1525 if os.path.lexists(dest):
1526 if os.path.islink(dest):
1527 FDroidPopen(['unlink', dest], output=False)
1529 FDroidPopen(['rm', '-rf', dest], output=False)
1531 logging.info("...but it didn't exist")
1533 remove_signing_keys(build_dir)
1535 # Add required external libraries
1537 logging.info("Collecting prebuilt libraries")
1538 libsdir = os.path.join(root_dir, 'libs')
1539 if not os.path.exists(libsdir):
1541 for lib in build.extlibs:
1543 logging.info("...installing extlib {0}".format(lib))
1544 libf = os.path.basename(lib)
1545 libsrc = os.path.join(extlib_dir, lib)
1546 if not os.path.exists(libsrc):
1547 raise BuildException("Missing extlib file {0}".format(libsrc))
1548 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1550 # Run a pre-build command if one is required
1552 logging.info("Running 'prebuild' commands in %s" % root_dir)
1554 cmd = replace_config_vars(build.prebuild, build)
1556 # Substitute source library paths into prebuild commands
1557 for name, number, libpath in srclibpaths:
1558 libpath = os.path.relpath(libpath, root_dir)
1559 cmd = cmd.replace('$$' + name + '$$', libpath)
1561 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1562 if p.returncode != 0:
1563 raise BuildException("Error running prebuild command for %s:%s" %
1564 (app.id, build.versionName), p.output)
1566 # Generate (or update) the ant build file, build.xml...
1567 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1568 parms = ['android', 'update', 'lib-project']
1569 lparms = ['android', 'update', 'project']
1572 parms += ['-t', build.target]
1573 lparms += ['-t', build.target]
1574 if build.androidupdate:
1575 update_dirs = build.androidupdate
1577 update_dirs = ant_subprojects(root_dir) + ['.']
1579 for d in update_dirs:
1580 subdir = os.path.join(root_dir, d)
1582 logging.debug("Updating main project")
1583 cmd = parms + ['-p', d]
1585 logging.debug("Updating subproject %s" % d)
1586 cmd = lparms + ['-p', d]
1587 p = SdkToolsPopen(cmd, cwd=root_dir)
1588 # Check to see whether an error was returned without a proper exit
1589 # code (this is the case for the 'no target set or target invalid'
1591 if p.returncode != 0 or p.output.startswith("Error: "):
1592 raise BuildException("Failed to update project at %s" % d, p.output)
1593 # Clean update dirs via ant
1595 logging.info("Cleaning subproject %s" % d)
1596 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1598 return (root_dir, srclibpaths)
1601 # Extend via globbing the paths from a field and return them as a map from
1602 # original path to resulting paths
1603 def getpaths_map(build_dir, globpaths):
1607 full_path = os.path.join(build_dir, p)
1608 full_path = os.path.normpath(full_path)
1609 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1611 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1615 # Extend via globbing the paths from a field and return them as a set
1616 def getpaths(build_dir, globpaths):
1617 paths_map = getpaths_map(build_dir, globpaths)
1619 for k, v in paths_map.items():
1626 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1632 self.path = os.path.join('stats', 'known_apks.txt')
1634 if os.path.isfile(self.path):
1635 with open(self.path, 'r', encoding='utf8') as f:
1637 t = line.rstrip().split(' ')
1639 self.apks[t[0]] = (t[1], None)
1641 self.apks[t[0]] = (t[1], datetime.strptime(t[2], '%Y-%m-%d'))
1642 self.changed = False
1644 def writeifchanged(self):
1645 if not self.changed:
1648 if not os.path.exists('stats'):
1652 for apk, app in self.apks.items():
1654 line = apk + ' ' + appid
1656 line += ' ' + added.strftime('%Y-%m-%d')
1659 with open(self.path, 'w', encoding='utf8') as f:
1660 for line in sorted(lst, key=natural_key):
1661 f.write(line + '\n')
1663 def recordapk(self, apk, app, default_date=None):
1665 Record an apk (if it's new, otherwise does nothing)
1666 Returns the date it was added as a datetime instance
1668 if apk not in self.apks:
1669 if default_date is None:
1670 default_date = datetime.utcnow()
1671 self.apks[apk] = (app, default_date)
1673 _, added = self.apks[apk]
1676 # Look up information - given the 'apkname', returns (app id, date added/None).
1677 # Or returns None for an unknown apk.
1678 def getapp(self, apkname):
1679 if apkname in self.apks:
1680 return self.apks[apkname]
1683 # Get the most recent 'num' apps added to the repo, as a list of package ids
1684 # with the most recent first.
1685 def getlatest(self, num):
1687 for apk, app in self.apks.items():
1691 if apps[appid] > added:
1695 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1696 lst = [app for app, _ in sortedapps]
1701 def get_file_extension(filename):
1702 """get the normalized file extension, can be blank string but never None"""
1703 if isinstance(filename, bytes):
1704 filename = filename.decode('utf-8')
1705 return os.path.splitext(filename)[1].lower()[1:]
1708 def isApkAndDebuggable(apkfile, config):
1709 """Returns True if the given file is an APK and is debuggable
1711 :param apkfile: full path to the apk to check"""
1713 if get_file_extension(apkfile) != 'apk':
1716 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1718 if p.returncode != 0:
1719 logging.critical("Failed to get apk manifest information")
1721 for line in p.output.splitlines():
1722 if 'android:debuggable' in line and not line.endswith('0x0'):
1729 self.returncode = None
1733 def SdkToolsPopen(commands, cwd=None, output=True):
1735 if cmd not in config:
1736 config[cmd] = find_sdk_tools_cmd(commands[0])
1737 abscmd = config[cmd]
1739 logging.critical("Could not find '%s' on your system" % cmd)
1742 test_aapt_version(config['aapt'])
1743 return FDroidPopen([abscmd] + commands[1:],
1744 cwd=cwd, output=output)
1747 def FDroidPopenBytes(commands, cwd=None, output=True, stderr_to_stdout=True):
1749 Run a command and capture the possibly huge output as bytes.
1751 :param commands: command and argument list like in subprocess.Popen
1752 :param cwd: optionally specifies a working directory
1753 :returns: A PopenResult.
1758 set_FDroidPopen_env()
1761 cwd = os.path.normpath(cwd)
1762 logging.debug("Directory: %s" % cwd)
1763 logging.debug("> %s" % ' '.join(commands))
1765 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
1766 result = PopenResult()
1769 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
1770 stdout=subprocess.PIPE, stderr=stderr_param)
1771 except OSError as e:
1772 raise BuildException("OSError while trying to execute " +
1773 ' '.join(commands) + ': ' + str(e))
1775 if not stderr_to_stdout and options.verbose:
1776 stderr_queue = Queue()
1777 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
1779 while not stderr_reader.eof():
1780 while not stderr_queue.empty():
1781 line = stderr_queue.get()
1782 sys.stderr.buffer.write(line)
1787 stdout_queue = Queue()
1788 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1791 # Check the queue for output (until there is no more to get)
1792 while not stdout_reader.eof():
1793 while not stdout_queue.empty():
1794 line = stdout_queue.get()
1795 if output and options.verbose:
1796 # Output directly to console
1797 sys.stderr.buffer.write(line)
1803 result.returncode = p.wait()
1804 result.output = buf.getvalue()
1809 def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
1811 Run a command and capture the possibly huge output as a str.
1813 :param commands: command and argument list like in subprocess.Popen
1814 :param cwd: optionally specifies a working directory
1815 :returns: A PopenResult.
1817 result = FDroidPopenBytes(commands, cwd, output, stderr_to_stdout)
1818 result.output = result.output.decode('utf-8', 'ignore')
1822 gradle_comment = re.compile(r'[ ]*//')
1823 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1824 gradle_line_matches = [
1825 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1826 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1827 re.compile(r'.*\.readLine\(.*'),
1831 def remove_signing_keys(build_dir):
1832 for root, dirs, files in os.walk(build_dir):
1833 if 'build.gradle' in files:
1834 path = os.path.join(root, 'build.gradle')
1836 with open(path, "r", encoding='utf8') as o:
1837 lines = o.readlines()
1843 with open(path, "w", encoding='utf8') as o:
1844 while i < len(lines):
1847 while line.endswith('\\\n'):
1848 line = line.rstrip('\\\n') + lines[i]
1851 if gradle_comment.match(line):
1856 opened += line.count('{')
1857 opened -= line.count('}')
1860 if gradle_signing_configs.match(line):
1865 if any(s.match(line) for s in gradle_line_matches):
1873 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1876 'project.properties',
1878 'default.properties',
1879 'ant.properties', ]:
1880 if propfile in files:
1881 path = os.path.join(root, propfile)
1883 with open(path, "r", encoding='iso-8859-1') as o:
1884 lines = o.readlines()
1888 with open(path, "w", encoding='iso-8859-1') as o:
1890 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1897 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1900 def set_FDroidPopen_env(build=None):
1902 set up the environment variables for the build environment
1904 There is only a weak standard, the variables used by gradle, so also set
1905 up the most commonly used environment variables for SDK and NDK. Also, if
1906 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
1908 global env, orig_path
1912 orig_path = env['PATH']
1913 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
1914 env[n] = config['sdk_path']
1915 for k, v in config['java_paths'].items():
1916 env['JAVA%s_HOME' % k] = v
1918 missinglocale = True
1919 for k, v in env.items():
1920 if k == 'LANG' and v != 'C':
1921 missinglocale = False
1923 missinglocale = False
1925 env['LANG'] = 'en_US.UTF-8'
1927 if build is not None:
1928 path = build.ndk_path()
1929 paths = orig_path.split(os.pathsep)
1930 if path not in paths:
1931 paths = [path] + paths
1932 env['PATH'] = os.pathsep.join(paths)
1933 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
1934 env[n] = build.ndk_path()
1937 def replace_build_vars(cmd, build):
1938 cmd = cmd.replace('$$COMMIT$$', build.commit)
1939 cmd = cmd.replace('$$VERSION$$', build.versionName)
1940 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
1944 def replace_config_vars(cmd, build):
1945 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1946 cmd = cmd.replace('$$NDK$$', build.ndk_path())
1947 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1948 cmd = cmd.replace('$$QT$$', config['qt_sdk_path'] or '')
1949 if build is not None:
1950 cmd = replace_build_vars(cmd, build)
1954 def place_srclib(root_dir, number, libpath):
1957 relpath = os.path.relpath(libpath, root_dir)
1958 proppath = os.path.join(root_dir, 'project.properties')
1961 if os.path.isfile(proppath):
1962 with open(proppath, "r", encoding='iso-8859-1') as o:
1963 lines = o.readlines()
1965 with open(proppath, "w", encoding='iso-8859-1') as o:
1968 if line.startswith('android.library.reference.%d=' % number):
1969 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1974 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1977 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
1980 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
1981 """Verify that two apks are the same
1983 One of the inputs is signed, the other is unsigned. The signature metadata
1984 is transferred from the signed to the unsigned apk, and then jarsigner is
1985 used to verify that the signature from the signed apk is also varlid for
1986 the unsigned one. If the APK given as unsigned actually does have a
1987 signature, it will be stripped out and ignored.
1989 There are two SHA1 git commit IDs that fdroidserver includes in the builds
1990 it makes: fdroidserverid and buildserverid. Originally, these were inserted
1991 into AndroidManifest.xml, but that makes the build not reproducible. So
1992 instead they are included as separate files in the APK's META-INF/ folder.
1993 If those files exist in the signed APK, they will be part of the signature
1994 and need to also be included in the unsigned APK for it to validate.
1996 :param signed_apk: Path to a signed apk file
1997 :param unsigned_apk: Path to an unsigned apk file expected to match it
1998 :param tmp_dir: Path to directory for temporary files
1999 :returns: None if the verification is successful, otherwise a string
2000 describing what went wrong.
2003 signed = ZipFile(signed_apk, 'r')
2004 meta_inf_files = ['META-INF/MANIFEST.MF']
2005 for f in signed.namelist():
2006 if apk_sigfile.match(f) \
2007 or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
2008 meta_inf_files.append(f)
2009 if len(meta_inf_files) < 3:
2010 return "Signature files missing from {0}".format(signed_apk)
2012 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2013 unsigned = ZipFile(unsigned_apk, 'r')
2014 # only read the signature from the signed APK, everything else from unsigned
2015 with ZipFile(tmp_apk, 'w') as tmp:
2016 for filename in meta_inf_files:
2017 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2018 for info in unsigned.infolist():
2019 if info.filename in meta_inf_files:
2020 logging.warning('Ignoring ' + info.filename + ' from ' + unsigned_apk)
2022 if info.filename in tmp.namelist():
2023 return "duplicate filename found: " + info.filename
2024 tmp.writestr(info, unsigned.read(info.filename))
2028 verified = verify_apk_signature(tmp_apk)
2031 logging.info("...NOT verified - {0}".format(tmp_apk))
2032 return compare_apks(signed_apk, tmp_apk, tmp_dir, os.path.dirname(unsigned_apk))
2034 logging.info("...successfully verified")
2038 def verify_apk_signature(apk, jar=False):
2039 """verify the signature on an APK
2041 Try to use apksigner whenever possible since jarsigner is very
2042 shitty: unsigned APKs pass as "verified"! So this has to turn on
2043 -strict then check for result 4.
2045 You can set :param: jar to True if you want to use this method
2046 to verify jar signatures.
2048 if set_command_in_config('apksigner'):
2049 args = [config['apksigner'], 'verify']
2051 args += ['--min-sdk-version=1']
2052 return subprocess.call(args + [apk]) == 0
2054 logging.warning("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner")
2055 return subprocess.call([config['jarsigner'], '-strict', '-verify', apk]) == 4
2058 apk_badchars = re.compile('''[/ :;'"]''')
2061 def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
2064 Returns None if the apk content is the same (apart from the signing key),
2065 otherwise a string describing what's different, or what went wrong when
2066 trying to do the comparison.
2072 absapk1 = os.path.abspath(apk1)
2073 absapk2 = os.path.abspath(apk2)
2075 if set_command_in_config('diffoscope'):
2076 logfilename = os.path.join(log_dir, os.path.basename(absapk1))
2077 htmlfile = logfilename + '.diffoscope.html'
2078 textfile = logfilename + '.diffoscope.txt'
2079 if subprocess.call([config['diffoscope'],
2080 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2081 '--html', htmlfile, '--text', textfile,
2082 absapk1, absapk2]) != 0:
2083 return("Failed to unpack " + apk1)
2085 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2086 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2087 for d in [apk1dir, apk2dir]:
2088 if os.path.exists(d):
2091 os.mkdir(os.path.join(d, 'jar-xf'))
2093 if subprocess.call(['jar', 'xf',
2094 os.path.abspath(apk1)],
2095 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2096 return("Failed to unpack " + apk1)
2097 if subprocess.call(['jar', 'xf',
2098 os.path.abspath(apk2)],
2099 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2100 return("Failed to unpack " + apk2)
2102 if set_command_in_config('apktool'):
2103 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2105 return("Failed to unpack " + apk1)
2106 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2108 return("Failed to unpack " + apk2)
2110 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2111 lines = p.output.splitlines()
2112 if len(lines) != 1 or 'META-INF' not in lines[0]:
2113 meld = find_command('meld')
2114 if meld is not None:
2115 p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
2116 return("Unexpected diff output - " + p.output)
2118 # since everything verifies, delete the comparison to keep cruft down
2119 shutil.rmtree(apk1dir)
2120 shutil.rmtree(apk2dir)
2122 # If we get here, it seems like they're the same!
2126 def set_command_in_config(command):
2127 '''Try to find specified command in the path, if it hasn't been
2128 manually set in config.py. If found, it is added to the config
2129 dict. The return value says whether the command is available.
2132 if command in config:
2135 tmp = find_command(command)
2137 config[command] = tmp
2142 def find_command(command):
2143 '''find the full path of a command, or None if it can't be found in the PATH'''
2146 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2148 fpath, fname = os.path.split(command)
2153 for path in os.environ["PATH"].split(os.pathsep):
2154 path = path.strip('"')
2155 exe_file = os.path.join(path, command)
2156 if is_exe(exe_file):
2163 '''generate a random password for when generating keys'''
2164 h = hashlib.sha256()
2165 h.update(os.urandom(16)) # salt
2166 h.update(socket.getfqdn().encode('utf-8'))
2167 passwd = base64.b64encode(h.digest()).strip()
2168 return passwd.decode('utf-8')
2171 def genkeystore(localconfig):
2173 Generate a new key with password provided in :param localconfig and add it to new keystore
2174 :return: hexed public key, public key fingerprint
2176 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2177 keystoredir = os.path.dirname(localconfig['keystore'])
2178 if keystoredir is None or keystoredir == '':
2179 keystoredir = os.path.join(os.getcwd(), keystoredir)
2180 if not os.path.exists(keystoredir):
2181 os.makedirs(keystoredir, mode=0o700)
2183 write_password_file("keystorepass", localconfig['keystorepass'])
2184 write_password_file("keypass", localconfig['keypass'])
2185 p = FDroidPopen([config['keytool'], '-genkey',
2186 '-keystore', localconfig['keystore'],
2187 '-alias', localconfig['repo_keyalias'],
2188 '-keyalg', 'RSA', '-keysize', '4096',
2189 '-sigalg', 'SHA256withRSA',
2190 '-validity', '10000',
2191 '-storepass:file', config['keystorepassfile'],
2192 '-keypass:file', config['keypassfile'],
2193 '-dname', localconfig['keydname']])
2194 # TODO keypass should be sent via stdin
2195 if p.returncode != 0:
2196 raise BuildException("Failed to generate key", p.output)
2197 os.chmod(localconfig['keystore'], 0o0600)
2198 if not options.quiet:
2199 # now show the lovely key that was just generated
2200 p = FDroidPopen([config['keytool'], '-list', '-v',
2201 '-keystore', localconfig['keystore'],
2202 '-alias', localconfig['repo_keyalias'],
2203 '-storepass:file', config['keystorepassfile']])
2204 logging.info(p.output.strip() + '\n\n')
2205 # get the public key
2206 p = FDroidPopenBytes([config['keytool'], '-exportcert',
2207 '-keystore', localconfig['keystore'],
2208 '-alias', localconfig['repo_keyalias'],
2209 '-storepass:file', config['keystorepassfile']]
2210 + config['smartcardoptions'],
2211 output=False, stderr_to_stdout=False)
2212 if p.returncode != 0 or len(p.output) < 20:
2213 raise BuildException("Failed to get public key", p.output)
2215 fingerprint = get_cert_fingerprint(pubkey)
2216 return hexlify(pubkey), fingerprint
2219 def get_cert_fingerprint(pubkey):
2221 Generate a certificate fingerprint the same way keytool does it
2222 (but with slightly different formatting)
2224 digest = hashlib.sha256(pubkey).digest()
2225 ret = [' '.join("%02X" % b for b in bytearray(digest))]
2226 return " ".join(ret)
2229 def get_certificate(certificate_file):
2231 Extracts a certificate from the given file.
2232 :param certificate_file: file bytes (as string) representing the certificate
2233 :return: A binary representation of the certificate's public key, or None in case of error
2235 content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
2236 if content.getComponentByName('contentType') != rfc2315.signedData:
2238 content = decoder.decode(content.getComponentByName('content'),
2239 asn1Spec=rfc2315.SignedData())[0]
2241 certificates = content.getComponentByName('certificates')
2242 cert = certificates[0].getComponentByName('certificate')
2244 logging.error("Certificates not found.")
2246 return encoder.encode(cert)
2249 def write_to_config(thisconfig, key, value=None, config_file=None):
2250 '''write a key/value to the local config.py
2252 NOTE: only supports writing string variables.
2254 :param thisconfig: config dictionary
2255 :param key: variable name in config.py to be overwritten/added
2256 :param value: optional value to be written, instead of fetched
2257 from 'thisconfig' dictionary.
2260 origkey = key + '_orig'
2261 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2262 cfg = config_file if config_file else 'config.py'
2265 with open(cfg, 'r', encoding="utf-8") as f:
2266 lines = f.readlines()
2268 # make sure the file ends with a carraige return
2270 if not lines[-1].endswith('\n'):
2273 # regex for finding and replacing python string variable
2274 # definitions/initializations
2275 pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
2276 repl = key + ' = "' + value + '"'
2277 pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
2278 repl2 = key + " = '" + value + "'"
2280 # If we replaced this line once, we make sure won't be a
2281 # second instance of this line for this key in the document.
2284 with open(cfg, 'w', encoding="utf-8") as f:
2286 if pattern.match(line) or pattern2.match(line):
2288 line = pattern.sub(repl, line)
2289 line = pattern2.sub(repl2, line)
2300 def parse_xml(path):
2301 return XMLElementTree.parse(path).getroot()
2304 def string_is_integer(string):
2312 def get_per_app_repos():
2313 '''per-app repos are dirs named with the packageName of a single app'''
2315 # Android packageNames are Java packages, they may contain uppercase or
2316 # lowercase letters ('A' through 'Z'), numbers, and underscores
2317 # ('_'). However, individual package name parts may only start with
2318 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2319 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2322 for root, dirs, files in os.walk(os.getcwd()):
2324 print('checking', root, 'for', d)
2325 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2326 # standard parts of an fdroid repo, so never packageNames
2329 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
2335 def is_repo_file(filename):
2336 '''Whether the file in a repo is a build product to be delivered to users'''
2337 if isinstance(filename, str):
2338 filename = filename.encode('utf-8', errors="surrogateescape")
2339 return os.path.isfile(filename) \
2340 and not filename.endswith(b'.asc') \
2341 and not filename.endswith(b'.sig') \
2342 and os.path.basename(filename) not in [
2344 b'index_unsigned.jar',