3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
38 import xml.etree.ElementTree as XMLElementTree
40 from datetime import datetime
41 from distutils.version import LooseVersion
42 from queue import Queue
43 from zipfile import ZipFile
45 import fdroidserver.metadata
46 from .asynchronousfilereader import AsynchronousFileReader
49 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
58 'sdk_path': "$ANDROID_HOME",
63 'r12b': "$ANDROID_NDK",
67 'build_tools': "25.0.2",
68 'force_build_tools': False,
73 'accepted_formats': ['txt', 'yml'],
74 'sync_from_local_copy_dir': False,
75 'per_app_repos': False,
76 'make_current_version_link': True,
77 'current_version_name_source': 'Name',
78 'update_stats': False,
82 'stats_to_carbon': False,
84 'build_server_always': False,
85 'keystore': 'keystore.jks',
86 'smartcardoptions': [],
92 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
93 'repo_name': "My First FDroid Repo Demo",
94 'repo_icon': "fdroid-icon.png",
95 'repo_description': '''
96 This is a repository of apps to be used with FDroid. Applications in this
97 repository are either official binaries built by the original application
98 developers, or are binaries built from source by the admin of f-droid.org
99 using the tools on https://gitlab.com/u/fdroid.
105 def setup_global_opts(parser):
106 parser.add_argument("-v", "--verbose", action="store_true", default=False,
107 help="Spew out even more information than normal")
108 parser.add_argument("-q", "--quiet", action="store_true", default=False,
109 help="Restrict output to warnings and errors")
112 def fill_config_defaults(thisconfig):
113 for k, v in default_config.items():
114 if k not in thisconfig:
117 # Expand paths (~users and $vars)
118 def expand_path(path):
122 path = os.path.expanduser(path)
123 path = os.path.expandvars(path)
128 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
133 thisconfig[k + '_orig'] = v
135 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
136 if thisconfig['java_paths'] is None:
137 thisconfig['java_paths'] = dict()
139 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
140 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
141 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
142 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
143 if os.getenv('JAVA_HOME') is not None:
144 pathlist.append(os.getenv('JAVA_HOME'))
145 if os.getenv('PROGRAMFILES') is not None:
146 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
147 for d in sorted(pathlist):
148 if os.path.islink(d):
150 j = os.path.basename(d)
151 # the last one found will be the canonical one, so order appropriately
153 r'^1\.([6-9])\.0\.jdk$', # OSX
154 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
155 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
156 r'^jdk([6-9])-openjdk$', # Arch
157 r'^java-([6-9])-openjdk$', # Arch
158 r'^java-([6-9])-jdk$', # Arch (oracle)
159 r'^java-1\.([6-9])\.0-.*$', # RedHat
160 r'^java-([6-9])-oracle$', # Debian WebUpd8
161 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
162 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
164 m = re.match(regex, j)
167 for p in [d, os.path.join(d, 'Contents', 'Home')]:
168 if os.path.exists(os.path.join(p, 'bin', 'javac')):
169 thisconfig['java_paths'][m.group(1)] = p
171 for java_version in ('7', '8', '9'):
172 if java_version not in thisconfig['java_paths']:
174 java_home = thisconfig['java_paths'][java_version]
175 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
176 if os.path.exists(jarsigner):
177 thisconfig['jarsigner'] = jarsigner
178 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
179 break # Java7 is preferred, so quit if found
181 for k in ['ndk_paths', 'java_paths']:
187 thisconfig[k][k2] = exp
188 thisconfig[k][k2 + '_orig'] = v
191 def regsub_file(pattern, repl, path):
192 with open(path, 'rb') as f:
194 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
195 with open(path, 'wb') as f:
199 def read_config(opts, config_file='config.py'):
200 """Read the repository config
202 The config is read from config_file, which is in the current
203 directory when any of the repo management commands are used. If
204 there is a local metadata file in the git repo, then config.py is
205 not required, just use defaults.
208 global config, options
210 if config is not None:
217 if os.path.isfile(config_file):
218 logging.debug("Reading %s" % config_file)
219 with io.open(config_file, "rb") as f:
220 code = compile(f.read(), config_file, 'exec')
221 exec(code, None, config)
222 elif len(get_local_metadata_files()) == 0:
223 logging.critical("Missing config file - is this a repo directory?")
226 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
228 if not type(config[k]) in (str, list, tuple):
229 logging.warn('"' + k + '" will be in random order!'
230 + ' Use () or [] brackets if order is important!')
232 # smartcardoptions must be a list since its command line args for Popen
233 if 'smartcardoptions' in config:
234 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
235 elif 'keystore' in config and config['keystore'] == 'NONE':
236 # keystore='NONE' means use smartcard, these are required defaults
237 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
238 'SunPKCS11-OpenSC', '-providerClass',
239 'sun.security.pkcs11.SunPKCS11',
240 '-providerArg', 'opensc-fdroid.cfg']
242 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
243 st = os.stat(config_file)
244 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
245 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
247 fill_config_defaults(config)
249 for k in ["keystorepass", "keypass"]:
251 write_password_file(k)
253 for k in ["repo_description", "archive_description"]:
255 config[k] = clean_description(config[k])
257 if 'serverwebroot' in config:
258 if isinstance(config['serverwebroot'], str):
259 roots = [config['serverwebroot']]
260 elif all(isinstance(item, str) for item in config['serverwebroot']):
261 roots = config['serverwebroot']
263 raise TypeError('only accepts strings, lists, and tuples')
265 for rootstr in roots:
266 # since this is used with rsync, where trailing slashes have
267 # meaning, ensure there is always a trailing slash
268 if rootstr[-1] != '/':
270 rootlist.append(rootstr.replace('//', '/'))
271 config['serverwebroot'] = rootlist
273 if 'servergitmirrors' in config:
274 if isinstance(config['servergitmirrors'], str):
275 roots = [config['servergitmirrors']]
276 elif all(isinstance(item, str) for item in config['servergitmirrors']):
277 roots = config['servergitmirrors']
279 raise TypeError('only accepts strings, lists, and tuples')
280 config['servergitmirrors'] = roots
285 def find_sdk_tools_cmd(cmd):
286 '''find a working path to a tool from the Android SDK'''
289 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
290 # try to find a working path to this command, in all the recent possible paths
291 if 'build_tools' in config:
292 build_tools = os.path.join(config['sdk_path'], 'build-tools')
293 # if 'build_tools' was manually set and exists, check only that one
294 configed_build_tools = os.path.join(build_tools, config['build_tools'])
295 if os.path.exists(configed_build_tools):
296 tooldirs.append(configed_build_tools)
298 # no configed version, so hunt known paths for it
299 for f in sorted(os.listdir(build_tools), reverse=True):
300 if os.path.isdir(os.path.join(build_tools, f)):
301 tooldirs.append(os.path.join(build_tools, f))
302 tooldirs.append(build_tools)
303 sdk_tools = os.path.join(config['sdk_path'], 'tools')
304 if os.path.exists(sdk_tools):
305 tooldirs.append(sdk_tools)
306 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
307 if os.path.exists(sdk_platform_tools):
308 tooldirs.append(sdk_platform_tools)
309 tooldirs.append('/usr/bin')
311 path = os.path.join(d, cmd)
312 if os.path.isfile(path):
314 test_aapt_version(path)
316 # did not find the command, exit with error message
317 ensure_build_tools_exists(config)
320 def test_aapt_version(aapt):
321 '''Check whether the version of aapt is new enough'''
322 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
323 if output is None or output == '':
324 logging.error(aapt + ' failed to execute!')
326 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
331 # the Debian package has the version string like "v0.2-23.0.2"
332 if '.' not in bugfix and LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.2166767'):
333 logging.warning(aapt + ' is too old, fdroid requires build-tools-23.0.0 or newer!')
335 logging.warning('Unknown version of aapt, might cause problems: ' + output)
338 def test_sdk_exists(thisconfig):
339 if 'sdk_path' not in thisconfig:
340 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
341 test_aapt_version(thisconfig['aapt'])
344 logging.error("'sdk_path' not set in config.py!")
346 if thisconfig['sdk_path'] == default_config['sdk_path']:
347 logging.error('No Android SDK found!')
348 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
349 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
351 if not os.path.exists(thisconfig['sdk_path']):
352 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
354 if not os.path.isdir(thisconfig['sdk_path']):
355 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
357 for d in ['build-tools', 'platform-tools', 'tools']:
358 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
359 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
360 thisconfig['sdk_path'], d))
365 def ensure_build_tools_exists(thisconfig):
366 if not test_sdk_exists(thisconfig):
368 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
369 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
370 if not os.path.isdir(versioned_build_tools):
371 logging.critical('Android Build Tools path "'
372 + versioned_build_tools + '" does not exist!')
376 def write_password_file(pwtype, password=None):
378 writes out passwords to a protected file instead of passing passwords as
379 command line argments
381 filename = '.fdroid.' + pwtype + '.txt'
382 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
384 os.write(fd, config[pwtype].encode('utf-8'))
386 os.write(fd, password.encode('utf-8'))
388 config[pwtype + 'file'] = filename
393 sign a JAR file with Java's jarsigner.
395 This does use old hashing algorithms, i.e. SHA1, but that's not
396 broken yet for file verification. This could be set to SHA256,
397 but then Android < 4.3 would not be able to verify it.
398 https://code.google.com/p/android/issues/detail?id=38321
400 args = [config['jarsigner'], '-keystore', config['keystore'],
401 '-storepass:file', config['keystorepassfile'],
402 '-digestalg', 'SHA1', '-sigalg', 'SHA1withRSA',
403 jar, config['repo_keyalias']]
404 if config['keystore'] == 'NONE':
405 args += config['smartcardoptions']
406 else: # smardcards never use -keypass
407 args += ['-keypass:file', config['keypassfile']]
408 p = FDroidPopen(args)
409 if p.returncode != 0:
410 logging.critical("Failed to sign %s!" % jar)
414 def sign_index_v1(repodir, json_name):
416 sign index-v1.json to make index-v1.jar
418 This is a bit different than index.jar: instead of their being index.xml
419 and index_unsigned.jar, the presense of index-v1.json means that there is
420 unsigned data. That file is then stuck into a jar and signed by the
421 signing process. index-v1.json is never published to the repo. It is
422 included in the binary transparency log, if that is enabled.
424 name, ext = get_extension(json_name)
425 index_file = os.path.join(repodir, json_name)
426 jar_file = os.path.join(repodir, name + '.jar')
427 with zipfile.ZipFile(jar_file, 'w', zipfile.ZIP_DEFLATED) as jar:
428 jar.write(index_file, json_name)
432 def get_local_metadata_files():
433 '''get any metadata files local to an app's source repo
435 This tries to ignore anything that does not count as app metdata,
436 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
439 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
442 def read_pkg_args(args, allow_vercodes=False):
444 Given the arguments in the form of multiple appid:[vc] strings, this returns
445 a dictionary with the set of vercodes specified for each package.
453 if allow_vercodes and ':' in p:
454 package, vercode = p.split(':')
456 package, vercode = p, None
457 if package not in vercodes:
458 vercodes[package] = [vercode] if vercode else []
460 elif vercode and vercode not in vercodes[package]:
461 vercodes[package] += [vercode] if vercode else []
466 def read_app_args(args, allapps, allow_vercodes=False):
468 On top of what read_pkg_args does, this returns the whole app metadata, but
469 limiting the builds list to the builds matching the vercodes specified.
472 vercodes = read_pkg_args(args, allow_vercodes)
478 for appid, app in allapps.items():
479 if appid in vercodes:
482 if len(apps) != len(vercodes):
485 logging.critical("No such package: %s" % p)
486 raise FDroidException("Found invalid app ids in arguments")
488 raise FDroidException("No packages specified")
491 for appid, app in apps.items():
495 app.builds = [b for b in app.builds if b.versionCode in vc]
496 if len(app.builds) != len(vercodes[appid]):
498 allvcs = [b.versionCode for b in app.builds]
499 for v in vercodes[appid]:
501 logging.critical("No such vercode %s for app %s" % (v, appid))
504 raise FDroidException("Found invalid vercodes for some apps")
509 def get_extension(filename):
510 base, ext = os.path.splitext(filename)
513 return base, ext.lower()[1:]
516 def has_extension(filename, ext):
517 _, f_ext = get_extension(filename)
521 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
524 def clean_description(description):
525 'Remove unneeded newlines and spaces from a block of description text'
527 # this is split up by paragraph to make removing the newlines easier
528 for paragraph in re.split(r'\n\n', description):
529 paragraph = re.sub('\r', '', paragraph)
530 paragraph = re.sub('\n', ' ', paragraph)
531 paragraph = re.sub(' {2,}', ' ', paragraph)
532 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
533 returnstring += paragraph + '\n\n'
534 return returnstring.rstrip('\n')
537 def publishednameinfo(filename):
538 filename = os.path.basename(filename)
539 m = publish_name_regex.match(filename)
541 result = (m.group(1), m.group(2))
542 except AttributeError:
543 raise FDroidException("Invalid name for published file: %s" % filename)
547 def get_release_filename(app, build):
549 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
551 return "%s_%s.apk" % (app.id, build.versionCode)
554 def getsrcname(app, build):
555 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
567 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
570 def get_build_dir(app):
571 '''get the dir that this app will be built in'''
573 if app.RepoType == 'srclib':
574 return os.path.join('build', 'srclib', app.Repo)
576 return os.path.join('build', app.id)
580 '''checkout code from VCS and return instance of vcs and the build dir'''
581 build_dir = get_build_dir(app)
583 # Set up vcs interface and make sure we have the latest code...
584 logging.debug("Getting {0} vcs interface for {1}"
585 .format(app.RepoType, app.Repo))
586 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
590 vcs = getvcs(app.RepoType, remote, build_dir)
592 return vcs, build_dir
595 def getvcs(vcstype, remote, local):
597 return vcs_git(remote, local)
598 if vcstype == 'git-svn':
599 return vcs_gitsvn(remote, local)
601 return vcs_hg(remote, local)
603 return vcs_bzr(remote, local)
604 if vcstype == 'srclib':
605 if local != os.path.join('build', 'srclib', remote):
606 raise VCSException("Error: srclib paths are hard-coded!")
607 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
609 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
610 raise VCSException("Invalid vcs type " + vcstype)
613 def getsrclibvcs(name):
614 if name not in fdroidserver.metadata.srclibs:
615 raise VCSException("Missing srclib " + name)
616 return fdroidserver.metadata.srclibs[name]['Repo Type']
621 def __init__(self, remote, local):
623 # svn, git-svn and bzr may require auth
625 if self.repotype() in ('git-svn', 'bzr'):
627 if self.repotype == 'git-svn':
628 raise VCSException("Authentication is not supported for git-svn")
629 self.username, remote = remote.split('@')
630 if ':' not in self.username:
631 raise VCSException("Password required with username")
632 self.username, self.password = self.username.split(':')
636 self.clone_failed = False
637 self.refreshed = False
643 # Take the local repository to a clean version of the given revision, which
644 # is specificed in the VCS's native format. Beforehand, the repository can
645 # be dirty, or even non-existent. If the repository does already exist
646 # locally, it will be updated from the origin, but only once in the
647 # lifetime of the vcs object.
648 # None is acceptable for 'rev' if you know you are cloning a clean copy of
649 # the repo - otherwise it must specify a valid revision.
650 def gotorevision(self, rev, refresh=True):
652 if self.clone_failed:
653 raise VCSException("Downloading the repository already failed once, not trying again.")
655 # The .fdroidvcs-id file for a repo tells us what VCS type
656 # and remote that directory was created from, allowing us to drop it
657 # automatically if either of those things changes.
658 fdpath = os.path.join(self.local, '..',
659 '.fdroidvcs-' + os.path.basename(self.local))
660 fdpath = os.path.normpath(fdpath)
661 cdata = self.repotype() + ' ' + self.remote
664 if os.path.exists(self.local):
665 if os.path.exists(fdpath):
666 with open(fdpath, 'r') as f:
667 fsdata = f.read().strip()
672 logging.info("Repository details for %s changed - deleting" % (
676 logging.info("Repository details for %s missing - deleting" % (
679 shutil.rmtree(self.local)
683 self.refreshed = True
686 self.gotorevisionx(rev)
687 except FDroidException as e:
690 # If necessary, write the .fdroidvcs file.
691 if writeback and not self.clone_failed:
692 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
693 with open(fdpath, 'w+') as f:
699 # Derived classes need to implement this. It's called once basic checking
700 # has been performend.
701 def gotorevisionx(self, rev):
702 raise VCSException("This VCS type doesn't define gotorevisionx")
704 # Initialise and update submodules
705 def initsubmodules(self):
706 raise VCSException('Submodules not supported for this vcs type')
708 # Get a list of all known tags
710 if not self._gettags:
711 raise VCSException('gettags not supported for this vcs type')
713 for tag in self._gettags():
714 if re.match('[-A-Za-z0-9_. /]+$', tag):
718 # Get a list of all the known tags, sorted from newest to oldest
719 def latesttags(self):
720 raise VCSException('latesttags not supported for this vcs type')
722 # Get current commit reference (hash, revision, etc)
724 raise VCSException('getref not supported for this vcs type')
726 # Returns the srclib (name, path) used in setting up the current
737 # If the local directory exists, but is somehow not a git repository, git
738 # will traverse up the directory tree until it finds one that is (i.e.
739 # fdroidserver) and then we'll proceed to destroy it! This is called as
742 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
743 result = p.output.rstrip()
744 if not result.endswith(self.local):
745 raise VCSException('Repository mismatch')
747 def gotorevisionx(self, rev):
748 if not os.path.exists(self.local):
750 p = FDroidPopen(['git', 'clone', self.remote, self.local])
751 if p.returncode != 0:
752 self.clone_failed = True
753 raise VCSException("Git clone failed", p.output)
757 # Discard any working tree changes
758 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
759 'git', 'reset', '--hard'], cwd=self.local, output=False)
760 if p.returncode != 0:
761 raise VCSException("Git reset failed", p.output)
762 # Remove untracked files now, in case they're tracked in the target
763 # revision (it happens!)
764 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
765 'git', 'clean', '-dffx'], cwd=self.local, output=False)
766 if p.returncode != 0:
767 raise VCSException("Git clean failed", p.output)
768 if not self.refreshed:
769 # Get latest commits and tags from remote
770 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
771 if p.returncode != 0:
772 raise VCSException("Git fetch failed", p.output)
773 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
774 if p.returncode != 0:
775 raise VCSException("Git fetch failed", p.output)
776 # Recreate origin/HEAD as git clone would do it, in case it disappeared
777 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
778 if p.returncode != 0:
779 lines = p.output.splitlines()
780 if 'Multiple remote HEAD branches' not in lines[0]:
781 raise VCSException("Git remote set-head failed", p.output)
782 branch = lines[1].split(' ')[-1]
783 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
784 if p2.returncode != 0:
785 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
786 self.refreshed = True
787 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
788 # a github repo. Most of the time this is the same as origin/master.
789 rev = rev or 'origin/HEAD'
790 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
791 if p.returncode != 0:
792 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
793 # Get rid of any uncontrolled files left behind
794 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
795 if p.returncode != 0:
796 raise VCSException("Git clean failed", p.output)
798 def initsubmodules(self):
800 submfile = os.path.join(self.local, '.gitmodules')
801 if not os.path.isfile(submfile):
802 raise VCSException("No git submodules available")
804 # fix submodules not accessible without an account and public key auth
805 with open(submfile, 'r') as f:
806 lines = f.readlines()
807 with open(submfile, 'w') as f:
809 if 'git@github.com' in line:
810 line = line.replace('git@github.com:', 'https://github.com/')
811 if 'git@gitlab.com' in line:
812 line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
815 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
816 if p.returncode != 0:
817 raise VCSException("Git submodule sync failed", p.output)
818 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
819 if p.returncode != 0:
820 raise VCSException("Git submodule update failed", p.output)
824 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
825 return p.output.splitlines()
827 tag_format = re.compile(r'tag: ([^),]*)')
829 def latesttags(self):
831 p = FDroidPopen(['git', 'log', '--tags',
832 '--simplify-by-decoration', '--pretty=format:%d'],
833 cwd=self.local, output=False)
835 for line in p.output.splitlines():
836 for tag in self.tag_format.findall(line):
841 class vcs_gitsvn(vcs):
846 # If the local directory exists, but is somehow not a git repository, git
847 # will traverse up the directory tree until it finds one that is (i.e.
848 # fdroidserver) and then we'll proceed to destory it! This is called as
851 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
852 result = p.output.rstrip()
853 if not result.endswith(self.local):
854 raise VCSException('Repository mismatch')
856 def gotorevisionx(self, rev):
857 if not os.path.exists(self.local):
859 gitsvn_args = ['git', 'svn', 'clone']
860 if ';' in self.remote:
861 remote_split = self.remote.split(';')
862 for i in remote_split[1:]:
863 if i.startswith('trunk='):
864 gitsvn_args.extend(['-T', i[6:]])
865 elif i.startswith('tags='):
866 gitsvn_args.extend(['-t', i[5:]])
867 elif i.startswith('branches='):
868 gitsvn_args.extend(['-b', i[9:]])
869 gitsvn_args.extend([remote_split[0], self.local])
870 p = FDroidPopen(gitsvn_args, output=False)
871 if p.returncode != 0:
872 self.clone_failed = True
873 raise VCSException("Git svn clone failed", p.output)
875 gitsvn_args.extend([self.remote, self.local])
876 p = FDroidPopen(gitsvn_args, output=False)
877 if p.returncode != 0:
878 self.clone_failed = True
879 raise VCSException("Git svn clone failed", p.output)
883 # Discard any working tree changes
884 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
885 if p.returncode != 0:
886 raise VCSException("Git reset failed", p.output)
887 # Remove untracked files now, in case they're tracked in the target
888 # revision (it happens!)
889 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
890 if p.returncode != 0:
891 raise VCSException("Git clean failed", p.output)
892 if not self.refreshed:
893 # Get new commits, branches and tags from repo
894 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
895 if p.returncode != 0:
896 raise VCSException("Git svn fetch failed")
897 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
898 if p.returncode != 0:
899 raise VCSException("Git svn rebase failed", p.output)
900 self.refreshed = True
902 rev = rev or 'master'
904 nospaces_rev = rev.replace(' ', '%20')
905 # Try finding a svn tag
906 for treeish in ['origin/', '']:
907 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
908 if p.returncode == 0:
910 if p.returncode != 0:
911 # No tag found, normal svn rev translation
912 # Translate svn rev into git format
913 rev_split = rev.split('/')
916 for treeish in ['origin/', '']:
917 if len(rev_split) > 1:
918 treeish += rev_split[0]
919 svn_rev = rev_split[1]
922 # if no branch is specified, then assume trunk (i.e. 'master' branch):
926 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
928 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
929 git_rev = p.output.rstrip()
931 if p.returncode == 0 and git_rev:
934 if p.returncode != 0 or not git_rev:
935 # Try a plain git checkout as a last resort
936 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
937 if p.returncode != 0:
938 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
940 # Check out the git rev equivalent to the svn rev
941 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
942 if p.returncode != 0:
943 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
945 # Get rid of any uncontrolled files left behind
946 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
947 if p.returncode != 0:
948 raise VCSException("Git clean failed", p.output)
952 for treeish in ['origin/', '']:
953 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
959 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
960 if p.returncode != 0:
962 return p.output.strip()
970 def gotorevisionx(self, rev):
971 if not os.path.exists(self.local):
972 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
973 if p.returncode != 0:
974 self.clone_failed = True
975 raise VCSException("Hg clone failed", p.output)
977 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
978 if p.returncode != 0:
979 raise VCSException("Hg status failed", p.output)
980 for line in p.output.splitlines():
981 if not line.startswith('? '):
982 raise VCSException("Unexpected output from hg status -uS: " + line)
983 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
984 if not self.refreshed:
985 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
986 if p.returncode != 0:
987 raise VCSException("Hg pull failed", p.output)
988 self.refreshed = True
990 rev = rev or 'default'
993 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
994 if p.returncode != 0:
995 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
996 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
997 # Also delete untracked files, we have to enable purge extension for that:
998 if "'purge' is provided by the following extension" in p.output:
999 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
1000 myfile.write("\n[extensions]\nhgext.purge=\n")
1001 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1002 if p.returncode != 0:
1003 raise VCSException("HG purge failed", p.output)
1004 elif p.returncode != 0:
1005 raise VCSException("HG purge failed", p.output)
1008 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
1009 return p.output.splitlines()[1:]
1017 def gotorevisionx(self, rev):
1018 if not os.path.exists(self.local):
1019 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
1020 if p.returncode != 0:
1021 self.clone_failed = True
1022 raise VCSException("Bzr branch failed", p.output)
1024 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
1025 if p.returncode != 0:
1026 raise VCSException("Bzr revert failed", p.output)
1027 if not self.refreshed:
1028 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
1029 if p.returncode != 0:
1030 raise VCSException("Bzr update failed", p.output)
1031 self.refreshed = True
1033 revargs = list(['-r', rev] if rev else [])
1034 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
1035 if p.returncode != 0:
1036 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1039 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
1040 return [tag.split(' ')[0].strip() for tag in
1041 p.output.splitlines()]
1044 def unescape_string(string):
1047 if string[0] == '"' and string[-1] == '"':
1050 return string.replace("\\'", "'")
1053 def retrieve_string(app_dir, string, xmlfiles=None):
1055 if not string.startswith('@string/'):
1056 return unescape_string(string)
1058 if xmlfiles is None:
1061 os.path.join(app_dir, 'res'),
1062 os.path.join(app_dir, 'src', 'main', 'res'),
1064 for r, d, f in os.walk(res_dir):
1065 if os.path.basename(r) == 'values':
1066 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
1068 name = string[len('@string/'):]
1070 def element_content(element):
1071 if element.text is None:
1073 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1074 return s.decode('utf-8').strip()
1076 for path in xmlfiles:
1077 if not os.path.isfile(path):
1079 xml = parse_xml(path)
1080 element = xml.find('string[@name="' + name + '"]')
1081 if element is not None:
1082 content = element_content(element)
1083 return retrieve_string(app_dir, content, xmlfiles)
1088 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1089 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1092 def manifest_paths(app_dir, flavours):
1093 '''Return list of existing files that will be used to find the highest vercode'''
1095 possible_manifests = \
1096 [os.path.join(app_dir, 'AndroidManifest.xml'),
1097 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1098 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1099 os.path.join(app_dir, 'build.gradle')]
1101 for flavour in flavours:
1102 if flavour == 'yes':
1104 possible_manifests.append(
1105 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1107 return [path for path in possible_manifests if os.path.isfile(path)]
1110 def fetch_real_name(app_dir, flavours):
1111 '''Retrieve the package name. Returns the name, or None if not found.'''
1112 for path in manifest_paths(app_dir, flavours):
1113 if not has_extension(path, 'xml') or not os.path.isfile(path):
1115 logging.debug("fetch_real_name: Checking manifest at " + path)
1116 xml = parse_xml(path)
1117 app = xml.find('application')
1120 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1122 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1123 result = retrieve_string_singleline(app_dir, label)
1125 result = result.strip()
1130 def get_library_references(root_dir):
1132 proppath = os.path.join(root_dir, 'project.properties')
1133 if not os.path.isfile(proppath):
1135 with open(proppath, 'r', encoding='iso-8859-1') as f:
1137 if not line.startswith('android.library.reference.'):
1139 path = line.split('=')[1].strip()
1140 relpath = os.path.join(root_dir, path)
1141 if not os.path.isdir(relpath):
1143 logging.debug("Found subproject at %s" % path)
1144 libraries.append(path)
1148 def ant_subprojects(root_dir):
1149 subprojects = get_library_references(root_dir)
1150 for subpath in subprojects:
1151 subrelpath = os.path.join(root_dir, subpath)
1152 for p in get_library_references(subrelpath):
1153 relp = os.path.normpath(os.path.join(subpath, p))
1154 if relp not in subprojects:
1155 subprojects.insert(0, relp)
1159 def remove_debuggable_flags(root_dir):
1160 # Remove forced debuggable flags
1161 logging.debug("Removing debuggable flags from %s" % root_dir)
1162 for root, dirs, files in os.walk(root_dir):
1163 if 'AndroidManifest.xml' in files:
1164 regsub_file(r'android:debuggable="[^"]*"',
1166 os.path.join(root, 'AndroidManifest.xml'))
1169 vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search
1170 vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1171 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1174 def app_matches_packagename(app, package):
1177 appid = app.UpdateCheckName or app.id
1178 if appid is None or appid == "Ignore":
1180 return appid == package
1183 def parse_androidmanifests(paths, app):
1185 Extract some information from the AndroidManifest.xml at the given path.
1186 Returns (version, vercode, package), any or all of which might be None.
1187 All values returned are strings.
1190 ignoreversions = app.UpdateCheckIgnore
1191 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1194 return (None, None, None)
1202 if not os.path.isfile(path):
1205 logging.debug("Parsing manifest at {0}".format(path))
1210 if has_extension(path, 'gradle'):
1211 with open(path, 'r') as f:
1213 if gradle_comment.match(line):
1215 # Grab first occurence of each to avoid running into
1216 # alternative flavours and builds.
1218 matches = psearch_g(line)
1220 s = matches.group(2)
1221 if app_matches_packagename(app, s):
1224 matches = vnsearch_g(line)
1226 version = matches.group(2)
1228 matches = vcsearch_g(line)
1230 vercode = matches.group(1)
1233 xml = parse_xml(path)
1234 if "package" in xml.attrib:
1235 s = xml.attrib["package"]
1236 if app_matches_packagename(app, s):
1238 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1239 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1240 base_dir = os.path.dirname(path)
1241 version = retrieve_string_singleline(base_dir, version)
1242 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1243 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1244 if string_is_integer(a):
1247 logging.warning("Problem with xml at {0}".format(path))
1249 # Remember package name, may be defined separately from version+vercode
1251 package = max_package
1253 logging.debug("..got package={0}, version={1}, vercode={2}"
1254 .format(package, version, vercode))
1256 # Always grab the package name and version name in case they are not
1257 # together with the highest version code
1258 if max_package is None and package is not None:
1259 max_package = package
1260 if max_version is None and version is not None:
1261 max_version = version
1263 if vercode is not None \
1264 and (max_vercode is None or vercode > max_vercode):
1265 if not ignoresearch or not ignoresearch(version):
1266 if version is not None:
1267 max_version = version
1268 if vercode is not None:
1269 max_vercode = vercode
1270 if package is not None:
1271 max_package = package
1273 max_version = "Ignore"
1275 if max_version is None:
1276 max_version = "Unknown"
1278 if max_package and not is_valid_package_name(max_package):
1279 raise FDroidException("Invalid package name {0}".format(max_package))
1281 return (max_version, max_vercode, max_package)
1284 def is_valid_package_name(name):
1285 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1288 class FDroidException(Exception):
1290 def __init__(self, value, detail=None):
1292 self.detail = detail
1294 def shortened_detail(self):
1295 if len(self.detail) < 16000:
1297 return '[...]\n' + self.detail[-16000:]
1299 def get_wikitext(self):
1300 ret = repr(self.value) + "\n"
1303 ret += "<pre>\n" + self.shortened_detail() + "</pre>\n"
1309 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1313 class VCSException(FDroidException):
1317 class BuildException(FDroidException):
1321 # Get the specified source library.
1322 # Returns the path to it. Normally this is the path to be used when referencing
1323 # it, which may be a subdirectory of the actual project. If you want the base
1324 # directory of the project, pass 'basepath=True'.
1325 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1326 raw=False, prepare=True, preponly=False, refresh=True,
1335 name, ref = spec.split('@')
1337 number, name = name.split(':', 1)
1339 name, subdir = name.split('/', 1)
1341 if name not in fdroidserver.metadata.srclibs:
1342 raise VCSException('srclib ' + name + ' not found.')
1344 srclib = fdroidserver.metadata.srclibs[name]
1346 sdir = os.path.join(srclib_dir, name)
1349 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1350 vcs.srclib = (name, number, sdir)
1352 vcs.gotorevision(ref, refresh)
1359 libdir = os.path.join(sdir, subdir)
1360 elif srclib["Subdir"]:
1361 for subdir in srclib["Subdir"]:
1362 libdir_candidate = os.path.join(sdir, subdir)
1363 if os.path.exists(libdir_candidate):
1364 libdir = libdir_candidate
1370 remove_signing_keys(sdir)
1371 remove_debuggable_flags(sdir)
1375 if srclib["Prepare"]:
1376 cmd = replace_config_vars(srclib["Prepare"], build)
1378 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1379 if p.returncode != 0:
1380 raise BuildException("Error running prepare command for srclib %s"
1386 return (name, number, libdir)
1389 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1392 # Prepare the source code for a particular build
1393 # 'vcs' - the appropriate vcs object for the application
1394 # 'app' - the application details from the metadata
1395 # 'build' - the build details from the metadata
1396 # 'build_dir' - the path to the build directory, usually
1398 # 'srclib_dir' - the path to the source libraries directory, usually
1400 # 'extlib_dir' - the path to the external libraries directory, usually
1402 # Returns the (root, srclibpaths) where:
1403 # 'root' is the root directory, which may be the same as 'build_dir' or may
1404 # be a subdirectory of it.
1405 # 'srclibpaths' is information on the srclibs being used
1406 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1408 # Optionally, the actual app source can be in a subdirectory
1410 root_dir = os.path.join(build_dir, build.subdir)
1412 root_dir = build_dir
1414 # Get a working copy of the right revision
1415 logging.info("Getting source for revision " + build.commit)
1416 vcs.gotorevision(build.commit, refresh)
1418 # Initialise submodules if required
1419 if build.submodules:
1420 logging.info("Initialising submodules")
1421 vcs.initsubmodules()
1423 # Check that a subdir (if we're using one) exists. This has to happen
1424 # after the checkout, since it might not exist elsewhere
1425 if not os.path.exists(root_dir):
1426 raise BuildException('Missing subdir ' + root_dir)
1428 # Run an init command if one is required
1430 cmd = replace_config_vars(build.init, build)
1431 logging.info("Running 'init' commands in %s" % root_dir)
1433 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1434 if p.returncode != 0:
1435 raise BuildException("Error running init command for %s:%s" %
1436 (app.id, build.versionName), p.output)
1438 # Apply patches if any
1440 logging.info("Applying patches")
1441 for patch in build.patch:
1442 patch = patch.strip()
1443 logging.info("Applying " + patch)
1444 patch_path = os.path.join('metadata', app.id, patch)
1445 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1446 if p.returncode != 0:
1447 raise BuildException("Failed to apply patch %s" % patch_path)
1449 # Get required source libraries
1452 logging.info("Collecting source libraries")
1453 for lib in build.srclibs:
1454 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1455 refresh=refresh, build=build))
1457 for name, number, libpath in srclibpaths:
1458 place_srclib(root_dir, int(number) if number else None, libpath)
1460 basesrclib = vcs.getsrclib()
1461 # If one was used for the main source, add that too.
1463 srclibpaths.append(basesrclib)
1465 # Update the local.properties file
1466 localprops = [os.path.join(build_dir, 'local.properties')]
1468 parts = build.subdir.split(os.sep)
1471 cur = os.path.join(cur, d)
1472 localprops += [os.path.join(cur, 'local.properties')]
1473 for path in localprops:
1475 if os.path.isfile(path):
1476 logging.info("Updating local.properties file at %s" % path)
1477 with open(path, 'r', encoding='iso-8859-1') as f:
1481 logging.info("Creating local.properties file at %s" % path)
1482 # Fix old-fashioned 'sdk-location' by copying
1483 # from sdk.dir, if necessary
1485 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1486 re.S | re.M).group(1)
1487 props += "sdk-location=%s\n" % sdkloc
1489 props += "sdk.dir=%s\n" % config['sdk_path']
1490 props += "sdk-location=%s\n" % config['sdk_path']
1491 ndk_path = build.ndk_path()
1492 # if for any reason the path isn't valid or the directory
1493 # doesn't exist, some versions of Gradle will error with a
1494 # cryptic message (even if the NDK is not even necessary).
1495 # https://gitlab.com/fdroid/fdroidserver/issues/171
1496 if ndk_path and os.path.exists(ndk_path):
1498 props += "ndk.dir=%s\n" % ndk_path
1499 props += "ndk-location=%s\n" % ndk_path
1500 # Add java.encoding if necessary
1502 props += "java.encoding=%s\n" % build.encoding
1503 with open(path, 'w', encoding='iso-8859-1') as f:
1507 if build.build_method() == 'gradle':
1508 flavours = build.gradle
1511 n = build.target.split('-')[1]
1512 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1513 r'compileSdkVersion %s' % n,
1514 os.path.join(root_dir, 'build.gradle'))
1516 # Remove forced debuggable flags
1517 remove_debuggable_flags(root_dir)
1519 # Insert version code and number into the manifest if necessary
1520 if build.forceversion:
1521 logging.info("Changing the version name")
1522 for path in manifest_paths(root_dir, flavours):
1523 if not os.path.isfile(path):
1525 if has_extension(path, 'xml'):
1526 regsub_file(r'android:versionName="[^"]*"',
1527 r'android:versionName="%s"' % build.versionName,
1529 elif has_extension(path, 'gradle'):
1530 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1531 r"""\1versionName '%s'""" % build.versionName,
1534 if build.forcevercode:
1535 logging.info("Changing the version code")
1536 for path in manifest_paths(root_dir, flavours):
1537 if not os.path.isfile(path):
1539 if has_extension(path, 'xml'):
1540 regsub_file(r'android:versionCode="[^"]*"',
1541 r'android:versionCode="%s"' % build.versionCode,
1543 elif has_extension(path, 'gradle'):
1544 regsub_file(r'versionCode[ =]+[0-9]+',
1545 r'versionCode %s' % build.versionCode,
1548 # Delete unwanted files
1550 logging.info("Removing specified files")
1551 for part in getpaths(build_dir, build.rm):
1552 dest = os.path.join(build_dir, part)
1553 logging.info("Removing {0}".format(part))
1554 if os.path.lexists(dest):
1555 if os.path.islink(dest):
1556 FDroidPopen(['unlink', dest], output=False)
1558 FDroidPopen(['rm', '-rf', dest], output=False)
1560 logging.info("...but it didn't exist")
1562 remove_signing_keys(build_dir)
1564 # Add required external libraries
1566 logging.info("Collecting prebuilt libraries")
1567 libsdir = os.path.join(root_dir, 'libs')
1568 if not os.path.exists(libsdir):
1570 for lib in build.extlibs:
1572 logging.info("...installing extlib {0}".format(lib))
1573 libf = os.path.basename(lib)
1574 libsrc = os.path.join(extlib_dir, lib)
1575 if not os.path.exists(libsrc):
1576 raise BuildException("Missing extlib file {0}".format(libsrc))
1577 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1579 # Run a pre-build command if one is required
1581 logging.info("Running 'prebuild' commands in %s" % root_dir)
1583 cmd = replace_config_vars(build.prebuild, build)
1585 # Substitute source library paths into prebuild commands
1586 for name, number, libpath in srclibpaths:
1587 libpath = os.path.relpath(libpath, root_dir)
1588 cmd = cmd.replace('$$' + name + '$$', libpath)
1590 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1591 if p.returncode != 0:
1592 raise BuildException("Error running prebuild command for %s:%s" %
1593 (app.id, build.versionName), p.output)
1595 # Generate (or update) the ant build file, build.xml...
1596 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1597 parms = ['android', 'update', 'lib-project']
1598 lparms = ['android', 'update', 'project']
1601 parms += ['-t', build.target]
1602 lparms += ['-t', build.target]
1603 if build.androidupdate:
1604 update_dirs = build.androidupdate
1606 update_dirs = ant_subprojects(root_dir) + ['.']
1608 for d in update_dirs:
1609 subdir = os.path.join(root_dir, d)
1611 logging.debug("Updating main project")
1612 cmd = parms + ['-p', d]
1614 logging.debug("Updating subproject %s" % d)
1615 cmd = lparms + ['-p', d]
1616 p = SdkToolsPopen(cmd, cwd=root_dir)
1617 # Check to see whether an error was returned without a proper exit
1618 # code (this is the case for the 'no target set or target invalid'
1620 if p.returncode != 0 or p.output.startswith("Error: "):
1621 raise BuildException("Failed to update project at %s" % d, p.output)
1622 # Clean update dirs via ant
1624 logging.info("Cleaning subproject %s" % d)
1625 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1627 return (root_dir, srclibpaths)
1630 # Extend via globbing the paths from a field and return them as a map from
1631 # original path to resulting paths
1632 def getpaths_map(build_dir, globpaths):
1636 full_path = os.path.join(build_dir, p)
1637 full_path = os.path.normpath(full_path)
1638 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1640 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1644 # Extend via globbing the paths from a field and return them as a set
1645 def getpaths(build_dir, globpaths):
1646 paths_map = getpaths_map(build_dir, globpaths)
1648 for k, v in paths_map.items():
1655 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1661 self.path = os.path.join('stats', 'known_apks.txt')
1663 if os.path.isfile(self.path):
1664 with open(self.path, 'r', encoding='utf8') as f:
1666 t = line.rstrip().split(' ')
1668 self.apks[t[0]] = (t[1], None)
1670 self.apks[t[0]] = (t[1], datetime.strptime(t[2], '%Y-%m-%d'))
1671 self.changed = False
1673 def writeifchanged(self):
1674 if not self.changed:
1677 if not os.path.exists('stats'):
1681 for apk, app in self.apks.items():
1683 line = apk + ' ' + appid
1685 line += ' ' + added.strftime('%Y-%m-%d')
1688 with open(self.path, 'w', encoding='utf8') as f:
1689 for line in sorted(lst, key=natural_key):
1690 f.write(line + '\n')
1692 def recordapk(self, apk, app, default_date=None):
1694 Record an apk (if it's new, otherwise does nothing)
1695 Returns the date it was added as a datetime instance
1697 if apk not in self.apks:
1698 if default_date is None:
1699 default_date = datetime.utcnow()
1700 self.apks[apk] = (app, default_date)
1702 _, added = self.apks[apk]
1705 # Look up information - given the 'apkname', returns (app id, date added/None).
1706 # Or returns None for an unknown apk.
1707 def getapp(self, apkname):
1708 if apkname in self.apks:
1709 return self.apks[apkname]
1712 # Get the most recent 'num' apps added to the repo, as a list of package ids
1713 # with the most recent first.
1714 def getlatest(self, num):
1716 for apk, app in self.apks.items():
1720 if apps[appid] > added:
1724 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1725 lst = [app for app, _ in sortedapps]
1730 def get_file_extension(filename):
1731 """get the normalized file extension, can be blank string but never None"""
1733 return os.path.splitext(filename)[1].lower()[1:]
1736 def isApkAndDebuggable(apkfile, config):
1737 """Returns True if the given file is an APK and is debuggable
1739 :param apkfile: full path to the apk to check"""
1741 if get_file_extension(apkfile) != 'apk':
1744 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1746 if p.returncode != 0:
1747 logging.critical("Failed to get apk manifest information")
1749 for line in p.output.splitlines():
1750 if 'android:debuggable' in line and not line.endswith('0x0'):
1757 self.returncode = None
1761 def SdkToolsPopen(commands, cwd=None, output=True):
1763 if cmd not in config:
1764 config[cmd] = find_sdk_tools_cmd(commands[0])
1765 abscmd = config[cmd]
1767 logging.critical("Could not find '%s' on your system" % cmd)
1770 test_aapt_version(config['aapt'])
1771 return FDroidPopen([abscmd] + commands[1:],
1772 cwd=cwd, output=output)
1775 def FDroidPopenBytes(commands, cwd=None, output=True, stderr_to_stdout=True):
1777 Run a command and capture the possibly huge output as bytes.
1779 :param commands: command and argument list like in subprocess.Popen
1780 :param cwd: optionally specifies a working directory
1781 :returns: A PopenResult.
1786 set_FDroidPopen_env()
1789 cwd = os.path.normpath(cwd)
1790 logging.debug("Directory: %s" % cwd)
1791 logging.debug("> %s" % ' '.join(commands))
1793 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
1794 result = PopenResult()
1797 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
1798 stdout=subprocess.PIPE, stderr=stderr_param)
1799 except OSError as e:
1800 raise BuildException("OSError while trying to execute " +
1801 ' '.join(commands) + ': ' + str(e))
1803 if not stderr_to_stdout and options.verbose:
1804 stderr_queue = Queue()
1805 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
1807 while not stderr_reader.eof():
1808 while not stderr_queue.empty():
1809 line = stderr_queue.get()
1810 sys.stderr.buffer.write(line)
1815 stdout_queue = Queue()
1816 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1819 # Check the queue for output (until there is no more to get)
1820 while not stdout_reader.eof():
1821 while not stdout_queue.empty():
1822 line = stdout_queue.get()
1823 if output and options.verbose:
1824 # Output directly to console
1825 sys.stderr.buffer.write(line)
1831 result.returncode = p.wait()
1832 result.output = buf.getvalue()
1837 def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
1839 Run a command and capture the possibly huge output as a str.
1841 :param commands: command and argument list like in subprocess.Popen
1842 :param cwd: optionally specifies a working directory
1843 :returns: A PopenResult.
1845 result = FDroidPopenBytes(commands, cwd, output, stderr_to_stdout)
1846 result.output = result.output.decode('utf-8', 'ignore')
1850 gradle_comment = re.compile(r'[ ]*//')
1851 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1852 gradle_line_matches = [
1853 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1854 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1855 re.compile(r'.*\.readLine\(.*'),
1859 def remove_signing_keys(build_dir):
1860 for root, dirs, files in os.walk(build_dir):
1861 if 'build.gradle' in files:
1862 path = os.path.join(root, 'build.gradle')
1864 with open(path, "r", encoding='utf8') as o:
1865 lines = o.readlines()
1871 with open(path, "w", encoding='utf8') as o:
1872 while i < len(lines):
1875 while line.endswith('\\\n'):
1876 line = line.rstrip('\\\n') + lines[i]
1879 if gradle_comment.match(line):
1884 opened += line.count('{')
1885 opened -= line.count('}')
1888 if gradle_signing_configs.match(line):
1893 if any(s.match(line) for s in gradle_line_matches):
1901 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1904 'project.properties',
1906 'default.properties',
1907 'ant.properties', ]:
1908 if propfile in files:
1909 path = os.path.join(root, propfile)
1911 with open(path, "r", encoding='iso-8859-1') as o:
1912 lines = o.readlines()
1916 with open(path, "w", encoding='iso-8859-1') as o:
1918 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1925 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1928 def set_FDroidPopen_env(build=None):
1930 set up the environment variables for the build environment
1932 There is only a weak standard, the variables used by gradle, so also set
1933 up the most commonly used environment variables for SDK and NDK. Also, if
1934 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
1936 global env, orig_path
1940 orig_path = env['PATH']
1941 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
1942 env[n] = config['sdk_path']
1943 for k, v in config['java_paths'].items():
1944 env['JAVA%s_HOME' % k] = v
1946 missinglocale = True
1947 for k, v in env.items():
1948 if k == 'LANG' and v != 'C':
1949 missinglocale = False
1951 missinglocale = False
1953 env['LANG'] = 'en_US.UTF-8'
1955 if build is not None:
1956 path = build.ndk_path()
1957 paths = orig_path.split(os.pathsep)
1958 if path not in paths:
1959 paths = [path] + paths
1960 env['PATH'] = os.pathsep.join(paths)
1961 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
1962 env[n] = build.ndk_path()
1965 def replace_build_vars(cmd, build):
1966 cmd = cmd.replace('$$COMMIT$$', build.commit)
1967 cmd = cmd.replace('$$VERSION$$', build.versionName)
1968 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
1972 def replace_config_vars(cmd, build):
1973 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1974 cmd = cmd.replace('$$NDK$$', build.ndk_path())
1975 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1976 cmd = cmd.replace('$$QT$$', config['qt_sdk_path'] or '')
1977 if build is not None:
1978 cmd = replace_build_vars(cmd, build)
1982 def place_srclib(root_dir, number, libpath):
1985 relpath = os.path.relpath(libpath, root_dir)
1986 proppath = os.path.join(root_dir, 'project.properties')
1989 if os.path.isfile(proppath):
1990 with open(proppath, "r", encoding='iso-8859-1') as o:
1991 lines = o.readlines()
1993 with open(proppath, "w", encoding='iso-8859-1') as o:
1996 if line.startswith('android.library.reference.%d=' % number):
1997 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2002 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2005 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
2008 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
2009 """Verify that two apks are the same
2011 One of the inputs is signed, the other is unsigned. The signature metadata
2012 is transferred from the signed to the unsigned apk, and then jarsigner is
2013 used to verify that the signature from the signed apk is also varlid for
2015 :param signed_apk: Path to a signed apk file
2016 :param unsigned_apk: Path to an unsigned apk file expected to match it
2017 :param tmp_dir: Path to directory for temporary files
2018 :returns: None if the verification is successful, otherwise a string
2019 describing what went wrong.
2021 with ZipFile(signed_apk) as signed_apk_as_zip:
2022 meta_inf_files = ['META-INF/MANIFEST.MF']
2023 for f in signed_apk_as_zip.namelist():
2024 if apk_sigfile.match(f):
2025 meta_inf_files.append(f)
2026 if len(meta_inf_files) < 3:
2027 return "Signature files missing from {0}".format(signed_apk)
2028 signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
2029 with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
2030 for meta_inf_file in meta_inf_files:
2031 unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
2033 if subprocess.call([config['jarsigner'], '-verify', unsigned_apk]) != 0:
2034 logging.info("...NOT verified - {0}".format(signed_apk))
2035 return compare_apks(signed_apk, unsigned_apk, tmp_dir)
2036 logging.info("...successfully verified")
2040 apk_badchars = re.compile('''[/ :;'"]''')
2043 def compare_apks(apk1, apk2, tmp_dir):
2046 Returns None if the apk content is the same (apart from the signing key),
2047 otherwise a string describing what's different, or what went wrong when
2048 trying to do the comparison.
2051 absapk1 = os.path.abspath(apk1)
2052 absapk2 = os.path.abspath(apk2)
2054 # try to find diffoscope in the path, if it hasn't been manually configed
2055 if 'diffoscope' not in config:
2056 tmp = find_command('diffoscope')
2058 config['diffoscope'] = tmp
2059 if 'diffoscope' in config:
2060 htmlfile = absapk1 + '.diffoscope.html'
2061 textfile = absapk1 + '.diffoscope.txt'
2062 if subprocess.call([config['diffoscope'],
2063 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2064 '--html', htmlfile, '--text', textfile,
2065 absapk1, absapk2]) != 0:
2066 return("Failed to unpack " + apk1)
2068 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2069 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2070 for d in [apk1dir, apk2dir]:
2071 if os.path.exists(d):
2074 os.mkdir(os.path.join(d, 'jar-xf'))
2076 if subprocess.call(['jar', 'xf',
2077 os.path.abspath(apk1)],
2078 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2079 return("Failed to unpack " + apk1)
2080 if subprocess.call(['jar', 'xf',
2081 os.path.abspath(apk2)],
2082 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2083 return("Failed to unpack " + apk2)
2085 # try to find apktool in the path, if it hasn't been manually configed
2086 if 'apktool' not in config:
2087 tmp = find_command('apktool')
2089 config['apktool'] = tmp
2090 if 'apktool' in config:
2091 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2093 return("Failed to unpack " + apk1)
2094 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2096 return("Failed to unpack " + apk2)
2098 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2099 lines = p.output.splitlines()
2100 if len(lines) != 1 or 'META-INF' not in lines[0]:
2101 meld = find_command('meld')
2102 if meld is not None:
2103 p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
2104 return("Unexpected diff output - " + p.output)
2106 # since everything verifies, delete the comparison to keep cruft down
2107 shutil.rmtree(apk1dir)
2108 shutil.rmtree(apk2dir)
2110 # If we get here, it seems like they're the same!
2114 def find_command(command):
2115 '''find the full path of a command, or None if it can't be found in the PATH'''
2118 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2120 fpath, fname = os.path.split(command)
2125 for path in os.environ["PATH"].split(os.pathsep):
2126 path = path.strip('"')
2127 exe_file = os.path.join(path, command)
2128 if is_exe(exe_file):
2135 '''generate a random password for when generating keys'''
2136 h = hashlib.sha256()
2137 h.update(os.urandom(16)) # salt
2138 h.update(socket.getfqdn().encode('utf-8'))
2139 passwd = base64.b64encode(h.digest()).strip()
2140 return passwd.decode('utf-8')
2143 def genkeystore(localconfig):
2144 '''Generate a new key with random passwords and add it to new keystore'''
2145 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2146 keystoredir = os.path.dirname(localconfig['keystore'])
2147 if keystoredir is None or keystoredir == '':
2148 keystoredir = os.path.join(os.getcwd(), keystoredir)
2149 if not os.path.exists(keystoredir):
2150 os.makedirs(keystoredir, mode=0o700)
2152 write_password_file("keystorepass", localconfig['keystorepass'])
2153 write_password_file("keypass", localconfig['keypass'])
2154 p = FDroidPopen([config['keytool'], '-genkey',
2155 '-keystore', localconfig['keystore'],
2156 '-alias', localconfig['repo_keyalias'],
2157 '-keyalg', 'RSA', '-keysize', '4096',
2158 '-sigalg', 'SHA256withRSA',
2159 '-validity', '10000',
2160 '-storepass:file', config['keystorepassfile'],
2161 '-keypass:file', config['keypassfile'],
2162 '-dname', localconfig['keydname']])
2163 # TODO keypass should be sent via stdin
2164 if p.returncode != 0:
2165 raise BuildException("Failed to generate key", p.output)
2166 os.chmod(localconfig['keystore'], 0o0600)
2167 # now show the lovely key that was just generated
2168 p = FDroidPopen([config['keytool'], '-list', '-v',
2169 '-keystore', localconfig['keystore'],
2170 '-alias', localconfig['repo_keyalias'],
2171 '-storepass:file', config['keystorepassfile']])
2172 logging.info(p.output.strip() + '\n\n')
2175 def write_to_config(thisconfig, key, value=None):
2176 '''write a key/value to the local config.py'''
2178 origkey = key + '_orig'
2179 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2180 with open('config.py', 'r', encoding='utf8') as f:
2182 pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
2183 repl = '\n' + key + ' = "' + value + '"'
2184 data = re.sub(pattern, repl, data)
2185 # if this key is not in the file, append it
2186 if not re.match('\s*' + key + '\s*=\s*"', data):
2188 # make sure the file ends with a carraige return
2189 if not re.match('\n$', data):
2191 with open('config.py', 'w', encoding='utf8') as f:
2195 def parse_xml(path):
2196 return XMLElementTree.parse(path).getroot()
2199 def string_is_integer(string):
2207 def get_per_app_repos():
2208 '''per-app repos are dirs named with the packageName of a single app'''
2210 # Android packageNames are Java packages, they may contain uppercase or
2211 # lowercase letters ('A' through 'Z'), numbers, and underscores
2212 # ('_'). However, individual package name parts may only start with
2213 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2214 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2217 for root, dirs, files in os.walk(os.getcwd()):
2219 print('checking', root, 'for', d)
2220 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2221 # standard parts of an fdroid repo, so never packageNames
2224 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
2230 def is_repo_file(filename):
2231 '''Whether the file in a repo is a build product to be delivered to users'''
2232 return os.path.isfile(filename) \
2233 and not filename.endswith('.asc') \
2234 and not filename.endswith('.sig') \
2235 and os.path.basename(filename) not in [
2237 'index_unsigned.jar',