3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
37 import xml.etree.ElementTree as XMLElementTree
39 from queue import Queue
41 from zipfile import ZipFile
43 import fdroidserver.metadata
44 from .asynchronousfilereader import AsynchronousFileReader
47 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
56 'sdk_path': "$ANDROID_HOME",
60 'r12b': "$ANDROID_NDK",
62 'build_tools': "24.0.1",
63 'force_build_tools': False,
68 'accepted_formats': ['txt', 'yml'],
69 'sync_from_local_copy_dir': False,
70 'per_app_repos': False,
71 'make_current_version_link': True,
72 'current_version_name_source': 'Name',
73 'update_stats': False,
77 'stats_to_carbon': False,
79 'build_server_always': False,
80 'keystore': 'keystore.jks',
81 'smartcardoptions': [],
87 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
88 'repo_name': "My First FDroid Repo Demo",
89 'repo_icon': "fdroid-icon.png",
90 'repo_description': '''
91 This is a repository of apps to be used with FDroid. Applications in this
92 repository are either official binaries built by the original application
93 developers, or are binaries built from source by the admin of f-droid.org
94 using the tools on https://gitlab.com/u/fdroid.
100 def setup_global_opts(parser):
101 parser.add_argument("-v", "--verbose", action="store_true", default=False,
102 help="Spew out even more information than normal")
103 parser.add_argument("-q", "--quiet", action="store_true", default=False,
104 help="Restrict output to warnings and errors")
107 def fill_config_defaults(thisconfig):
108 for k, v in default_config.items():
109 if k not in thisconfig:
112 # Expand paths (~users and $vars)
113 def expand_path(path):
117 path = os.path.expanduser(path)
118 path = os.path.expandvars(path)
123 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
128 thisconfig[k + '_orig'] = v
130 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
131 if thisconfig['java_paths'] is None:
132 thisconfig['java_paths'] = dict()
134 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
135 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
136 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
137 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
138 if os.getenv('JAVA_HOME') is not None:
139 pathlist.append(os.getenv('JAVA_HOME'))
140 if os.getenv('PROGRAMFILES') is not None:
141 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
142 for d in sorted(pathlist):
143 if os.path.islink(d):
145 j = os.path.basename(d)
146 # the last one found will be the canonical one, so order appropriately
148 r'^1\.([6-9])\.0\.jdk$', # OSX
149 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
150 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
151 r'^jdk([6-9])-openjdk$', # Arch
152 r'^java-([6-9])-openjdk$', # Arch
153 r'^java-([6-9])-jdk$', # Arch (oracle)
154 r'^java-1\.([6-9])\.0-.*$', # RedHat
155 r'^java-([6-9])-oracle$', # Debian WebUpd8
156 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
157 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
159 m = re.match(regex, j)
162 for p in [d, os.path.join(d, 'Contents', 'Home')]:
163 if os.path.exists(os.path.join(p, 'bin', 'javac')):
164 thisconfig['java_paths'][m.group(1)] = p
166 for java_version in ('7', '8', '9'):
167 if java_version not in thisconfig['java_paths']:
169 java_home = thisconfig['java_paths'][java_version]
170 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
171 if os.path.exists(jarsigner):
172 thisconfig['jarsigner'] = jarsigner
173 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
174 break # Java7 is preferred, so quit if found
176 for k in ['ndk_paths', 'java_paths']:
182 thisconfig[k][k2] = exp
183 thisconfig[k][k2 + '_orig'] = v
186 def regsub_file(pattern, repl, path):
187 with open(path, 'rb') as f:
189 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
190 with open(path, 'wb') as f:
194 def read_config(opts, config_file='config.py'):
195 """Read the repository config
197 The config is read from config_file, which is in the current
198 directory when any of the repo management commands are used. If
199 there is a local metadata file in the git repo, then config.py is
200 not required, just use defaults.
203 global config, options
205 if config is not None:
212 if os.path.isfile(config_file):
213 logging.debug("Reading %s" % config_file)
214 with io.open(config_file, "rb") as f:
215 code = compile(f.read(), config_file, 'exec')
216 exec(code, None, config)
217 elif len(get_local_metadata_files()) == 0:
218 logging.critical("Missing config file - is this a repo directory?")
221 # smartcardoptions must be a list since its command line args for Popen
222 if 'smartcardoptions' in config:
223 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
224 elif 'keystore' in config and config['keystore'] == 'NONE':
225 # keystore='NONE' means use smartcard, these are required defaults
226 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
227 'SunPKCS11-OpenSC', '-providerClass',
228 'sun.security.pkcs11.SunPKCS11',
229 '-providerArg', 'opensc-fdroid.cfg']
231 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
232 st = os.stat(config_file)
233 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
234 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
236 fill_config_defaults(config)
238 for k in ["keystorepass", "keypass"]:
240 write_password_file(k)
242 for k in ["repo_description", "archive_description"]:
244 config[k] = clean_description(config[k])
246 if 'serverwebroot' in config:
247 if isinstance(config['serverwebroot'], str):
248 roots = [config['serverwebroot']]
249 elif all(isinstance(item, str) for item in config['serverwebroot']):
250 roots = config['serverwebroot']
252 raise TypeError('only accepts strings, lists, and tuples')
254 for rootstr in roots:
255 # since this is used with rsync, where trailing slashes have
256 # meaning, ensure there is always a trailing slash
257 if rootstr[-1] != '/':
259 rootlist.append(rootstr.replace('//', '/'))
260 config['serverwebroot'] = rootlist
265 def find_sdk_tools_cmd(cmd):
266 '''find a working path to a tool from the Android SDK'''
269 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
270 # try to find a working path to this command, in all the recent possible paths
271 if 'build_tools' in config:
272 build_tools = os.path.join(config['sdk_path'], 'build-tools')
273 # if 'build_tools' was manually set and exists, check only that one
274 configed_build_tools = os.path.join(build_tools, config['build_tools'])
275 if os.path.exists(configed_build_tools):
276 tooldirs.append(configed_build_tools)
278 # no configed version, so hunt known paths for it
279 for f in sorted(os.listdir(build_tools), reverse=True):
280 if os.path.isdir(os.path.join(build_tools, f)):
281 tooldirs.append(os.path.join(build_tools, f))
282 tooldirs.append(build_tools)
283 sdk_tools = os.path.join(config['sdk_path'], 'tools')
284 if os.path.exists(sdk_tools):
285 tooldirs.append(sdk_tools)
286 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
287 if os.path.exists(sdk_platform_tools):
288 tooldirs.append(sdk_platform_tools)
289 tooldirs.append('/usr/bin')
291 if os.path.isfile(os.path.join(d, cmd)):
292 return os.path.join(d, cmd)
293 # did not find the command, exit with error message
294 ensure_build_tools_exists(config)
297 def test_sdk_exists(thisconfig):
298 if 'sdk_path' not in thisconfig:
299 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
302 logging.error("'sdk_path' not set in config.py!")
304 if thisconfig['sdk_path'] == default_config['sdk_path']:
305 logging.error('No Android SDK found!')
306 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
307 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
309 if not os.path.exists(thisconfig['sdk_path']):
310 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
312 if not os.path.isdir(thisconfig['sdk_path']):
313 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
315 for d in ['build-tools', 'platform-tools', 'tools']:
316 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
317 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
318 thisconfig['sdk_path'], d))
323 def ensure_build_tools_exists(thisconfig):
324 if not test_sdk_exists(thisconfig):
326 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
327 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
328 if not os.path.isdir(versioned_build_tools):
329 logging.critical('Android Build Tools path "'
330 + versioned_build_tools + '" does not exist!')
334 def write_password_file(pwtype, password=None):
336 writes out passwords to a protected file instead of passing passwords as
337 command line argments
339 filename = '.fdroid.' + pwtype + '.txt'
340 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
342 os.write(fd, config[pwtype].encode('utf-8'))
344 os.write(fd, password.encode('utf-8'))
346 config[pwtype + 'file'] = filename
349 def get_local_metadata_files():
350 '''get any metadata files local to an app's source repo
352 This tries to ignore anything that does not count as app metdata,
353 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
356 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
359 # Given the arguments in the form of multiple appid:[vc] strings, this returns
360 # a dictionary with the set of vercodes specified for each package.
361 def read_pkg_args(args, allow_vercodes=False):
368 if allow_vercodes and ':' in p:
369 package, vercode = p.split(':')
371 package, vercode = p, None
372 if package not in vercodes:
373 vercodes[package] = [vercode] if vercode else []
375 elif vercode and vercode not in vercodes[package]:
376 vercodes[package] += [vercode] if vercode else []
381 # On top of what read_pkg_args does, this returns the whole app metadata, but
382 # limiting the builds list to the builds matching the vercodes specified.
383 def read_app_args(args, allapps, allow_vercodes=False):
385 vercodes = read_pkg_args(args, allow_vercodes)
391 for appid, app in allapps.items():
392 if appid in vercodes:
395 if len(apps) != len(vercodes):
398 logging.critical("No such package: %s" % p)
399 raise FDroidException("Found invalid app ids in arguments")
401 raise FDroidException("No packages specified")
404 for appid, app in apps.items():
408 app.builds = [b for b in app.builds if b.vercode in vc]
409 if len(app.builds) != len(vercodes[appid]):
411 allvcs = [b.vercode for b in app.builds]
412 for v in vercodes[appid]:
414 logging.critical("No such vercode %s for app %s" % (v, appid))
417 raise FDroidException("Found invalid vercodes for some apps")
422 def get_extension(filename):
423 base, ext = os.path.splitext(filename)
426 return base, ext.lower()[1:]
429 def has_extension(filename, ext):
430 _, f_ext = get_extension(filename)
434 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
437 def clean_description(description):
438 'Remove unneeded newlines and spaces from a block of description text'
440 # this is split up by paragraph to make removing the newlines easier
441 for paragraph in re.split(r'\n\n', description):
442 paragraph = re.sub('\r', '', paragraph)
443 paragraph = re.sub('\n', ' ', paragraph)
444 paragraph = re.sub(' {2,}', ' ', paragraph)
445 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
446 returnstring += paragraph + '\n\n'
447 return returnstring.rstrip('\n')
450 def apknameinfo(filename):
451 filename = os.path.basename(filename)
452 m = apk_regex.match(filename)
454 result = (m.group(1), m.group(2))
455 except AttributeError:
456 raise FDroidException("Invalid apk name: %s" % filename)
460 def getapkname(app, build):
461 return "%s_%s.apk" % (app.id, build.vercode)
464 def getsrcname(app, build):
465 return "%s_%s_src.tar.gz" % (app.id, build.vercode)
477 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
480 def getvcs(vcstype, remote, local):
482 return vcs_git(remote, local)
483 if vcstype == 'git-svn':
484 return vcs_gitsvn(remote, local)
486 return vcs_hg(remote, local)
488 return vcs_bzr(remote, local)
489 if vcstype == 'srclib':
490 if local != os.path.join('build', 'srclib', remote):
491 raise VCSException("Error: srclib paths are hard-coded!")
492 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
494 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
495 raise VCSException("Invalid vcs type " + vcstype)
498 def getsrclibvcs(name):
499 if name not in fdroidserver.metadata.srclibs:
500 raise VCSException("Missing srclib " + name)
501 return fdroidserver.metadata.srclibs[name]['Repo Type']
506 def __init__(self, remote, local):
508 # svn, git-svn and bzr may require auth
510 if self.repotype() in ('git-svn', 'bzr'):
512 if self.repotype == 'git-svn':
513 raise VCSException("Authentication is not supported for git-svn")
514 self.username, remote = remote.split('@')
515 if ':' not in self.username:
516 raise VCSException("Password required with username")
517 self.username, self.password = self.username.split(':')
521 self.clone_failed = False
522 self.refreshed = False
528 # Take the local repository to a clean version of the given revision, which
529 # is specificed in the VCS's native format. Beforehand, the repository can
530 # be dirty, or even non-existent. If the repository does already exist
531 # locally, it will be updated from the origin, but only once in the
532 # lifetime of the vcs object.
533 # None is acceptable for 'rev' if you know you are cloning a clean copy of
534 # the repo - otherwise it must specify a valid revision.
535 def gotorevision(self, rev, refresh=True):
537 if self.clone_failed:
538 raise VCSException("Downloading the repository already failed once, not trying again.")
540 # The .fdroidvcs-id file for a repo tells us what VCS type
541 # and remote that directory was created from, allowing us to drop it
542 # automatically if either of those things changes.
543 fdpath = os.path.join(self.local, '..',
544 '.fdroidvcs-' + os.path.basename(self.local))
545 fdpath = os.path.normpath(fdpath)
546 cdata = self.repotype() + ' ' + self.remote
549 if os.path.exists(self.local):
550 if os.path.exists(fdpath):
551 with open(fdpath, 'r') as f:
552 fsdata = f.read().strip()
557 logging.info("Repository details for %s changed - deleting" % (
561 logging.info("Repository details for %s missing - deleting" % (
564 shutil.rmtree(self.local)
568 self.refreshed = True
571 self.gotorevisionx(rev)
572 except FDroidException as e:
575 # If necessary, write the .fdroidvcs file.
576 if writeback and not self.clone_failed:
577 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
578 with open(fdpath, 'w+') as f:
584 # Derived classes need to implement this. It's called once basic checking
585 # has been performend.
586 def gotorevisionx(self, rev):
587 raise VCSException("This VCS type doesn't define gotorevisionx")
589 # Initialise and update submodules
590 def initsubmodules(self):
591 raise VCSException('Submodules not supported for this vcs type')
593 # Get a list of all known tags
595 if not self._gettags:
596 raise VCSException('gettags not supported for this vcs type')
598 for tag in self._gettags():
599 if re.match('[-A-Za-z0-9_. /]+$', tag):
603 # Get a list of all the known tags, sorted from newest to oldest
604 def latesttags(self):
605 raise VCSException('latesttags not supported for this vcs type')
607 # Get current commit reference (hash, revision, etc)
609 raise VCSException('getref not supported for this vcs type')
611 # Returns the srclib (name, path) used in setting up the current
622 # If the local directory exists, but is somehow not a git repository, git
623 # will traverse up the directory tree until it finds one that is (i.e.
624 # fdroidserver) and then we'll proceed to destroy it! This is called as
627 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
628 result = p.output.rstrip()
629 if not result.endswith(self.local):
630 raise VCSException('Repository mismatch')
632 def gotorevisionx(self, rev):
633 if not os.path.exists(self.local):
635 p = FDroidPopen(['git', 'clone', self.remote, self.local])
636 if p.returncode != 0:
637 self.clone_failed = True
638 raise VCSException("Git clone failed", p.output)
642 # Discard any working tree changes
643 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
644 'git', 'reset', '--hard'], cwd=self.local, output=False)
645 if p.returncode != 0:
646 raise VCSException("Git reset failed", p.output)
647 # Remove untracked files now, in case they're tracked in the target
648 # revision (it happens!)
649 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
650 'git', 'clean', '-dffx'], cwd=self.local, output=False)
651 if p.returncode != 0:
652 raise VCSException("Git clean failed", p.output)
653 if not self.refreshed:
654 # Get latest commits and tags from remote
655 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
656 if p.returncode != 0:
657 raise VCSException("Git fetch failed", p.output)
658 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
659 if p.returncode != 0:
660 raise VCSException("Git fetch failed", p.output)
661 # Recreate origin/HEAD as git clone would do it, in case it disappeared
662 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
663 if p.returncode != 0:
664 lines = p.output.splitlines()
665 if 'Multiple remote HEAD branches' not in lines[0]:
666 raise VCSException("Git remote set-head failed", p.output)
667 branch = lines[1].split(' ')[-1]
668 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
669 if p2.returncode != 0:
670 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
671 self.refreshed = True
672 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
673 # a github repo. Most of the time this is the same as origin/master.
674 rev = rev or 'origin/HEAD'
675 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
676 if p.returncode != 0:
677 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
678 # Get rid of any uncontrolled files left behind
679 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
680 if p.returncode != 0:
681 raise VCSException("Git clean failed", p.output)
683 def initsubmodules(self):
685 submfile = os.path.join(self.local, '.gitmodules')
686 if not os.path.isfile(submfile):
687 raise VCSException("No git submodules available")
689 # fix submodules not accessible without an account and public key auth
690 with open(submfile, 'r') as f:
691 lines = f.readlines()
692 with open(submfile, 'w') as f:
694 if 'git@github.com' in line:
695 line = line.replace('git@github.com:', 'https://github.com/')
696 if 'git@gitlab.com' in line:
697 line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
700 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
701 if p.returncode != 0:
702 raise VCSException("Git submodule sync failed", p.output)
703 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
704 if p.returncode != 0:
705 raise VCSException("Git submodule update failed", p.output)
709 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
710 return p.output.splitlines()
712 tag_format = re.compile(r'tag: ([^),]*)')
714 def latesttags(self):
716 p = FDroidPopen(['git', 'log', '--tags',
717 '--simplify-by-decoration', '--pretty=format:%d'],
718 cwd=self.local, output=False)
720 for line in p.output.splitlines():
721 for tag in self.tag_format.findall(line):
726 class vcs_gitsvn(vcs):
731 # If the local directory exists, but is somehow not a git repository, git
732 # will traverse up the directory tree until it finds one that is (i.e.
733 # fdroidserver) and then we'll proceed to destory it! This is called as
736 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
737 result = p.output.rstrip()
738 if not result.endswith(self.local):
739 raise VCSException('Repository mismatch')
741 def gotorevisionx(self, rev):
742 if not os.path.exists(self.local):
744 gitsvn_args = ['git', 'svn', 'clone']
745 if ';' in self.remote:
746 remote_split = self.remote.split(';')
747 for i in remote_split[1:]:
748 if i.startswith('trunk='):
749 gitsvn_args.extend(['-T', i[6:]])
750 elif i.startswith('tags='):
751 gitsvn_args.extend(['-t', i[5:]])
752 elif i.startswith('branches='):
753 gitsvn_args.extend(['-b', i[9:]])
754 gitsvn_args.extend([remote_split[0], self.local])
755 p = FDroidPopen(gitsvn_args, output=False)
756 if p.returncode != 0:
757 self.clone_failed = True
758 raise VCSException("Git svn clone failed", p.output)
760 gitsvn_args.extend([self.remote, self.local])
761 p = FDroidPopen(gitsvn_args, output=False)
762 if p.returncode != 0:
763 self.clone_failed = True
764 raise VCSException("Git svn clone failed", p.output)
768 # Discard any working tree changes
769 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
770 if p.returncode != 0:
771 raise VCSException("Git reset failed", p.output)
772 # Remove untracked files now, in case they're tracked in the target
773 # revision (it happens!)
774 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
775 if p.returncode != 0:
776 raise VCSException("Git clean failed", p.output)
777 if not self.refreshed:
778 # Get new commits, branches and tags from repo
779 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
780 if p.returncode != 0:
781 raise VCSException("Git svn fetch failed")
782 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
783 if p.returncode != 0:
784 raise VCSException("Git svn rebase failed", p.output)
785 self.refreshed = True
787 rev = rev or 'master'
789 nospaces_rev = rev.replace(' ', '%20')
790 # Try finding a svn tag
791 for treeish in ['origin/', '']:
792 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
793 if p.returncode == 0:
795 if p.returncode != 0:
796 # No tag found, normal svn rev translation
797 # Translate svn rev into git format
798 rev_split = rev.split('/')
801 for treeish in ['origin/', '']:
802 if len(rev_split) > 1:
803 treeish += rev_split[0]
804 svn_rev = rev_split[1]
807 # if no branch is specified, then assume trunk (i.e. 'master' branch):
811 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
813 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
814 git_rev = p.output.rstrip()
816 if p.returncode == 0 and git_rev:
819 if p.returncode != 0 or not git_rev:
820 # Try a plain git checkout as a last resort
821 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
822 if p.returncode != 0:
823 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
825 # Check out the git rev equivalent to the svn rev
826 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
827 if p.returncode != 0:
828 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
830 # Get rid of any uncontrolled files left behind
831 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
832 if p.returncode != 0:
833 raise VCSException("Git clean failed", p.output)
837 for treeish in ['origin/', '']:
838 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
844 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
845 if p.returncode != 0:
847 return p.output.strip()
855 def gotorevisionx(self, rev):
856 if not os.path.exists(self.local):
857 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
858 if p.returncode != 0:
859 self.clone_failed = True
860 raise VCSException("Hg clone failed", p.output)
862 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
863 if p.returncode != 0:
864 raise VCSException("Hg status failed", p.output)
865 for line in p.output.splitlines():
866 if not line.startswith('? '):
867 raise VCSException("Unexpected output from hg status -uS: " + line)
868 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
869 if not self.refreshed:
870 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
871 if p.returncode != 0:
872 raise VCSException("Hg pull failed", p.output)
873 self.refreshed = True
875 rev = rev or 'default'
878 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
879 if p.returncode != 0:
880 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
881 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
882 # Also delete untracked files, we have to enable purge extension for that:
883 if "'purge' is provided by the following extension" in p.output:
884 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
885 myfile.write("\n[extensions]\nhgext.purge=\n")
886 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
887 if p.returncode != 0:
888 raise VCSException("HG purge failed", p.output)
889 elif p.returncode != 0:
890 raise VCSException("HG purge failed", p.output)
893 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
894 return p.output.splitlines()[1:]
902 def gotorevisionx(self, rev):
903 if not os.path.exists(self.local):
904 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
905 if p.returncode != 0:
906 self.clone_failed = True
907 raise VCSException("Bzr branch failed", p.output)
909 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
910 if p.returncode != 0:
911 raise VCSException("Bzr revert failed", p.output)
912 if not self.refreshed:
913 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
914 if p.returncode != 0:
915 raise VCSException("Bzr update failed", p.output)
916 self.refreshed = True
918 revargs = list(['-r', rev] if rev else [])
919 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
920 if p.returncode != 0:
921 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
924 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
925 return [tag.split(' ')[0].strip() for tag in
926 p.output.splitlines()]
929 def unescape_string(string):
932 if string[0] == '"' and string[-1] == '"':
935 return string.replace("\\'", "'")
938 def retrieve_string(app_dir, string, xmlfiles=None):
940 if not string.startswith('@string/'):
941 return unescape_string(string)
946 os.path.join(app_dir, 'res'),
947 os.path.join(app_dir, 'src', 'main', 'res'),
949 for r, d, f in os.walk(res_dir):
950 if os.path.basename(r) == 'values':
951 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
953 name = string[len('@string/'):]
955 def element_content(element):
956 if element.text is None:
958 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
959 return s.decode('utf-8').strip()
961 for path in xmlfiles:
962 if not os.path.isfile(path):
964 xml = parse_xml(path)
965 element = xml.find('string[@name="' + name + '"]')
966 if element is not None:
967 content = element_content(element)
968 return retrieve_string(app_dir, content, xmlfiles)
973 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
974 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
977 # Return list of existing files that will be used to find the highest vercode
978 def manifest_paths(app_dir, flavours):
980 possible_manifests = \
981 [os.path.join(app_dir, 'AndroidManifest.xml'),
982 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
983 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
984 os.path.join(app_dir, 'build.gradle')]
986 for flavour in flavours:
989 possible_manifests.append(
990 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
992 return [path for path in possible_manifests if os.path.isfile(path)]
995 # Retrieve the package name. Returns the name, or None if not found.
996 def fetch_real_name(app_dir, flavours):
997 for path in manifest_paths(app_dir, flavours):
998 if not has_extension(path, 'xml') or not os.path.isfile(path):
1000 logging.debug("fetch_real_name: Checking manifest at " + path)
1001 xml = parse_xml(path)
1002 app = xml.find('application')
1005 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1007 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1008 result = retrieve_string_singleline(app_dir, label)
1010 result = result.strip()
1015 def get_library_references(root_dir):
1017 proppath = os.path.join(root_dir, 'project.properties')
1018 if not os.path.isfile(proppath):
1020 with open(proppath, 'r', encoding='iso-8859-1') as f:
1022 if not line.startswith('android.library.reference.'):
1024 path = line.split('=')[1].strip()
1025 relpath = os.path.join(root_dir, path)
1026 if not os.path.isdir(relpath):
1028 logging.debug("Found subproject at %s" % path)
1029 libraries.append(path)
1033 def ant_subprojects(root_dir):
1034 subprojects = get_library_references(root_dir)
1035 for subpath in subprojects:
1036 subrelpath = os.path.join(root_dir, subpath)
1037 for p in get_library_references(subrelpath):
1038 relp = os.path.normpath(os.path.join(subpath, p))
1039 if relp not in subprojects:
1040 subprojects.insert(0, relp)
1044 def remove_debuggable_flags(root_dir):
1045 # Remove forced debuggable flags
1046 logging.debug("Removing debuggable flags from %s" % root_dir)
1047 for root, dirs, files in os.walk(root_dir):
1048 if 'AndroidManifest.xml' in files:
1049 regsub_file(r'android:debuggable="[^"]*"',
1051 os.path.join(root, 'AndroidManifest.xml'))
1054 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
1055 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1056 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1059 def app_matches_packagename(app, package):
1062 appid = app.UpdateCheckName or app.id
1063 if appid is None or appid == "Ignore":
1065 return appid == package
1068 # Extract some information from the AndroidManifest.xml at the given path.
1069 # Returns (version, vercode, package), any or all of which might be None.
1070 # All values returned are strings.
1071 def parse_androidmanifests(paths, app):
1073 ignoreversions = app.UpdateCheckIgnore
1074 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1077 return (None, None, None)
1085 if not os.path.isfile(path):
1088 logging.debug("Parsing manifest at {0}".format(path))
1089 gradle = has_extension(path, 'gradle')
1095 with open(path, 'r') as f:
1097 if gradle_comment.match(line):
1099 # Grab first occurence of each to avoid running into
1100 # alternative flavours and builds.
1102 matches = psearch_g(line)
1104 s = matches.group(2)
1105 if app_matches_packagename(app, s):
1108 matches = vnsearch_g(line)
1110 version = matches.group(2)
1112 matches = vcsearch_g(line)
1114 vercode = matches.group(1)
1117 xml = parse_xml(path)
1118 if "package" in xml.attrib:
1119 s = xml.attrib["package"]
1120 if app_matches_packagename(app, s):
1122 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1123 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1124 base_dir = os.path.dirname(path)
1125 version = retrieve_string_singleline(base_dir, version)
1126 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1127 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1128 if string_is_integer(a):
1131 logging.warning("Problem with xml at {0}".format(path))
1133 # Remember package name, may be defined separately from version+vercode
1135 package = max_package
1137 logging.debug("..got package={0}, version={1}, vercode={2}"
1138 .format(package, version, vercode))
1140 # Always grab the package name and version name in case they are not
1141 # together with the highest version code
1142 if max_package is None and package is not None:
1143 max_package = package
1144 if max_version is None and version is not None:
1145 max_version = version
1147 if max_vercode is None or (vercode is not None and vercode > max_vercode):
1148 if not ignoresearch or not ignoresearch(version):
1149 if version is not None:
1150 max_version = version
1151 if vercode is not None:
1152 max_vercode = vercode
1153 if package is not None:
1154 max_package = package
1156 max_version = "Ignore"
1158 if max_version is None:
1159 max_version = "Unknown"
1161 if max_package and not is_valid_package_name(max_package):
1162 raise FDroidException("Invalid package name {0}".format(max_package))
1164 return (max_version, max_vercode, max_package)
1167 def is_valid_package_name(name):
1168 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1171 class FDroidException(Exception):
1173 def __init__(self, value, detail=None):
1175 self.detail = detail
1177 def shortened_detail(self):
1178 if len(self.detail) < 16000:
1180 return '[...]\n' + self.detail[-16000:]
1182 def get_wikitext(self):
1183 ret = repr(self.value) + "\n"
1186 ret += "<pre>\n" + self.shortened_detail() + "</pre>\n"
1192 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1196 class VCSException(FDroidException):
1200 class BuildException(FDroidException):
1204 # Get the specified source library.
1205 # Returns the path to it. Normally this is the path to be used when referencing
1206 # it, which may be a subdirectory of the actual project. If you want the base
1207 # directory of the project, pass 'basepath=True'.
1208 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1209 raw=False, prepare=True, preponly=False, refresh=True,
1218 name, ref = spec.split('@')
1220 number, name = name.split(':', 1)
1222 name, subdir = name.split('/', 1)
1224 if name not in fdroidserver.metadata.srclibs:
1225 raise VCSException('srclib ' + name + ' not found.')
1227 srclib = fdroidserver.metadata.srclibs[name]
1229 sdir = os.path.join(srclib_dir, name)
1232 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1233 vcs.srclib = (name, number, sdir)
1235 vcs.gotorevision(ref, refresh)
1242 libdir = os.path.join(sdir, subdir)
1243 elif srclib["Subdir"]:
1244 for subdir in srclib["Subdir"]:
1245 libdir_candidate = os.path.join(sdir, subdir)
1246 if os.path.exists(libdir_candidate):
1247 libdir = libdir_candidate
1253 remove_signing_keys(sdir)
1254 remove_debuggable_flags(sdir)
1258 if srclib["Prepare"]:
1259 cmd = replace_config_vars(srclib["Prepare"], build)
1261 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1262 if p.returncode != 0:
1263 raise BuildException("Error running prepare command for srclib %s"
1269 return (name, number, libdir)
1271 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1274 # Prepare the source code for a particular build
1275 # 'vcs' - the appropriate vcs object for the application
1276 # 'app' - the application details from the metadata
1277 # 'build' - the build details from the metadata
1278 # 'build_dir' - the path to the build directory, usually
1280 # 'srclib_dir' - the path to the source libraries directory, usually
1282 # 'extlib_dir' - the path to the external libraries directory, usually
1284 # Returns the (root, srclibpaths) where:
1285 # 'root' is the root directory, which may be the same as 'build_dir' or may
1286 # be a subdirectory of it.
1287 # 'srclibpaths' is information on the srclibs being used
1288 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1290 # Optionally, the actual app source can be in a subdirectory
1292 root_dir = os.path.join(build_dir, build.subdir)
1294 root_dir = build_dir
1296 # Get a working copy of the right revision
1297 logging.info("Getting source for revision " + build.commit)
1298 vcs.gotorevision(build.commit, refresh)
1300 # Initialise submodules if required
1301 if build.submodules:
1302 logging.info("Initialising submodules")
1303 vcs.initsubmodules()
1305 # Check that a subdir (if we're using one) exists. This has to happen
1306 # after the checkout, since it might not exist elsewhere
1307 if not os.path.exists(root_dir):
1308 raise BuildException('Missing subdir ' + root_dir)
1310 # Run an init command if one is required
1312 cmd = replace_config_vars(build.init, build)
1313 logging.info("Running 'init' commands in %s" % root_dir)
1315 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1316 if p.returncode != 0:
1317 raise BuildException("Error running init command for %s:%s" %
1318 (app.id, build.version), p.output)
1320 # Apply patches if any
1322 logging.info("Applying patches")
1323 for patch in build.patch:
1324 patch = patch.strip()
1325 logging.info("Applying " + patch)
1326 patch_path = os.path.join('metadata', app.id, patch)
1327 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1328 if p.returncode != 0:
1329 raise BuildException("Failed to apply patch %s" % patch_path)
1331 # Get required source libraries
1334 logging.info("Collecting source libraries")
1335 for lib in build.srclibs:
1336 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1337 refresh=refresh, build=build))
1339 for name, number, libpath in srclibpaths:
1340 place_srclib(root_dir, int(number) if number else None, libpath)
1342 basesrclib = vcs.getsrclib()
1343 # If one was used for the main source, add that too.
1345 srclibpaths.append(basesrclib)
1347 # Update the local.properties file
1348 localprops = [os.path.join(build_dir, 'local.properties')]
1350 parts = build.subdir.split(os.sep)
1353 cur = os.path.join(cur, d)
1354 localprops += [os.path.join(cur, 'local.properties')]
1355 for path in localprops:
1357 if os.path.isfile(path):
1358 logging.info("Updating local.properties file at %s" % path)
1359 with open(path, 'r', encoding='iso-8859-1') as f:
1363 logging.info("Creating local.properties file at %s" % path)
1364 # Fix old-fashioned 'sdk-location' by copying
1365 # from sdk.dir, if necessary
1367 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1368 re.S | re.M).group(1)
1369 props += "sdk-location=%s\n" % sdkloc
1371 props += "sdk.dir=%s\n" % config['sdk_path']
1372 props += "sdk-location=%s\n" % config['sdk_path']
1373 ndk_path = build.ndk_path()
1374 # if for any reason the path isn't valid or the directory
1375 # doesn't exist, some versions of Gradle will error with a
1376 # cryptic message (even if the NDK is not even necessary).
1377 # https://gitlab.com/fdroid/fdroidserver/issues/171
1378 if ndk_path and os.path.exists(ndk_path):
1380 props += "ndk.dir=%s\n" % ndk_path
1381 props += "ndk-location=%s\n" % ndk_path
1382 # Add java.encoding if necessary
1384 props += "java.encoding=%s\n" % build.encoding
1385 with open(path, 'w', encoding='iso-8859-1') as f:
1389 if build.build_method() == 'gradle':
1390 flavours = build.gradle
1393 n = build.target.split('-')[1]
1394 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1395 r'compileSdkVersion %s' % n,
1396 os.path.join(root_dir, 'build.gradle'))
1398 # Remove forced debuggable flags
1399 remove_debuggable_flags(root_dir)
1401 # Insert version code and number into the manifest if necessary
1402 if build.forceversion:
1403 logging.info("Changing the version name")
1404 for path in manifest_paths(root_dir, flavours):
1405 if not os.path.isfile(path):
1407 if has_extension(path, 'xml'):
1408 regsub_file(r'android:versionName="[^"]*"',
1409 r'android:versionName="%s"' % build.version,
1411 elif has_extension(path, 'gradle'):
1412 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1413 r"""\1versionName '%s'""" % build.version,
1416 if build.forcevercode:
1417 logging.info("Changing the version code")
1418 for path in manifest_paths(root_dir, flavours):
1419 if not os.path.isfile(path):
1421 if has_extension(path, 'xml'):
1422 regsub_file(r'android:versionCode="[^"]*"',
1423 r'android:versionCode="%s"' % build.vercode,
1425 elif has_extension(path, 'gradle'):
1426 regsub_file(r'versionCode[ =]+[0-9]+',
1427 r'versionCode %s' % build.vercode,
1430 # Delete unwanted files
1432 logging.info("Removing specified files")
1433 for part in getpaths(build_dir, build.rm):
1434 dest = os.path.join(build_dir, part)
1435 logging.info("Removing {0}".format(part))
1436 if os.path.lexists(dest):
1437 if os.path.islink(dest):
1438 FDroidPopen(['unlink', dest], output=False)
1440 FDroidPopen(['rm', '-rf', dest], output=False)
1442 logging.info("...but it didn't exist")
1444 remove_signing_keys(build_dir)
1446 # Add required external libraries
1448 logging.info("Collecting prebuilt libraries")
1449 libsdir = os.path.join(root_dir, 'libs')
1450 if not os.path.exists(libsdir):
1452 for lib in build.extlibs:
1454 logging.info("...installing extlib {0}".format(lib))
1455 libf = os.path.basename(lib)
1456 libsrc = os.path.join(extlib_dir, lib)
1457 if not os.path.exists(libsrc):
1458 raise BuildException("Missing extlib file {0}".format(libsrc))
1459 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1461 # Run a pre-build command if one is required
1463 logging.info("Running 'prebuild' commands in %s" % root_dir)
1465 cmd = replace_config_vars(build.prebuild, build)
1467 # Substitute source library paths into prebuild commands
1468 for name, number, libpath in srclibpaths:
1469 libpath = os.path.relpath(libpath, root_dir)
1470 cmd = cmd.replace('$$' + name + '$$', libpath)
1472 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1473 if p.returncode != 0:
1474 raise BuildException("Error running prebuild command for %s:%s" %
1475 (app.id, build.version), p.output)
1477 # Generate (or update) the ant build file, build.xml...
1478 if build.build_method() == 'ant' and build.update != ['no']:
1479 parms = ['android', 'update', 'lib-project']
1480 lparms = ['android', 'update', 'project']
1483 parms += ['-t', build.target]
1484 lparms += ['-t', build.target]
1486 update_dirs = build.update
1488 update_dirs = ant_subprojects(root_dir) + ['.']
1490 for d in update_dirs:
1491 subdir = os.path.join(root_dir, d)
1493 logging.debug("Updating main project")
1494 cmd = parms + ['-p', d]
1496 logging.debug("Updating subproject %s" % d)
1497 cmd = lparms + ['-p', d]
1498 p = SdkToolsPopen(cmd, cwd=root_dir)
1499 # Check to see whether an error was returned without a proper exit
1500 # code (this is the case for the 'no target set or target invalid'
1502 if p.returncode != 0 or p.output.startswith("Error: "):
1503 raise BuildException("Failed to update project at %s" % d, p.output)
1504 # Clean update dirs via ant
1506 logging.info("Cleaning subproject %s" % d)
1507 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1509 return (root_dir, srclibpaths)
1512 # Extend via globbing the paths from a field and return them as a map from
1513 # original path to resulting paths
1514 def getpaths_map(build_dir, globpaths):
1518 full_path = os.path.join(build_dir, p)
1519 full_path = os.path.normpath(full_path)
1520 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1522 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1526 # Extend via globbing the paths from a field and return them as a set
1527 def getpaths(build_dir, globpaths):
1528 paths_map = getpaths_map(build_dir, globpaths)
1530 for k, v in paths_map.items():
1537 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1543 self.path = os.path.join('stats', 'known_apks.txt')
1545 if os.path.isfile(self.path):
1546 with open(self.path, 'r', encoding='utf8') as f:
1548 t = line.rstrip().split(' ')
1550 self.apks[t[0]] = (t[1], None)
1552 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1553 self.changed = False
1555 def writeifchanged(self):
1556 if not self.changed:
1559 if not os.path.exists('stats'):
1563 for apk, app in self.apks.items():
1565 line = apk + ' ' + appid
1567 line += ' ' + time.strftime('%Y-%m-%d', added)
1570 with open(self.path, 'w', encoding='utf8') as f:
1571 for line in sorted(lst, key=natural_key):
1572 f.write(line + '\n')
1574 # Record an apk (if it's new, otherwise does nothing)
1575 # Returns the date it was added.
1576 def recordapk(self, apk, app, default_date=None):
1577 if apk not in self.apks:
1578 if default_date is None:
1579 default_date = time.gmtime(time.time())
1580 self.apks[apk] = (app, default_date)
1582 _, added = self.apks[apk]
1585 # Look up information - given the 'apkname', returns (app id, date added/None).
1586 # Or returns None for an unknown apk.
1587 def getapp(self, apkname):
1588 if apkname in self.apks:
1589 return self.apks[apkname]
1592 # Get the most recent 'num' apps added to the repo, as a list of package ids
1593 # with the most recent first.
1594 def getlatest(self, num):
1596 for apk, app in self.apks.items():
1600 if apps[appid] > added:
1604 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1605 lst = [app for app, _ in sortedapps]
1610 def isApkDebuggable(apkfile, config):
1611 """Returns True if the given apk file is debuggable
1613 :param apkfile: full path to the apk to check"""
1615 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1617 if p.returncode != 0:
1618 logging.critical("Failed to get apk manifest information")
1620 for line in p.output.splitlines():
1621 if 'android:debuggable' in line and not line.endswith('0x0'):
1628 self.returncode = None
1632 def SdkToolsPopen(commands, cwd=None, output=True):
1634 if cmd not in config:
1635 config[cmd] = find_sdk_tools_cmd(commands[0])
1636 abscmd = config[cmd]
1638 logging.critical("Could not find '%s' on your system" % cmd)
1640 return FDroidPopen([abscmd] + commands[1:],
1641 cwd=cwd, output=output)
1644 def FDroidPopenBytes(commands, cwd=None, output=True, stderr_to_stdout=True):
1646 Run a command and capture the possibly huge output as bytes.
1648 :param commands: command and argument list like in subprocess.Popen
1649 :param cwd: optionally specifies a working directory
1650 :returns: A PopenResult.
1655 set_FDroidPopen_env()
1658 cwd = os.path.normpath(cwd)
1659 logging.debug("Directory: %s" % cwd)
1660 logging.debug("> %s" % ' '.join(commands))
1662 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
1663 result = PopenResult()
1666 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
1667 stdout=subprocess.PIPE, stderr=stderr_param)
1668 except OSError as e:
1669 raise BuildException("OSError while trying to execute " +
1670 ' '.join(commands) + ': ' + str(e))
1672 if not stderr_to_stdout and options.verbose:
1673 stderr_queue = Queue()
1674 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
1676 while not stderr_reader.eof():
1677 while not stderr_queue.empty():
1678 line = stderr_queue.get()
1679 sys.stderr.buffer.write(line)
1684 stdout_queue = Queue()
1685 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1688 # Check the queue for output (until there is no more to get)
1689 while not stdout_reader.eof():
1690 while not stdout_queue.empty():
1691 line = stdout_queue.get()
1692 if output and options.verbose:
1693 # Output directly to console
1694 sys.stderr.buffer.write(line)
1700 result.returncode = p.wait()
1701 result.output = buf.getvalue()
1706 def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
1708 Run a command and capture the possibly huge output as a str.
1710 :param commands: command and argument list like in subprocess.Popen
1711 :param cwd: optionally specifies a working directory
1712 :returns: A PopenResult.
1714 result = FDroidPopenBytes(commands, cwd, output, stderr_to_stdout)
1715 result.output = result.output.decode('utf-8')
1719 gradle_comment = re.compile(r'[ ]*//')
1720 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1721 gradle_line_matches = [
1722 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1723 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1724 re.compile(r'.*\.readLine\(.*'),
1728 def remove_signing_keys(build_dir):
1729 for root, dirs, files in os.walk(build_dir):
1730 if 'build.gradle' in files:
1731 path = os.path.join(root, 'build.gradle')
1733 with open(path, "r", encoding='utf8') as o:
1734 lines = o.readlines()
1740 with open(path, "w", encoding='utf8') as o:
1741 while i < len(lines):
1744 while line.endswith('\\\n'):
1745 line = line.rstrip('\\\n') + lines[i]
1748 if gradle_comment.match(line):
1753 opened += line.count('{')
1754 opened -= line.count('}')
1757 if gradle_signing_configs.match(line):
1762 if any(s.match(line) for s in gradle_line_matches):
1770 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1773 'project.properties',
1775 'default.properties',
1776 'ant.properties', ]:
1777 if propfile in files:
1778 path = os.path.join(root, propfile)
1780 with open(path, "r", encoding='iso-8859-1') as o:
1781 lines = o.readlines()
1785 with open(path, "w", encoding='iso-8859-1') as o:
1787 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1794 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1797 def set_FDroidPopen_env(build=None):
1799 set up the environment variables for the build environment
1801 There is only a weak standard, the variables used by gradle, so also set
1802 up the most commonly used environment variables for SDK and NDK. Also, if
1803 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
1805 global env, orig_path
1809 orig_path = env['PATH']
1810 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
1811 env[n] = config['sdk_path']
1812 for k, v in config['java_paths'].items():
1813 env['JAVA%s_HOME' % k] = v
1815 missinglocale = True
1816 for k, v in env.items():
1817 if k == 'LANG' and v != 'C':
1818 missinglocale = False
1820 missinglocale = False
1822 env['LANG'] = 'en_US.UTF-8'
1824 if build is not None:
1825 path = build.ndk_path()
1826 paths = orig_path.split(os.pathsep)
1827 if path not in paths:
1828 paths = [path] + paths
1829 env['PATH'] = os.pathsep.join(paths)
1830 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
1831 env[n] = build.ndk_path()
1834 def replace_config_vars(cmd, build):
1835 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1836 cmd = cmd.replace('$$NDK$$', build.ndk_path())
1837 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1838 if build is not None:
1839 cmd = cmd.replace('$$COMMIT$$', build.commit)
1840 cmd = cmd.replace('$$VERSION$$', build.version)
1841 cmd = cmd.replace('$$VERCODE$$', build.vercode)
1845 def place_srclib(root_dir, number, libpath):
1848 relpath = os.path.relpath(libpath, root_dir)
1849 proppath = os.path.join(root_dir, 'project.properties')
1852 if os.path.isfile(proppath):
1853 with open(proppath, "r", encoding='iso-8859-1') as o:
1854 lines = o.readlines()
1856 with open(proppath, "w", encoding='iso-8859-1') as o:
1859 if line.startswith('android.library.reference.%d=' % number):
1860 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1865 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1867 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
1870 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
1871 """Verify that two apks are the same
1873 One of the inputs is signed, the other is unsigned. The signature metadata
1874 is transferred from the signed to the unsigned apk, and then jarsigner is
1875 used to verify that the signature from the signed apk is also varlid for
1877 :param signed_apk: Path to a signed apk file
1878 :param unsigned_apk: Path to an unsigned apk file expected to match it
1879 :param tmp_dir: Path to directory for temporary files
1880 :returns: None if the verification is successful, otherwise a string
1881 describing what went wrong.
1883 with ZipFile(signed_apk) as signed_apk_as_zip:
1884 meta_inf_files = ['META-INF/MANIFEST.MF']
1885 for f in signed_apk_as_zip.namelist():
1886 if apk_sigfile.match(f):
1887 meta_inf_files.append(f)
1888 if len(meta_inf_files) < 3:
1889 return "Signature files missing from {0}".format(signed_apk)
1890 signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
1891 with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
1892 for meta_inf_file in meta_inf_files:
1893 unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
1895 if subprocess.call([config['jarsigner'], '-verify', unsigned_apk]) != 0:
1896 logging.info("...NOT verified - {0}".format(signed_apk))
1897 return compare_apks(signed_apk, unsigned_apk, tmp_dir)
1898 logging.info("...successfully verified")
1901 apk_badchars = re.compile('''[/ :;'"]''')
1904 def compare_apks(apk1, apk2, tmp_dir):
1907 Returns None if the apk content is the same (apart from the signing key),
1908 otherwise a string describing what's different, or what went wrong when
1909 trying to do the comparison.
1912 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
1913 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
1914 for d in [apk1dir, apk2dir]:
1915 if os.path.exists(d):
1918 os.mkdir(os.path.join(d, 'jar-xf'))
1920 if subprocess.call(['jar', 'xf',
1921 os.path.abspath(apk1)],
1922 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
1923 return("Failed to unpack " + apk1)
1924 if subprocess.call(['jar', 'xf',
1925 os.path.abspath(apk2)],
1926 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
1927 return("Failed to unpack " + apk2)
1929 # try to find apktool in the path, if it hasn't been manually configed
1930 if 'apktool' not in config:
1931 tmp = find_command('apktool')
1933 config['apktool'] = tmp
1934 if 'apktool' in config:
1935 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
1937 return("Failed to unpack " + apk1)
1938 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
1940 return("Failed to unpack " + apk2)
1942 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
1943 lines = p.output.splitlines()
1944 if len(lines) != 1 or 'META-INF' not in lines[0]:
1945 meld = find_command('meld')
1946 if meld is not None:
1947 p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
1948 return("Unexpected diff output - " + p.output)
1950 # since everything verifies, delete the comparison to keep cruft down
1951 shutil.rmtree(apk1dir)
1952 shutil.rmtree(apk2dir)
1954 # If we get here, it seems like they're the same!
1958 def find_command(command):
1959 '''find the full path of a command, or None if it can't be found in the PATH'''
1962 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
1964 fpath, fname = os.path.split(command)
1969 for path in os.environ["PATH"].split(os.pathsep):
1970 path = path.strip('"')
1971 exe_file = os.path.join(path, command)
1972 if is_exe(exe_file):
1979 '''generate a random password for when generating keys'''
1980 h = hashlib.sha256()
1981 h.update(os.urandom(16)) # salt
1982 h.update(socket.getfqdn().encode('utf-8'))
1983 passwd = base64.b64encode(h.digest()).strip()
1984 return passwd.decode('utf-8')
1987 def genkeystore(localconfig):
1988 '''Generate a new key with random passwords and add it to new keystore'''
1989 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
1990 keystoredir = os.path.dirname(localconfig['keystore'])
1991 if keystoredir is None or keystoredir == '':
1992 keystoredir = os.path.join(os.getcwd(), keystoredir)
1993 if not os.path.exists(keystoredir):
1994 os.makedirs(keystoredir, mode=0o700)
1996 write_password_file("keystorepass", localconfig['keystorepass'])
1997 write_password_file("keypass", localconfig['keypass'])
1998 p = FDroidPopen([config['keytool'], '-genkey',
1999 '-keystore', localconfig['keystore'],
2000 '-alias', localconfig['repo_keyalias'],
2001 '-keyalg', 'RSA', '-keysize', '4096',
2002 '-sigalg', 'SHA256withRSA',
2003 '-validity', '10000',
2004 '-storepass:file', config['keystorepassfile'],
2005 '-keypass:file', config['keypassfile'],
2006 '-dname', localconfig['keydname']])
2007 # TODO keypass should be sent via stdin
2008 if p.returncode != 0:
2009 raise BuildException("Failed to generate key", p.output)
2010 os.chmod(localconfig['keystore'], 0o0600)
2011 # now show the lovely key that was just generated
2012 p = FDroidPopen([config['keytool'], '-list', '-v',
2013 '-keystore', localconfig['keystore'],
2014 '-alias', localconfig['repo_keyalias'],
2015 '-storepass:file', config['keystorepassfile']])
2016 logging.info(p.output.strip() + '\n\n')
2019 def write_to_config(thisconfig, key, value=None):
2020 '''write a key/value to the local config.py'''
2022 origkey = key + '_orig'
2023 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2024 with open('config.py', 'r', encoding='utf8') as f:
2026 pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
2027 repl = '\n' + key + ' = "' + value + '"'
2028 data = re.sub(pattern, repl, data)
2029 # if this key is not in the file, append it
2030 if not re.match('\s*' + key + '\s*=\s*"', data):
2032 # make sure the file ends with a carraige return
2033 if not re.match('\n$', data):
2035 with open('config.py', 'w', encoding='utf8') as f:
2039 def parse_xml(path):
2040 return XMLElementTree.parse(path).getroot()
2043 def string_is_integer(string):
2051 def get_per_app_repos():
2052 '''per-app repos are dirs named with the packageName of a single app'''
2054 # Android packageNames are Java packages, they may contain uppercase or
2055 # lowercase letters ('A' through 'Z'), numbers, and underscores
2056 # ('_'). However, individual package name parts may only start with
2057 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2058 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2061 for root, dirs, files in os.walk(os.getcwd()):
2063 print('checking', root, 'for', d)
2064 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2065 # standard parts of an fdroid repo, so never packageNames
2068 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):