3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
37 import xml.etree.ElementTree as XMLElementTree
39 from queue import Queue
41 from zipfile import ZipFile
43 import fdroidserver.metadata
44 from .asynchronousfilereader import AsynchronousFileReader
47 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
56 'sdk_path': "$ANDROID_HOME",
59 'r10e': "$ANDROID_NDK",
61 'build_tools': "23.0.3",
66 'accepted_formats': ['txt', 'yml'],
67 'sync_from_local_copy_dir': False,
68 'per_app_repos': False,
69 'make_current_version_link': True,
70 'current_version_name_source': 'Name',
71 'update_stats': False,
75 'stats_to_carbon': False,
77 'build_server_always': False,
78 'keystore': 'keystore.jks',
79 'smartcardoptions': [],
85 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
86 'repo_name': "My First FDroid Repo Demo",
87 'repo_icon': "fdroid-icon.png",
88 'repo_description': '''
89 This is a repository of apps to be used with FDroid. Applications in this
90 repository are either official binaries built by the original application
91 developers, or are binaries built from source by the admin of f-droid.org
92 using the tools on https://gitlab.com/u/fdroid.
98 def setup_global_opts(parser):
99 parser.add_argument("-v", "--verbose", action="store_true", default=False,
100 help="Spew out even more information than normal")
101 parser.add_argument("-q", "--quiet", action="store_true", default=False,
102 help="Restrict output to warnings and errors")
105 def fill_config_defaults(thisconfig):
106 for k, v in default_config.items():
107 if k not in thisconfig:
110 # Expand paths (~users and $vars)
111 def expand_path(path):
115 path = os.path.expanduser(path)
116 path = os.path.expandvars(path)
121 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
126 thisconfig[k + '_orig'] = v
128 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
129 if thisconfig['java_paths'] is None:
130 thisconfig['java_paths'] = dict()
132 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
133 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
134 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
135 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
136 if os.getenv('JAVA_HOME') is not None:
137 pathlist += os.getenv('JAVA_HOME')
138 if os.getenv('PROGRAMFILES') is not None:
139 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
140 for d in sorted(pathlist):
141 if os.path.islink(d):
143 j = os.path.basename(d)
144 # the last one found will be the canonical one, so order appropriately
146 r'^1\.([6-9])\.0\.jdk$', # OSX
147 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
148 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
149 r'^jdk([6-9])-openjdk$', # Arch
150 r'^java-([6-9])-openjdk$', # Arch
151 r'^java-([6-9])-jdk$', # Arch (oracle)
152 r'^java-1\.([6-9])\.0-.*$', # RedHat
153 r'^java-([6-9])-oracle$', # Debian WebUpd8
154 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
155 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
157 m = re.match(regex, j)
160 osxhome = os.path.join(d, 'Contents', 'Home')
161 if os.path.exists(osxhome):
162 thisconfig['java_paths'][m.group(1)] = osxhome
164 thisconfig['java_paths'][m.group(1)] = d
166 for java_version in ('7', '8', '9'):
167 if java_version not in thisconfig['java_paths']:
169 java_home = thisconfig['java_paths'][java_version]
170 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
171 if os.path.exists(jarsigner):
172 thisconfig['jarsigner'] = jarsigner
173 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
174 break # Java7 is preferred, so quit if found
176 for k in ['ndk_paths', 'java_paths']:
182 thisconfig[k][k2] = exp
183 thisconfig[k][k2 + '_orig'] = v
186 def regsub_file(pattern, repl, path):
187 with open(path, 'rb') as f:
189 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
190 with open(path, 'wb') as f:
194 def read_config(opts, config_file='config.py'):
195 """Read the repository config
197 The config is read from config_file, which is in the current
198 directory when any of the repo management commands are used. If
199 there is a local metadata file in the git repo, then config.py is
200 not required, just use defaults.
203 global config, options
205 if config is not None:
212 if os.path.isfile(config_file):
213 logging.debug("Reading %s" % config_file)
214 with io.open(config_file, "rb") as f:
215 code = compile(f.read(), config_file, 'exec')
216 exec(code, None, config)
217 elif len(get_local_metadata_files()) == 0:
218 logging.critical("Missing config file - is this a repo directory?")
221 # smartcardoptions must be a list since its command line args for Popen
222 if 'smartcardoptions' in config:
223 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
224 elif 'keystore' in config and config['keystore'] == 'NONE':
225 # keystore='NONE' means use smartcard, these are required defaults
226 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
227 'SunPKCS11-OpenSC', '-providerClass',
228 'sun.security.pkcs11.SunPKCS11',
229 '-providerArg', 'opensc-fdroid.cfg']
231 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
232 st = os.stat(config_file)
233 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
234 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
236 fill_config_defaults(config)
238 for k in ["keystorepass", "keypass"]:
240 write_password_file(k)
242 for k in ["repo_description", "archive_description"]:
244 config[k] = clean_description(config[k])
246 if 'serverwebroot' in config:
247 if isinstance(config['serverwebroot'], str):
248 roots = [config['serverwebroot']]
249 elif all(isinstance(item, str) for item in config['serverwebroot']):
250 roots = config['serverwebroot']
252 raise TypeError('only accepts strings, lists, and tuples')
254 for rootstr in roots:
255 # since this is used with rsync, where trailing slashes have
256 # meaning, ensure there is always a trailing slash
257 if rootstr[-1] != '/':
259 rootlist.append(rootstr.replace('//', '/'))
260 config['serverwebroot'] = rootlist
265 def find_sdk_tools_cmd(cmd):
266 '''find a working path to a tool from the Android SDK'''
269 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
270 # try to find a working path to this command, in all the recent possible paths
271 if 'build_tools' in config:
272 build_tools = os.path.join(config['sdk_path'], 'build-tools')
273 # if 'build_tools' was manually set and exists, check only that one
274 configed_build_tools = os.path.join(build_tools, config['build_tools'])
275 if os.path.exists(configed_build_tools):
276 tooldirs.append(configed_build_tools)
278 # no configed version, so hunt known paths for it
279 for f in sorted(os.listdir(build_tools), reverse=True):
280 if os.path.isdir(os.path.join(build_tools, f)):
281 tooldirs.append(os.path.join(build_tools, f))
282 tooldirs.append(build_tools)
283 sdk_tools = os.path.join(config['sdk_path'], 'tools')
284 if os.path.exists(sdk_tools):
285 tooldirs.append(sdk_tools)
286 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
287 if os.path.exists(sdk_platform_tools):
288 tooldirs.append(sdk_platform_tools)
289 tooldirs.append('/usr/bin')
291 if os.path.isfile(os.path.join(d, cmd)):
292 return os.path.join(d, cmd)
293 # did not find the command, exit with error message
294 ensure_build_tools_exists(config)
297 def test_sdk_exists(thisconfig):
298 if 'sdk_path' not in thisconfig:
299 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
302 logging.error("'sdk_path' not set in config.py!")
304 if thisconfig['sdk_path'] == default_config['sdk_path']:
305 logging.error('No Android SDK found!')
306 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
307 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
309 if not os.path.exists(thisconfig['sdk_path']):
310 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
312 if not os.path.isdir(thisconfig['sdk_path']):
313 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
315 for d in ['build-tools', 'platform-tools', 'tools']:
316 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
317 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
318 thisconfig['sdk_path'], d))
323 def ensure_build_tools_exists(thisconfig):
324 if not test_sdk_exists(thisconfig):
326 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
327 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
328 if not os.path.isdir(versioned_build_tools):
329 logging.critical('Android Build Tools path "'
330 + versioned_build_tools + '" does not exist!')
334 def write_password_file(pwtype, password=None):
336 writes out passwords to a protected file instead of passing passwords as
337 command line argments
339 filename = '.fdroid.' + pwtype + '.txt'
340 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
342 os.write(fd, config[pwtype].encode('utf-8'))
344 os.write(fd, password.encode('utf-8'))
346 config[pwtype + 'file'] = filename
349 def get_local_metadata_files():
350 '''get any metadata files local to an app's source repo
352 This tries to ignore anything that does not count as app metdata,
353 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
356 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
359 # Given the arguments in the form of multiple appid:[vc] strings, this returns
360 # a dictionary with the set of vercodes specified for each package.
361 def read_pkg_args(args, allow_vercodes=False):
368 if allow_vercodes and ':' in p:
369 package, vercode = p.split(':')
371 package, vercode = p, None
372 if package not in vercodes:
373 vercodes[package] = [vercode] if vercode else []
375 elif vercode and vercode not in vercodes[package]:
376 vercodes[package] += [vercode] if vercode else []
381 # On top of what read_pkg_args does, this returns the whole app metadata, but
382 # limiting the builds list to the builds matching the vercodes specified.
383 def read_app_args(args, allapps, allow_vercodes=False):
385 vercodes = read_pkg_args(args, allow_vercodes)
391 for appid, app in allapps.items():
392 if appid in vercodes:
395 if len(apps) != len(vercodes):
398 logging.critical("No such package: %s" % p)
399 raise FDroidException("Found invalid app ids in arguments")
401 raise FDroidException("No packages specified")
404 for appid, app in apps.items():
408 app.builds = [b for b in app.builds if b.vercode in vc]
409 if len(app.builds) != len(vercodes[appid]):
411 allvcs = [b.vercode for b in app.builds]
412 for v in vercodes[appid]:
414 logging.critical("No such vercode %s for app %s" % (v, appid))
417 raise FDroidException("Found invalid vercodes for some apps")
422 def get_extension(filename):
423 base, ext = os.path.splitext(filename)
426 return base, ext.lower()[1:]
429 def has_extension(filename, ext):
430 _, f_ext = get_extension(filename)
434 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
437 def clean_description(description):
438 'Remove unneeded newlines and spaces from a block of description text'
440 # this is split up by paragraph to make removing the newlines easier
441 for paragraph in re.split(r'\n\n', description):
442 paragraph = re.sub('\r', '', paragraph)
443 paragraph = re.sub('\n', ' ', paragraph)
444 paragraph = re.sub(' {2,}', ' ', paragraph)
445 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
446 returnstring += paragraph + '\n\n'
447 return returnstring.rstrip('\n')
450 def apknameinfo(filename):
451 filename = os.path.basename(filename)
452 m = apk_regex.match(filename)
454 result = (m.group(1), m.group(2))
455 except AttributeError:
456 raise FDroidException("Invalid apk name: %s" % filename)
460 def getapkname(app, build):
461 return "%s_%s.apk" % (app.id, build.vercode)
464 def getsrcname(app, build):
465 return "%s_%s_src.tar.gz" % (app.id, build.vercode)
477 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
480 def getvcs(vcstype, remote, local):
482 return vcs_git(remote, local)
483 if vcstype == 'git-svn':
484 return vcs_gitsvn(remote, local)
486 return vcs_hg(remote, local)
488 return vcs_bzr(remote, local)
489 if vcstype == 'srclib':
490 if local != os.path.join('build', 'srclib', remote):
491 raise VCSException("Error: srclib paths are hard-coded!")
492 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
494 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
495 raise VCSException("Invalid vcs type " + vcstype)
498 def getsrclibvcs(name):
499 if name not in fdroidserver.metadata.srclibs:
500 raise VCSException("Missing srclib " + name)
501 return fdroidserver.metadata.srclibs[name]['Repo Type']
506 def __init__(self, remote, local):
508 # svn, git-svn and bzr may require auth
510 if self.repotype() in ('git-svn', 'bzr'):
512 if self.repotype == 'git-svn':
513 raise VCSException("Authentication is not supported for git-svn")
514 self.username, remote = remote.split('@')
515 if ':' not in self.username:
516 raise VCSException("Password required with username")
517 self.username, self.password = self.username.split(':')
521 self.clone_failed = False
522 self.refreshed = False
528 # Take the local repository to a clean version of the given revision, which
529 # is specificed in the VCS's native format. Beforehand, the repository can
530 # be dirty, or even non-existent. If the repository does already exist
531 # locally, it will be updated from the origin, but only once in the
532 # lifetime of the vcs object.
533 # None is acceptable for 'rev' if you know you are cloning a clean copy of
534 # the repo - otherwise it must specify a valid revision.
535 def gotorevision(self, rev, refresh=True):
537 if self.clone_failed:
538 raise VCSException("Downloading the repository already failed once, not trying again.")
540 # The .fdroidvcs-id file for a repo tells us what VCS type
541 # and remote that directory was created from, allowing us to drop it
542 # automatically if either of those things changes.
543 fdpath = os.path.join(self.local, '..',
544 '.fdroidvcs-' + os.path.basename(self.local))
545 fdpath = os.path.normpath(fdpath)
546 cdata = self.repotype() + ' ' + self.remote
549 if os.path.exists(self.local):
550 if os.path.exists(fdpath):
551 with open(fdpath, 'r') as f:
552 fsdata = f.read().strip()
557 logging.info("Repository details for %s changed - deleting" % (
561 logging.info("Repository details for %s missing - deleting" % (
564 shutil.rmtree(self.local)
568 self.refreshed = True
571 self.gotorevisionx(rev)
572 except FDroidException as e:
575 # If necessary, write the .fdroidvcs file.
576 if writeback and not self.clone_failed:
577 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
578 with open(fdpath, 'w+') as f:
584 # Derived classes need to implement this. It's called once basic checking
585 # has been performend.
586 def gotorevisionx(self, rev):
587 raise VCSException("This VCS type doesn't define gotorevisionx")
589 # Initialise and update submodules
590 def initsubmodules(self):
591 raise VCSException('Submodules not supported for this vcs type')
593 # Get a list of all known tags
595 if not self._gettags:
596 raise VCSException('gettags not supported for this vcs type')
598 for tag in self._gettags():
599 if re.match('[-A-Za-z0-9_. /]+$', tag):
603 # Get a list of all the known tags, sorted from newest to oldest
604 def latesttags(self):
605 raise VCSException('latesttags not supported for this vcs type')
607 # Get current commit reference (hash, revision, etc)
609 raise VCSException('getref not supported for this vcs type')
611 # Returns the srclib (name, path) used in setting up the current
622 # If the local directory exists, but is somehow not a git repository, git
623 # will traverse up the directory tree until it finds one that is (i.e.
624 # fdroidserver) and then we'll proceed to destroy it! This is called as
627 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
628 result = p.output.rstrip()
629 if not result.endswith(self.local):
630 raise VCSException('Repository mismatch')
632 def gotorevisionx(self, rev):
633 if not os.path.exists(self.local):
635 p = FDroidPopen(['git', 'clone', self.remote, self.local])
636 if p.returncode != 0:
637 self.clone_failed = True
638 raise VCSException("Git clone failed", p.output)
642 # Discard any working tree changes
643 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
644 'git', 'reset', '--hard'], cwd=self.local, output=False)
645 if p.returncode != 0:
646 raise VCSException("Git reset failed", p.output)
647 # Remove untracked files now, in case they're tracked in the target
648 # revision (it happens!)
649 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
650 'git', 'clean', '-dffx'], cwd=self.local, output=False)
651 if p.returncode != 0:
652 raise VCSException("Git clean failed", p.output)
653 if not self.refreshed:
654 # Get latest commits and tags from remote
655 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
656 if p.returncode != 0:
657 raise VCSException("Git fetch failed", p.output)
658 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
659 if p.returncode != 0:
660 raise VCSException("Git fetch failed", p.output)
661 # Recreate origin/HEAD as git clone would do it, in case it disappeared
662 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
663 if p.returncode != 0:
664 lines = p.output.splitlines()
665 if 'Multiple remote HEAD branches' not in lines[0]:
666 raise VCSException("Git remote set-head failed", p.output)
667 branch = lines[1].split(' ')[-1]
668 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
669 if p2.returncode != 0:
670 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
671 self.refreshed = True
672 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
673 # a github repo. Most of the time this is the same as origin/master.
674 rev = rev or 'origin/HEAD'
675 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
676 if p.returncode != 0:
677 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
678 # Get rid of any uncontrolled files left behind
679 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
680 if p.returncode != 0:
681 raise VCSException("Git clean failed", p.output)
683 def initsubmodules(self):
685 submfile = os.path.join(self.local, '.gitmodules')
686 if not os.path.isfile(submfile):
687 raise VCSException("No git submodules available")
689 # fix submodules not accessible without an account and public key auth
690 with open(submfile, 'r') as f:
691 lines = f.readlines()
692 with open(submfile, 'w') as f:
694 if 'git@github.com' in line:
695 line = line.replace('git@github.com:', 'https://github.com/')
696 if 'git@gitlab.com' in line:
697 line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
700 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
701 if p.returncode != 0:
702 raise VCSException("Git submodule sync failed", p.output)
703 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
704 if p.returncode != 0:
705 raise VCSException("Git submodule update failed", p.output)
709 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
710 return p.output.splitlines()
712 tag_format = re.compile(r'tag: ([^),]*)')
714 def latesttags(self):
716 p = FDroidPopen(['git', 'log', '--tags',
717 '--simplify-by-decoration', '--pretty=format:%d'],
718 cwd=self.local, output=False)
720 for line in p.output.splitlines():
721 for tag in self.tag_format.findall(line):
726 class vcs_gitsvn(vcs):
731 # If the local directory exists, but is somehow not a git repository, git
732 # will traverse up the directory tree until it finds one that is (i.e.
733 # fdroidserver) and then we'll proceed to destory it! This is called as
736 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
737 result = p.output.rstrip()
738 if not result.endswith(self.local):
739 raise VCSException('Repository mismatch')
741 def gotorevisionx(self, rev):
742 if not os.path.exists(self.local):
744 gitsvn_args = ['git', 'svn', 'clone']
745 if ';' in self.remote:
746 remote_split = self.remote.split(';')
747 for i in remote_split[1:]:
748 if i.startswith('trunk='):
749 gitsvn_args.extend(['-T', i[6:]])
750 elif i.startswith('tags='):
751 gitsvn_args.extend(['-t', i[5:]])
752 elif i.startswith('branches='):
753 gitsvn_args.extend(['-b', i[9:]])
754 gitsvn_args.extend([remote_split[0], self.local])
755 p = FDroidPopen(gitsvn_args, output=False)
756 if p.returncode != 0:
757 self.clone_failed = True
758 raise VCSException("Git svn clone failed", p.output)
760 gitsvn_args.extend([self.remote, self.local])
761 p = FDroidPopen(gitsvn_args, output=False)
762 if p.returncode != 0:
763 self.clone_failed = True
764 raise VCSException("Git svn clone failed", p.output)
768 # Discard any working tree changes
769 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
770 if p.returncode != 0:
771 raise VCSException("Git reset failed", p.output)
772 # Remove untracked files now, in case they're tracked in the target
773 # revision (it happens!)
774 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
775 if p.returncode != 0:
776 raise VCSException("Git clean failed", p.output)
777 if not self.refreshed:
778 # Get new commits, branches and tags from repo
779 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
780 if p.returncode != 0:
781 raise VCSException("Git svn fetch failed")
782 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
783 if p.returncode != 0:
784 raise VCSException("Git svn rebase failed", p.output)
785 self.refreshed = True
787 rev = rev or 'master'
789 nospaces_rev = rev.replace(' ', '%20')
790 # Try finding a svn tag
791 for treeish in ['origin/', '']:
792 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
793 if p.returncode == 0:
795 if p.returncode != 0:
796 # No tag found, normal svn rev translation
797 # Translate svn rev into git format
798 rev_split = rev.split('/')
801 for treeish in ['origin/', '']:
802 if len(rev_split) > 1:
803 treeish += rev_split[0]
804 svn_rev = rev_split[1]
807 # if no branch is specified, then assume trunk (i.e. 'master' branch):
811 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
813 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
814 git_rev = p.output.rstrip()
816 if p.returncode == 0 and git_rev:
819 if p.returncode != 0 or not git_rev:
820 # Try a plain git checkout as a last resort
821 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
822 if p.returncode != 0:
823 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
825 # Check out the git rev equivalent to the svn rev
826 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
827 if p.returncode != 0:
828 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
830 # Get rid of any uncontrolled files left behind
831 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
832 if p.returncode != 0:
833 raise VCSException("Git clean failed", p.output)
837 for treeish in ['origin/', '']:
838 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
844 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
845 if p.returncode != 0:
847 return p.output.strip()
855 def gotorevisionx(self, rev):
856 if not os.path.exists(self.local):
857 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
858 if p.returncode != 0:
859 self.clone_failed = True
860 raise VCSException("Hg clone failed", p.output)
862 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
863 if p.returncode != 0:
864 raise VCSException("Hg status failed", p.output)
865 for line in p.output.splitlines():
866 if not line.startswith('? '):
867 raise VCSException("Unexpected output from hg status -uS: " + line)
868 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
869 if not self.refreshed:
870 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
871 if p.returncode != 0:
872 raise VCSException("Hg pull failed", p.output)
873 self.refreshed = True
875 rev = rev or 'default'
878 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
879 if p.returncode != 0:
880 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
881 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
882 # Also delete untracked files, we have to enable purge extension for that:
883 if "'purge' is provided by the following extension" in p.output:
884 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
885 myfile.write("\n[extensions]\nhgext.purge=\n")
886 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
887 if p.returncode != 0:
888 raise VCSException("HG purge failed", p.output)
889 elif p.returncode != 0:
890 raise VCSException("HG purge failed", p.output)
893 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
894 return p.output.splitlines()[1:]
902 def gotorevisionx(self, rev):
903 if not os.path.exists(self.local):
904 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
905 if p.returncode != 0:
906 self.clone_failed = True
907 raise VCSException("Bzr branch failed", p.output)
909 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
910 if p.returncode != 0:
911 raise VCSException("Bzr revert failed", p.output)
912 if not self.refreshed:
913 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
914 if p.returncode != 0:
915 raise VCSException("Bzr update failed", p.output)
916 self.refreshed = True
918 revargs = list(['-r', rev] if rev else [])
919 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
920 if p.returncode != 0:
921 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
924 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
925 return [tag.split(' ')[0].strip() for tag in
926 p.output.splitlines()]
929 def unescape_string(string):
932 if string[0] == '"' and string[-1] == '"':
935 return string.replace("\\'", "'")
938 def retrieve_string(app_dir, string, xmlfiles=None):
940 if not string.startswith('@string/'):
941 return unescape_string(string)
946 os.path.join(app_dir, 'res'),
947 os.path.join(app_dir, 'src', 'main', 'res'),
949 for r, d, f in os.walk(res_dir):
950 if os.path.basename(r) == 'values':
951 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
953 name = string[len('@string/'):]
955 def element_content(element):
956 if element.text is None:
958 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
959 return s.decode('utf-8').strip()
961 for path in xmlfiles:
962 if not os.path.isfile(path):
964 xml = parse_xml(path)
965 element = xml.find('string[@name="' + name + '"]')
966 if element is not None:
967 content = element_content(element)
968 return retrieve_string(app_dir, content, xmlfiles)
973 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
974 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
977 # Return list of existing files that will be used to find the highest vercode
978 def manifest_paths(app_dir, flavours):
980 possible_manifests = \
981 [os.path.join(app_dir, 'AndroidManifest.xml'),
982 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
983 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
984 os.path.join(app_dir, 'build.gradle')]
986 for flavour in flavours:
989 possible_manifests.append(
990 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
992 return [path for path in possible_manifests if os.path.isfile(path)]
995 # Retrieve the package name. Returns the name, or None if not found.
996 def fetch_real_name(app_dir, flavours):
997 for path in manifest_paths(app_dir, flavours):
998 if not has_extension(path, 'xml') or not os.path.isfile(path):
1000 logging.debug("fetch_real_name: Checking manifest at " + path)
1001 xml = parse_xml(path)
1002 app = xml.find('application')
1005 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1007 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1008 result = retrieve_string_singleline(app_dir, label)
1010 result = result.strip()
1015 def get_library_references(root_dir):
1017 proppath = os.path.join(root_dir, 'project.properties')
1018 if not os.path.isfile(proppath):
1020 with open(proppath, 'r', encoding='iso-8859-1') as f:
1022 if not line.startswith('android.library.reference.'):
1024 path = line.split('=')[1].strip()
1025 relpath = os.path.join(root_dir, path)
1026 if not os.path.isdir(relpath):
1028 logging.debug("Found subproject at %s" % path)
1029 libraries.append(path)
1033 def ant_subprojects(root_dir):
1034 subprojects = get_library_references(root_dir)
1035 for subpath in subprojects:
1036 subrelpath = os.path.join(root_dir, subpath)
1037 for p in get_library_references(subrelpath):
1038 relp = os.path.normpath(os.path.join(subpath, p))
1039 if relp not in subprojects:
1040 subprojects.insert(0, relp)
1044 def remove_debuggable_flags(root_dir):
1045 # Remove forced debuggable flags
1046 logging.debug("Removing debuggable flags from %s" % root_dir)
1047 for root, dirs, files in os.walk(root_dir):
1048 if 'AndroidManifest.xml' in files:
1049 regsub_file(r'android:debuggable="[^"]*"',
1051 os.path.join(root, 'AndroidManifest.xml'))
1054 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
1055 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1056 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1059 def app_matches_packagename(app, package):
1062 appid = app.UpdateCheckName or app.id
1063 if appid is None or appid == "Ignore":
1065 return appid == package
1068 # Extract some information from the AndroidManifest.xml at the given path.
1069 # Returns (version, vercode, package), any or all of which might be None.
1070 # All values returned are strings.
1071 def parse_androidmanifests(paths, app):
1073 ignoreversions = app.UpdateCheckIgnore
1074 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1077 return (None, None, None)
1085 if not os.path.isfile(path):
1088 logging.debug("Parsing manifest at {0}".format(path))
1089 gradle = has_extension(path, 'gradle')
1095 with open(path, 'r') as f:
1097 if gradle_comment.match(line):
1099 # Grab first occurence of each to avoid running into
1100 # alternative flavours and builds.
1102 matches = psearch_g(line)
1104 s = matches.group(2)
1105 if app_matches_packagename(app, s):
1108 matches = vnsearch_g(line)
1110 version = matches.group(2)
1112 matches = vcsearch_g(line)
1114 vercode = matches.group(1)
1117 xml = parse_xml(path)
1118 if "package" in xml.attrib:
1119 s = xml.attrib["package"]
1120 if app_matches_packagename(app, s):
1122 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1123 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1124 base_dir = os.path.dirname(path)
1125 version = retrieve_string_singleline(base_dir, version)
1126 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1127 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1128 if string_is_integer(a):
1131 logging.warning("Problem with xml at {0}".format(path))
1133 # Remember package name, may be defined separately from version+vercode
1135 package = max_package
1137 logging.debug("..got package={0}, version={1}, vercode={2}"
1138 .format(package, version, vercode))
1140 # Always grab the package name and version name in case they are not
1141 # together with the highest version code
1142 if max_package is None and package is not None:
1143 max_package = package
1144 if max_version is None and version is not None:
1145 max_version = version
1147 if max_vercode is None or (vercode is not None and vercode > max_vercode):
1148 if not ignoresearch or not ignoresearch(version):
1149 if version is not None:
1150 max_version = version
1151 if vercode is not None:
1152 max_vercode = vercode
1153 if package is not None:
1154 max_package = package
1156 max_version = "Ignore"
1158 if max_version is None:
1159 max_version = "Unknown"
1161 if max_package and not is_valid_package_name(max_package):
1162 raise FDroidException("Invalid package name {0}".format(max_package))
1164 return (max_version, max_vercode, max_package)
1167 def is_valid_package_name(name):
1168 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1171 class FDroidException(Exception):
1173 def __init__(self, value, detail=None):
1175 self.detail = detail
1177 def shortened_detail(self):
1178 if len(self.detail) < 16000:
1180 return '[...]\n' + self.detail[-16000:]
1182 def get_wikitext(self):
1183 ret = repr(self.value) + "\n"
1186 ret += "<pre>\n" + self.shortened_detail() + "</pre>\n"
1192 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1196 class VCSException(FDroidException):
1200 class BuildException(FDroidException):
1204 # Get the specified source library.
1205 # Returns the path to it. Normally this is the path to be used when referencing
1206 # it, which may be a subdirectory of the actual project. If you want the base
1207 # directory of the project, pass 'basepath=True'.
1208 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1209 raw=False, prepare=True, preponly=False, refresh=True,
1218 name, ref = spec.split('@')
1220 number, name = name.split(':', 1)
1222 name, subdir = name.split('/', 1)
1224 if name not in fdroidserver.metadata.srclibs:
1225 raise VCSException('srclib ' + name + ' not found.')
1227 srclib = fdroidserver.metadata.srclibs[name]
1229 sdir = os.path.join(srclib_dir, name)
1232 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1233 vcs.srclib = (name, number, sdir)
1235 vcs.gotorevision(ref, refresh)
1242 libdir = os.path.join(sdir, subdir)
1243 elif srclib["Subdir"]:
1244 for subdir in srclib["Subdir"]:
1245 libdir_candidate = os.path.join(sdir, subdir)
1246 if os.path.exists(libdir_candidate):
1247 libdir = libdir_candidate
1253 remove_signing_keys(sdir)
1254 remove_debuggable_flags(sdir)
1258 if srclib["Prepare"]:
1259 cmd = replace_config_vars(srclib["Prepare"], build)
1261 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1262 if p.returncode != 0:
1263 raise BuildException("Error running prepare command for srclib %s"
1269 return (name, number, libdir)
1271 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1274 # Prepare the source code for a particular build
1275 # 'vcs' - the appropriate vcs object for the application
1276 # 'app' - the application details from the metadata
1277 # 'build' - the build details from the metadata
1278 # 'build_dir' - the path to the build directory, usually
1280 # 'srclib_dir' - the path to the source libraries directory, usually
1282 # 'extlib_dir' - the path to the external libraries directory, usually
1284 # Returns the (root, srclibpaths) where:
1285 # 'root' is the root directory, which may be the same as 'build_dir' or may
1286 # be a subdirectory of it.
1287 # 'srclibpaths' is information on the srclibs being used
1288 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1290 # Optionally, the actual app source can be in a subdirectory
1292 root_dir = os.path.join(build_dir, build.subdir)
1294 root_dir = build_dir
1296 # Get a working copy of the right revision
1297 logging.info("Getting source for revision " + build.commit)
1298 vcs.gotorevision(build.commit, refresh)
1300 # Initialise submodules if required
1301 if build.submodules:
1302 logging.info("Initialising submodules")
1303 vcs.initsubmodules()
1305 # Check that a subdir (if we're using one) exists. This has to happen
1306 # after the checkout, since it might not exist elsewhere
1307 if not os.path.exists(root_dir):
1308 raise BuildException('Missing subdir ' + root_dir)
1310 # Run an init command if one is required
1312 cmd = replace_config_vars(build.init, build)
1313 logging.info("Running 'init' commands in %s" % root_dir)
1315 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1316 if p.returncode != 0:
1317 raise BuildException("Error running init command for %s:%s" %
1318 (app.id, build.version), p.output)
1320 # Apply patches if any
1322 logging.info("Applying patches")
1323 for patch in build.patch:
1324 patch = patch.strip()
1325 logging.info("Applying " + patch)
1326 patch_path = os.path.join('metadata', app.id, patch)
1327 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1328 if p.returncode != 0:
1329 raise BuildException("Failed to apply patch %s" % patch_path)
1331 # Get required source libraries
1334 logging.info("Collecting source libraries")
1335 for lib in build.srclibs:
1336 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1337 refresh=refresh, build=build))
1339 for name, number, libpath in srclibpaths:
1340 place_srclib(root_dir, int(number) if number else None, libpath)
1342 basesrclib = vcs.getsrclib()
1343 # If one was used for the main source, add that too.
1345 srclibpaths.append(basesrclib)
1347 # Update the local.properties file
1348 localprops = [os.path.join(build_dir, 'local.properties')]
1350 parts = build.subdir.split(os.sep)
1353 cur = os.path.join(cur, d)
1354 localprops += [os.path.join(cur, 'local.properties')]
1355 for path in localprops:
1357 if os.path.isfile(path):
1358 logging.info("Updating local.properties file at %s" % path)
1359 with open(path, 'r', encoding='iso-8859-1') as f:
1363 logging.info("Creating local.properties file at %s" % path)
1364 # Fix old-fashioned 'sdk-location' by copying
1365 # from sdk.dir, if necessary
1367 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1368 re.S | re.M).group(1)
1369 props += "sdk-location=%s\n" % sdkloc
1371 props += "sdk.dir=%s\n" % config['sdk_path']
1372 props += "sdk-location=%s\n" % config['sdk_path']
1373 ndk_path = build.ndk_path()
1376 props += "ndk.dir=%s\n" % ndk_path
1377 props += "ndk-location=%s\n" % ndk_path
1378 # Add java.encoding if necessary
1380 props += "java.encoding=%s\n" % build.encoding
1381 with open(path, 'w', encoding='iso-8859-1') as f:
1385 if build.build_method() == 'gradle':
1386 flavours = build.gradle
1389 n = build.target.split('-')[1]
1390 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1391 r'compileSdkVersion %s' % n,
1392 os.path.join(root_dir, 'build.gradle'))
1394 # Remove forced debuggable flags
1395 remove_debuggable_flags(root_dir)
1397 # Insert version code and number into the manifest if necessary
1398 if build.forceversion:
1399 logging.info("Changing the version name")
1400 for path in manifest_paths(root_dir, flavours):
1401 if not os.path.isfile(path):
1403 if has_extension(path, 'xml'):
1404 regsub_file(r'android:versionName="[^"]*"',
1405 r'android:versionName="%s"' % build.version,
1407 elif has_extension(path, 'gradle'):
1408 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1409 r"""\1versionName '%s'""" % build.version,
1412 if build.forcevercode:
1413 logging.info("Changing the version code")
1414 for path in manifest_paths(root_dir, flavours):
1415 if not os.path.isfile(path):
1417 if has_extension(path, 'xml'):
1418 regsub_file(r'android:versionCode="[^"]*"',
1419 r'android:versionCode="%s"' % build.vercode,
1421 elif has_extension(path, 'gradle'):
1422 regsub_file(r'versionCode[ =]+[0-9]+',
1423 r'versionCode %s' % build.vercode,
1426 # Delete unwanted files
1428 logging.info("Removing specified files")
1429 for part in getpaths(build_dir, build.rm):
1430 dest = os.path.join(build_dir, part)
1431 logging.info("Removing {0}".format(part))
1432 if os.path.lexists(dest):
1433 if os.path.islink(dest):
1434 FDroidPopen(['unlink', dest], output=False)
1436 FDroidPopen(['rm', '-rf', dest], output=False)
1438 logging.info("...but it didn't exist")
1440 remove_signing_keys(build_dir)
1442 # Add required external libraries
1444 logging.info("Collecting prebuilt libraries")
1445 libsdir = os.path.join(root_dir, 'libs')
1446 if not os.path.exists(libsdir):
1448 for lib in build.extlibs:
1450 logging.info("...installing extlib {0}".format(lib))
1451 libf = os.path.basename(lib)
1452 libsrc = os.path.join(extlib_dir, lib)
1453 if not os.path.exists(libsrc):
1454 raise BuildException("Missing extlib file {0}".format(libsrc))
1455 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1457 # Run a pre-build command if one is required
1459 logging.info("Running 'prebuild' commands in %s" % root_dir)
1461 cmd = replace_config_vars(build.prebuild, build)
1463 # Substitute source library paths into prebuild commands
1464 for name, number, libpath in srclibpaths:
1465 libpath = os.path.relpath(libpath, root_dir)
1466 cmd = cmd.replace('$$' + name + '$$', libpath)
1468 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1469 if p.returncode != 0:
1470 raise BuildException("Error running prebuild command for %s:%s" %
1471 (app.id, build.version), p.output)
1473 # Generate (or update) the ant build file, build.xml...
1474 if build.build_method() == 'ant' and build.update != ['no']:
1475 parms = ['android', 'update', 'lib-project']
1476 lparms = ['android', 'update', 'project']
1479 parms += ['-t', build.target]
1480 lparms += ['-t', build.target]
1482 update_dirs = build.update
1484 update_dirs = ant_subprojects(root_dir) + ['.']
1486 for d in update_dirs:
1487 subdir = os.path.join(root_dir, d)
1489 logging.debug("Updating main project")
1490 cmd = parms + ['-p', d]
1492 logging.debug("Updating subproject %s" % d)
1493 cmd = lparms + ['-p', d]
1494 p = SdkToolsPopen(cmd, cwd=root_dir)
1495 # Check to see whether an error was returned without a proper exit
1496 # code (this is the case for the 'no target set or target invalid'
1498 if p.returncode != 0 or p.output.startswith("Error: "):
1499 raise BuildException("Failed to update project at %s" % d, p.output)
1500 # Clean update dirs via ant
1502 logging.info("Cleaning subproject %s" % d)
1503 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1505 return (root_dir, srclibpaths)
1508 # Extend via globbing the paths from a field and return them as a map from
1509 # original path to resulting paths
1510 def getpaths_map(build_dir, globpaths):
1514 full_path = os.path.join(build_dir, p)
1515 full_path = os.path.normpath(full_path)
1516 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1518 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1522 # Extend via globbing the paths from a field and return them as a set
1523 def getpaths(build_dir, globpaths):
1524 paths_map = getpaths_map(build_dir, globpaths)
1526 for k, v in paths_map.items():
1533 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1539 self.path = os.path.join('stats', 'known_apks.txt')
1541 if os.path.isfile(self.path):
1542 with open(self.path, 'r', encoding='utf8') as f:
1544 t = line.rstrip().split(' ')
1546 self.apks[t[0]] = (t[1], None)
1548 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1549 self.changed = False
1551 def writeifchanged(self):
1552 if not self.changed:
1555 if not os.path.exists('stats'):
1559 for apk, app in self.apks.items():
1561 line = apk + ' ' + appid
1563 line += ' ' + time.strftime('%Y-%m-%d', added)
1566 with open(self.path, 'w', encoding='utf8') as f:
1567 for line in sorted(lst, key=natural_key):
1568 f.write(line + '\n')
1570 # Record an apk (if it's new, otherwise does nothing)
1571 # Returns the date it was added.
1572 def recordapk(self, apk, app):
1573 if apk not in self.apks:
1574 self.apks[apk] = (app, time.gmtime(time.time()))
1576 _, added = self.apks[apk]
1579 # Look up information - given the 'apkname', returns (app id, date added/None).
1580 # Or returns None for an unknown apk.
1581 def getapp(self, apkname):
1582 if apkname in self.apks:
1583 return self.apks[apkname]
1586 # Get the most recent 'num' apps added to the repo, as a list of package ids
1587 # with the most recent first.
1588 def getlatest(self, num):
1590 for apk, app in self.apks.items():
1594 if apps[appid] > added:
1598 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1599 lst = [app for app, _ in sortedapps]
1604 def isApkDebuggable(apkfile, config):
1605 """Returns True if the given apk file is debuggable
1607 :param apkfile: full path to the apk to check"""
1609 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1611 if p.returncode != 0:
1612 logging.critical("Failed to get apk manifest information")
1614 for line in p.output.splitlines():
1615 if 'android:debuggable' in line and not line.endswith('0x0'):
1622 self.returncode = None
1626 def SdkToolsPopen(commands, cwd=None, output=True):
1628 if cmd not in config:
1629 config[cmd] = find_sdk_tools_cmd(commands[0])
1630 abscmd = config[cmd]
1632 logging.critical("Could not find '%s' on your system" % cmd)
1634 return FDroidPopen([abscmd] + commands[1:],
1635 cwd=cwd, output=output)
1638 def FDroidPopenBytes(commands, cwd=None, output=True, stderr_to_stdout=True):
1640 Run a command and capture the possibly huge output as bytes.
1642 :param commands: command and argument list like in subprocess.Popen
1643 :param cwd: optionally specifies a working directory
1644 :returns: A PopenResult.
1649 set_FDroidPopen_env()
1652 cwd = os.path.normpath(cwd)
1653 logging.debug("Directory: %s" % cwd)
1654 logging.debug("> %s" % ' '.join(commands))
1656 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
1657 result = PopenResult()
1660 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
1661 stdout=subprocess.PIPE, stderr=stderr_param)
1662 except OSError as e:
1663 raise BuildException("OSError while trying to execute " +
1664 ' '.join(commands) + ': ' + str(e))
1666 if not stderr_to_stdout and options.verbose:
1667 stderr_queue = Queue()
1668 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
1670 while not stderr_reader.eof():
1671 while not stderr_queue.empty():
1672 line = stderr_queue.get()
1673 sys.stderr.buffer.write(line)
1678 stdout_queue = Queue()
1679 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1682 # Check the queue for output (until there is no more to get)
1683 while not stdout_reader.eof():
1684 while not stdout_queue.empty():
1685 line = stdout_queue.get()
1686 if output and options.verbose:
1687 # Output directly to console
1688 sys.stderr.buffer.write(line)
1694 result.returncode = p.wait()
1695 result.output = buf.getvalue()
1700 def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
1702 Run a command and capture the possibly huge output as a str.
1704 :param commands: command and argument list like in subprocess.Popen
1705 :param cwd: optionally specifies a working directory
1706 :returns: A PopenResult.
1708 result = FDroidPopenBytes(commands, cwd, output, stderr_to_stdout)
1709 result.output = result.output.decode('utf-8')
1713 gradle_comment = re.compile(r'[ ]*//')
1714 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1715 gradle_line_matches = [
1716 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1717 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1718 re.compile(r'.*\.readLine\(.*'),
1722 def remove_signing_keys(build_dir):
1723 for root, dirs, files in os.walk(build_dir):
1724 if 'build.gradle' in files:
1725 path = os.path.join(root, 'build.gradle')
1727 with open(path, "r", encoding='utf8') as o:
1728 lines = o.readlines()
1734 with open(path, "w", encoding='utf8') as o:
1735 while i < len(lines):
1738 while line.endswith('\\\n'):
1739 line = line.rstrip('\\\n') + lines[i]
1742 if gradle_comment.match(line):
1747 opened += line.count('{')
1748 opened -= line.count('}')
1751 if gradle_signing_configs.match(line):
1756 if any(s.match(line) for s in gradle_line_matches):
1764 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1767 'project.properties',
1769 'default.properties',
1770 'ant.properties', ]:
1771 if propfile in files:
1772 path = os.path.join(root, propfile)
1774 with open(path, "r", encoding='iso-8859-1') as o:
1775 lines = o.readlines()
1779 with open(path, "w", encoding='iso-8859-1') as o:
1781 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1788 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1791 def set_FDroidPopen_env(build=None):
1793 set up the environment variables for the build environment
1795 There is only a weak standard, the variables used by gradle, so also set
1796 up the most commonly used environment variables for SDK and NDK
1798 global env, orig_path
1802 orig_path = env['PATH']
1803 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
1804 env[n] = config['sdk_path']
1805 for k, v in config['java_paths'].items():
1806 env['JAVA%s_HOME' % k] = v
1808 if build is not None:
1809 path = build.ndk_path()
1810 paths = orig_path.split(os.pathsep)
1811 if path not in paths:
1812 paths = [path] + paths
1813 env['PATH'] = os.pathsep.join(paths)
1814 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
1815 env[n] = build.ndk_path()
1818 def replace_config_vars(cmd, build):
1819 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1820 cmd = cmd.replace('$$NDK$$', build.ndk_path())
1821 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1822 if build is not None:
1823 cmd = cmd.replace('$$COMMIT$$', build.commit)
1824 cmd = cmd.replace('$$VERSION$$', build.version)
1825 cmd = cmd.replace('$$VERCODE$$', build.vercode)
1829 def place_srclib(root_dir, number, libpath):
1832 relpath = os.path.relpath(libpath, root_dir)
1833 proppath = os.path.join(root_dir, 'project.properties')
1836 if os.path.isfile(proppath):
1837 with open(proppath, "r", encoding='iso-8859-1') as o:
1838 lines = o.readlines()
1840 with open(proppath, "w", encoding='iso-8859-1') as o:
1843 if line.startswith('android.library.reference.%d=' % number):
1844 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1849 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1851 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
1854 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
1855 """Verify that two apks are the same
1857 One of the inputs is signed, the other is unsigned. The signature metadata
1858 is transferred from the signed to the unsigned apk, and then jarsigner is
1859 used to verify that the signature from the signed apk is also varlid for
1861 :param signed_apk: Path to a signed apk file
1862 :param unsigned_apk: Path to an unsigned apk file expected to match it
1863 :param tmp_dir: Path to directory for temporary files
1864 :returns: None if the verification is successful, otherwise a string
1865 describing what went wrong.
1867 with ZipFile(signed_apk) as signed_apk_as_zip:
1868 meta_inf_files = ['META-INF/MANIFEST.MF']
1869 for f in signed_apk_as_zip.namelist():
1870 if apk_sigfile.match(f):
1871 meta_inf_files.append(f)
1872 if len(meta_inf_files) < 3:
1873 return "Signature files missing from {0}".format(signed_apk)
1874 signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
1875 with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
1876 for meta_inf_file in meta_inf_files:
1877 unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
1879 if subprocess.call([config['jarsigner'], '-verify', unsigned_apk]) != 0:
1880 logging.info("...NOT verified - {0}".format(signed_apk))
1881 return compare_apks(signed_apk, unsigned_apk, tmp_dir)
1882 logging.info("...successfully verified")
1885 apk_badchars = re.compile('''[/ :;'"]''')
1888 def compare_apks(apk1, apk2, tmp_dir):
1891 Returns None if the apk content is the same (apart from the signing key),
1892 otherwise a string describing what's different, or what went wrong when
1893 trying to do the comparison.
1896 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
1897 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
1898 for d in [apk1dir, apk2dir]:
1899 if os.path.exists(d):
1902 os.mkdir(os.path.join(d, 'jar-xf'))
1904 if subprocess.call(['jar', 'xf',
1905 os.path.abspath(apk1)],
1906 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
1907 return("Failed to unpack " + apk1)
1908 if subprocess.call(['jar', 'xf',
1909 os.path.abspath(apk2)],
1910 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
1911 return("Failed to unpack " + apk2)
1913 # try to find apktool in the path, if it hasn't been manually configed
1914 if 'apktool' not in config:
1915 tmp = find_command('apktool')
1917 config['apktool'] = tmp
1918 if 'apktool' in config:
1919 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
1921 return("Failed to unpack " + apk1)
1922 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
1924 return("Failed to unpack " + apk2)
1926 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
1927 lines = p.output.splitlines()
1928 if len(lines) != 1 or 'META-INF' not in lines[0]:
1929 meld = find_command('meld')
1930 if meld is not None:
1931 p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
1932 return("Unexpected diff output - " + p.output)
1934 # since everything verifies, delete the comparison to keep cruft down
1935 shutil.rmtree(apk1dir)
1936 shutil.rmtree(apk2dir)
1938 # If we get here, it seems like they're the same!
1942 def find_command(command):
1943 '''find the full path of a command, or None if it can't be found in the PATH'''
1946 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
1948 fpath, fname = os.path.split(command)
1953 for path in os.environ["PATH"].split(os.pathsep):
1954 path = path.strip('"')
1955 exe_file = os.path.join(path, command)
1956 if is_exe(exe_file):
1963 '''generate a random password for when generating keys'''
1964 h = hashlib.sha256()
1965 h.update(os.urandom(16)) # salt
1966 h.update(socket.getfqdn().encode('utf-8'))
1967 passwd = base64.b64encode(h.digest()).strip()
1968 return passwd.decode('utf-8')
1971 def genkeystore(localconfig):
1972 '''Generate a new key with random passwords and add it to new keystore'''
1973 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
1974 keystoredir = os.path.dirname(localconfig['keystore'])
1975 if keystoredir is None or keystoredir == '':
1976 keystoredir = os.path.join(os.getcwd(), keystoredir)
1977 if not os.path.exists(keystoredir):
1978 os.makedirs(keystoredir, mode=0o700)
1980 write_password_file("keystorepass", localconfig['keystorepass'])
1981 write_password_file("keypass", localconfig['keypass'])
1982 p = FDroidPopen([config['keytool'], '-genkey',
1983 '-keystore', localconfig['keystore'],
1984 '-alias', localconfig['repo_keyalias'],
1985 '-keyalg', 'RSA', '-keysize', '4096',
1986 '-sigalg', 'SHA256withRSA',
1987 '-validity', '10000',
1988 '-storepass:file', config['keystorepassfile'],
1989 '-keypass:file', config['keypassfile'],
1990 '-dname', localconfig['keydname']])
1991 # TODO keypass should be sent via stdin
1992 if p.returncode != 0:
1993 raise BuildException("Failed to generate key", p.output)
1994 os.chmod(localconfig['keystore'], 0o0600)
1995 # now show the lovely key that was just generated
1996 p = FDroidPopen([config['keytool'], '-list', '-v',
1997 '-keystore', localconfig['keystore'],
1998 '-alias', localconfig['repo_keyalias'],
1999 '-storepass:file', config['keystorepassfile']])
2000 logging.info(p.output.strip() + '\n\n')
2003 def write_to_config(thisconfig, key, value=None):
2004 '''write a key/value to the local config.py'''
2006 origkey = key + '_orig'
2007 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2008 with open('config.py', 'r', encoding='utf8') as f:
2010 pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
2011 repl = '\n' + key + ' = "' + value + '"'
2012 data = re.sub(pattern, repl, data)
2013 # if this key is not in the file, append it
2014 if not re.match('\s*' + key + '\s*=\s*"', data):
2016 # make sure the file ends with a carraige return
2017 if not re.match('\n$', data):
2019 with open('config.py', 'w', encoding='utf8') as f:
2023 def parse_xml(path):
2024 return XMLElementTree.parse(path).getroot()
2027 def string_is_integer(string):
2035 def get_per_app_repos():
2036 '''per-app repos are dirs named with the packageName of a single app'''
2038 # Android packageNames are Java packages, they may contain uppercase or
2039 # lowercase letters ('A' through 'Z'), numbers, and underscores
2040 # ('_'). However, individual package name parts may only start with
2041 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2042 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2045 for root, dirs, files in os.walk(os.getcwd()):
2047 print('checking', root, 'for', d)
2048 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2049 # standard parts of an fdroid repo, so never packageNames
2052 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):