1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
35 import xml.etree.ElementTree as XMLElementTree
39 from Queue import Queue
42 from queue import Queue
44 from zipfile import ZipFile
47 from fdroidserver.asynchronousfilereader import AsynchronousFileReader
50 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
59 'sdk_path': "$ANDROID_HOME",
62 'r10e': "$ANDROID_NDK",
64 'build_tools': "23.0.2",
69 'accepted_formats': ['txt', 'yaml'],
70 'sync_from_local_copy_dir': False,
71 'per_app_repos': False,
72 'make_current_version_link': True,
73 'current_version_name_source': 'Name',
74 'update_stats': False,
78 'stats_to_carbon': False,
80 'build_server_always': False,
81 'keystore': 'keystore.jks',
82 'smartcardoptions': [],
88 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
89 'repo_name': "My First FDroid Repo Demo",
90 'repo_icon': "fdroid-icon.png",
91 'repo_description': '''
92 This is a repository of apps to be used with FDroid. Applications in this
93 repository are either official binaries built by the original application
94 developers, or are binaries built from source by the admin of f-droid.org
95 using the tools on https://gitlab.com/u/fdroid.
101 def setup_global_opts(parser):
102 parser.add_argument("-v", "--verbose", action="store_true", default=False,
103 help="Spew out even more information than normal")
104 parser.add_argument("-q", "--quiet", action="store_true", default=False,
105 help="Restrict output to warnings and errors")
108 def fill_config_defaults(thisconfig):
109 for k, v in default_config.items():
110 if k not in thisconfig:
113 # Expand paths (~users and $vars)
114 def expand_path(path):
118 path = os.path.expanduser(path)
119 path = os.path.expandvars(path)
124 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
129 thisconfig[k + '_orig'] = v
131 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
132 if thisconfig['java_paths'] is None:
133 thisconfig['java_paths'] = dict()
134 for d in sorted(glob.glob('/usr/lib/jvm/j*[6-9]*')
135 + glob.glob('/usr/java/jdk1.[6-9]*')
136 + glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
137 + glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')):
138 if os.path.islink(d):
140 j = os.path.basename(d)
141 # the last one found will be the canonical one, so order appropriately
143 r'^1\.([6-9])\.0\.jdk$', # OSX
144 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
145 r'^jdk([6-9])-openjdk$', # Arch
146 r'^java-([6-9])-openjdk$', # Arch
147 r'^java-([6-9])-jdk$', # Arch (oracle)
148 r'^java-1\.([6-9])\.0-.*$', # RedHat
149 r'^java-([6-9])-oracle$', # Debian WebUpd8
150 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
151 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
153 m = re.match(regex, j)
156 osxhome = os.path.join(d, 'Contents', 'Home')
157 if os.path.exists(osxhome):
158 thisconfig['java_paths'][m.group(1)] = osxhome
160 thisconfig['java_paths'][m.group(1)] = d
162 for java_version in ('7', '8', '9'):
163 if java_version not in thisconfig['java_paths']:
165 java_home = thisconfig['java_paths'][java_version]
166 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
167 if os.path.exists(jarsigner):
168 thisconfig['jarsigner'] = jarsigner
169 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
170 break # Java7 is preferred, so quit if found
172 for k in ['ndk_paths', 'java_paths']:
178 thisconfig[k][k2] = exp
179 thisconfig[k][k2 + '_orig'] = v
182 def regsub_file(pattern, repl, path):
183 with open(path, 'r') as f:
185 text = re.sub(pattern, repl, text)
186 with open(path, 'w') as f:
190 def read_config(opts, config_file='config.py'):
191 """Read the repository config
193 The config is read from config_file, which is in the current directory when
194 any of the repo management commands are used.
196 global config, options, env, orig_path
198 if config is not None:
200 if not os.path.isfile(config_file):
201 logging.critical("Missing config file - is this a repo directory?")
208 logging.debug("Reading %s" % config_file)
209 execfile(config_file, config)
211 # smartcardoptions must be a list since its command line args for Popen
212 if 'smartcardoptions' in config:
213 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
214 elif 'keystore' in config and config['keystore'] == 'NONE':
215 # keystore='NONE' means use smartcard, these are required defaults
216 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
217 'SunPKCS11-OpenSC', '-providerClass',
218 'sun.security.pkcs11.SunPKCS11',
219 '-providerArg', 'opensc-fdroid.cfg']
221 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
222 st = os.stat(config_file)
223 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
224 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
226 fill_config_defaults(config)
228 # There is no standard, so just set up the most common environment
231 orig_path = env['PATH']
232 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
233 env[n] = config['sdk_path']
235 for k, v in config['java_paths'].items():
236 env['JAVA%s_HOME' % k] = v
238 for k in ["keystorepass", "keypass"]:
240 write_password_file(k)
242 for k in ["repo_description", "archive_description"]:
244 config[k] = clean_description(config[k])
246 if 'serverwebroot' in config:
247 if isinstance(config['serverwebroot'], basestring):
248 roots = [config['serverwebroot']]
249 elif all(isinstance(item, basestring) for item in config['serverwebroot']):
250 roots = config['serverwebroot']
252 raise TypeError('only accepts strings, lists, and tuples')
254 for rootstr in roots:
255 # since this is used with rsync, where trailing slashes have
256 # meaning, ensure there is always a trailing slash
257 if rootstr[-1] != '/':
259 rootlist.append(rootstr.replace('//', '/'))
260 config['serverwebroot'] = rootlist
265 def find_sdk_tools_cmd(cmd):
266 '''find a working path to a tool from the Android SDK'''
269 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
270 # try to find a working path to this command, in all the recent possible paths
271 if 'build_tools' in config:
272 build_tools = os.path.join(config['sdk_path'], 'build-tools')
273 # if 'build_tools' was manually set and exists, check only that one
274 configed_build_tools = os.path.join(build_tools, config['build_tools'])
275 if os.path.exists(configed_build_tools):
276 tooldirs.append(configed_build_tools)
278 # no configed version, so hunt known paths for it
279 for f in sorted(os.listdir(build_tools), reverse=True):
280 if os.path.isdir(os.path.join(build_tools, f)):
281 tooldirs.append(os.path.join(build_tools, f))
282 tooldirs.append(build_tools)
283 sdk_tools = os.path.join(config['sdk_path'], 'tools')
284 if os.path.exists(sdk_tools):
285 tooldirs.append(sdk_tools)
286 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
287 if os.path.exists(sdk_platform_tools):
288 tooldirs.append(sdk_platform_tools)
289 tooldirs.append('/usr/bin')
291 if os.path.isfile(os.path.join(d, cmd)):
292 return os.path.join(d, cmd)
293 # did not find the command, exit with error message
294 ensure_build_tools_exists(config)
297 def test_sdk_exists(thisconfig):
298 if 'sdk_path' not in thisconfig:
299 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
302 logging.error("'sdk_path' not set in config.py!")
304 if thisconfig['sdk_path'] == default_config['sdk_path']:
305 logging.error('No Android SDK found!')
306 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
307 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
309 if not os.path.exists(thisconfig['sdk_path']):
310 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
312 if not os.path.isdir(thisconfig['sdk_path']):
313 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
315 for d in ['build-tools', 'platform-tools', 'tools']:
316 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
317 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
318 thisconfig['sdk_path'], d))
323 def ensure_build_tools_exists(thisconfig):
324 if not test_sdk_exists(thisconfig):
326 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
327 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
328 if not os.path.isdir(versioned_build_tools):
329 logging.critical('Android Build Tools path "'
330 + versioned_build_tools + '" does not exist!')
334 def write_password_file(pwtype, password=None):
336 writes out passwords to a protected file instead of passing passwords as
337 command line argments
339 filename = '.fdroid.' + pwtype + '.txt'
340 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
342 os.write(fd, config[pwtype])
344 os.write(fd, password)
346 config[pwtype + 'file'] = filename
349 # Given the arguments in the form of multiple appid:[vc] strings, this returns
350 # a dictionary with the set of vercodes specified for each package.
351 def read_pkg_args(args, allow_vercodes=False):
358 if allow_vercodes and ':' in p:
359 package, vercode = p.split(':')
361 package, vercode = p, None
362 if package not in vercodes:
363 vercodes[package] = [vercode] if vercode else []
365 elif vercode and vercode not in vercodes[package]:
366 vercodes[package] += [vercode] if vercode else []
371 # On top of what read_pkg_args does, this returns the whole app metadata, but
372 # limiting the builds list to the builds matching the vercodes specified.
373 def read_app_args(args, allapps, allow_vercodes=False):
375 vercodes = read_pkg_args(args, allow_vercodes)
381 for appid, app in allapps.iteritems():
382 if appid in vercodes:
385 if len(apps) != len(vercodes):
388 logging.critical("No such package: %s" % p)
389 raise FDroidException("Found invalid app ids in arguments")
391 raise FDroidException("No packages specified")
394 for appid, app in apps.iteritems():
398 app.builds = [b for b in app.builds if b.vercode in vc]
399 if len(app.builds) != len(vercodes[appid]):
401 allvcs = [b.vercode for b in app.builds]
402 for v in vercodes[appid]:
404 logging.critical("No such vercode %s for app %s" % (v, appid))
407 raise FDroidException("Found invalid vercodes for some apps")
412 def get_extension(filename):
413 base, ext = os.path.splitext(filename)
416 return base, ext.lower()[1:]
419 def has_extension(filename, ext):
420 _, f_ext = get_extension(filename)
424 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
427 def clean_description(description):
428 'Remove unneeded newlines and spaces from a block of description text'
430 # this is split up by paragraph to make removing the newlines easier
431 for paragraph in re.split(r'\n\n', description):
432 paragraph = re.sub('\r', '', paragraph)
433 paragraph = re.sub('\n', ' ', paragraph)
434 paragraph = re.sub(' {2,}', ' ', paragraph)
435 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
436 returnstring += paragraph + '\n\n'
437 return returnstring.rstrip('\n')
440 def apknameinfo(filename):
441 filename = os.path.basename(filename)
442 m = apk_regex.match(filename)
444 result = (m.group(1), m.group(2))
445 except AttributeError:
446 raise FDroidException("Invalid apk name: %s" % filename)
450 def getapkname(app, build):
451 return "%s_%s.apk" % (app.id, build.vercode)
454 def getsrcname(app, build):
455 return "%s_%s_src.tar.gz" % (app.id, build.vercode)
467 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
470 def getvcs(vcstype, remote, local):
472 return vcs_git(remote, local)
473 if vcstype == 'git-svn':
474 return vcs_gitsvn(remote, local)
476 return vcs_hg(remote, local)
478 return vcs_bzr(remote, local)
479 if vcstype == 'srclib':
480 if local != os.path.join('build', 'srclib', remote):
481 raise VCSException("Error: srclib paths are hard-coded!")
482 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
484 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
485 raise VCSException("Invalid vcs type " + vcstype)
488 def getsrclibvcs(name):
489 if name not in metadata.srclibs:
490 raise VCSException("Missing srclib " + name)
491 return metadata.srclibs[name]['Repo Type']
496 def __init__(self, remote, local):
498 # svn, git-svn and bzr may require auth
500 if self.repotype() in ('git-svn', 'bzr'):
502 if self.repotype == 'git-svn':
503 raise VCSException("Authentication is not supported for git-svn")
504 self.username, remote = remote.split('@')
505 if ':' not in self.username:
506 raise VCSException("Password required with username")
507 self.username, self.password = self.username.split(':')
511 self.clone_failed = False
512 self.refreshed = False
518 # Take the local repository to a clean version of the given revision, which
519 # is specificed in the VCS's native format. Beforehand, the repository can
520 # be dirty, or even non-existent. If the repository does already exist
521 # locally, it will be updated from the origin, but only once in the
522 # lifetime of the vcs object.
523 # None is acceptable for 'rev' if you know you are cloning a clean copy of
524 # the repo - otherwise it must specify a valid revision.
525 def gotorevision(self, rev, refresh=True):
527 if self.clone_failed:
528 raise VCSException("Downloading the repository already failed once, not trying again.")
530 # The .fdroidvcs-id file for a repo tells us what VCS type
531 # and remote that directory was created from, allowing us to drop it
532 # automatically if either of those things changes.
533 fdpath = os.path.join(self.local, '..',
534 '.fdroidvcs-' + os.path.basename(self.local))
535 cdata = self.repotype() + ' ' + self.remote
538 if os.path.exists(self.local):
539 if os.path.exists(fdpath):
540 with open(fdpath, 'r') as f:
541 fsdata = f.read().strip()
546 logging.info("Repository details for %s changed - deleting" % (
550 logging.info("Repository details for %s missing - deleting" % (
553 shutil.rmtree(self.local)
557 self.refreshed = True
560 self.gotorevisionx(rev)
561 except FDroidException as e:
564 # If necessary, write the .fdroidvcs file.
565 if writeback and not self.clone_failed:
566 with open(fdpath, 'w') as f:
572 # Derived classes need to implement this. It's called once basic checking
573 # has been performend.
574 def gotorevisionx(self, rev):
575 raise VCSException("This VCS type doesn't define gotorevisionx")
577 # Initialise and update submodules
578 def initsubmodules(self):
579 raise VCSException('Submodules not supported for this vcs type')
581 # Get a list of all known tags
583 if not self._gettags:
584 raise VCSException('gettags not supported for this vcs type')
586 for tag in self._gettags():
587 if re.match('[-A-Za-z0-9_. /]+$', tag):
591 def latesttags(self, tags, number):
592 """Get the most recent tags in a given list.
594 :param tags: a list of tags
595 :param number: the number to return
596 :returns: A list containing the most recent tags in the provided
597 list, up to the maximum number given.
599 raise VCSException('latesttags not supported for this vcs type')
601 # Get current commit reference (hash, revision, etc)
603 raise VCSException('getref not supported for this vcs type')
605 # Returns the srclib (name, path) used in setting up the current
616 # If the local directory exists, but is somehow not a git repository, git
617 # will traverse up the directory tree until it finds one that is (i.e.
618 # fdroidserver) and then we'll proceed to destroy it! This is called as
621 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
622 result = p.output.rstrip()
623 if not result.endswith(self.local):
624 raise VCSException('Repository mismatch')
626 def gotorevisionx(self, rev):
627 if not os.path.exists(self.local):
629 p = FDroidPopen(['git', 'clone', self.remote, self.local])
630 if p.returncode != 0:
631 self.clone_failed = True
632 raise VCSException("Git clone failed", p.output)
636 # Discard any working tree changes
637 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
638 'git', 'reset', '--hard'], cwd=self.local, output=False)
639 if p.returncode != 0:
640 raise VCSException("Git reset failed", p.output)
641 # Remove untracked files now, in case they're tracked in the target
642 # revision (it happens!)
643 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
644 'git', 'clean', '-dffx'], cwd=self.local, output=False)
645 if p.returncode != 0:
646 raise VCSException("Git clean failed", p.output)
647 if not self.refreshed:
648 # Get latest commits and tags from remote
649 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
650 if p.returncode != 0:
651 raise VCSException("Git fetch failed", p.output)
652 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
653 if p.returncode != 0:
654 raise VCSException("Git fetch failed", p.output)
655 # Recreate origin/HEAD as git clone would do it, in case it disappeared
656 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
657 if p.returncode != 0:
658 lines = p.output.splitlines()
659 if 'Multiple remote HEAD branches' not in lines[0]:
660 raise VCSException("Git remote set-head failed", p.output)
661 branch = lines[1].split(' ')[-1]
662 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
663 if p2.returncode != 0:
664 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
665 self.refreshed = True
666 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
667 # a github repo. Most of the time this is the same as origin/master.
668 rev = rev or 'origin/HEAD'
669 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
670 if p.returncode != 0:
671 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
672 # Get rid of any uncontrolled files left behind
673 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
674 if p.returncode != 0:
675 raise VCSException("Git clean failed", p.output)
677 def initsubmodules(self):
679 submfile = os.path.join(self.local, '.gitmodules')
680 if not os.path.isfile(submfile):
681 raise VCSException("No git submodules available")
683 # fix submodules not accessible without an account and public key auth
684 with open(submfile, 'r') as f:
685 lines = f.readlines()
686 with open(submfile, 'w') as f:
688 if 'git@github.com' in line:
689 line = line.replace('git@github.com:', 'https://github.com/')
690 if 'git@gitlab.com' in line:
691 line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
694 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
695 if p.returncode != 0:
696 raise VCSException("Git submodule sync failed", p.output)
697 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
698 if p.returncode != 0:
699 raise VCSException("Git submodule update failed", p.output)
703 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
704 return p.output.splitlines()
706 def latesttags(self, tags, number):
711 ['git', 'show', '--format=format:%ct', '-s', tag],
712 cwd=self.local, output=False)
713 # Timestamp is on the last line. For a normal tag, it's the only
714 # line, but for annotated tags, the rest of the info precedes it.
715 ts = int(p.output.splitlines()[-1])
718 for _, t in sorted(tl)[-number:]:
723 class vcs_gitsvn(vcs):
728 # If the local directory exists, but is somehow not a git repository, git
729 # will traverse up the directory tree until it finds one that is (i.e.
730 # fdroidserver) and then we'll proceed to destory it! This is called as
733 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
734 result = p.output.rstrip()
735 if not result.endswith(self.local):
736 raise VCSException('Repository mismatch')
738 def gotorevisionx(self, rev):
739 if not os.path.exists(self.local):
741 gitsvn_args = ['git', 'svn', 'clone']
742 if ';' in self.remote:
743 remote_split = self.remote.split(';')
744 for i in remote_split[1:]:
745 if i.startswith('trunk='):
746 gitsvn_args.extend(['-T', i[6:]])
747 elif i.startswith('tags='):
748 gitsvn_args.extend(['-t', i[5:]])
749 elif i.startswith('branches='):
750 gitsvn_args.extend(['-b', i[9:]])
751 gitsvn_args.extend([remote_split[0], self.local])
752 p = FDroidPopen(gitsvn_args, output=False)
753 if p.returncode != 0:
754 self.clone_failed = True
755 raise VCSException("Git svn clone failed", p.output)
757 gitsvn_args.extend([self.remote, self.local])
758 p = FDroidPopen(gitsvn_args, output=False)
759 if p.returncode != 0:
760 self.clone_failed = True
761 raise VCSException("Git svn clone failed", p.output)
765 # Discard any working tree changes
766 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
767 if p.returncode != 0:
768 raise VCSException("Git reset failed", p.output)
769 # Remove untracked files now, in case they're tracked in the target
770 # revision (it happens!)
771 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
772 if p.returncode != 0:
773 raise VCSException("Git clean failed", p.output)
774 if not self.refreshed:
775 # Get new commits, branches and tags from repo
776 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
777 if p.returncode != 0:
778 raise VCSException("Git svn fetch failed")
779 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
780 if p.returncode != 0:
781 raise VCSException("Git svn rebase failed", p.output)
782 self.refreshed = True
784 rev = rev or 'master'
786 nospaces_rev = rev.replace(' ', '%20')
787 # Try finding a svn tag
788 for treeish in ['origin/', '']:
789 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
790 if p.returncode == 0:
792 if p.returncode != 0:
793 # No tag found, normal svn rev translation
794 # Translate svn rev into git format
795 rev_split = rev.split('/')
798 for treeish in ['origin/', '']:
799 if len(rev_split) > 1:
800 treeish += rev_split[0]
801 svn_rev = rev_split[1]
804 # if no branch is specified, then assume trunk (i.e. 'master' branch):
808 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
810 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
811 git_rev = p.output.rstrip()
813 if p.returncode == 0 and git_rev:
816 if p.returncode != 0 or not git_rev:
817 # Try a plain git checkout as a last resort
818 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
819 if p.returncode != 0:
820 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
822 # Check out the git rev equivalent to the svn rev
823 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
824 if p.returncode != 0:
825 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
827 # Get rid of any uncontrolled files left behind
828 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
829 if p.returncode != 0:
830 raise VCSException("Git clean failed", p.output)
834 for treeish in ['origin/', '']:
835 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
841 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
842 if p.returncode != 0:
844 return p.output.strip()
852 def gotorevisionx(self, rev):
853 if not os.path.exists(self.local):
854 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
855 if p.returncode != 0:
856 self.clone_failed = True
857 raise VCSException("Hg clone failed", p.output)
859 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
860 if p.returncode != 0:
861 raise VCSException("Hg status failed", p.output)
862 for line in p.output.splitlines():
863 if not line.startswith('? '):
864 raise VCSException("Unexpected output from hg status -uS: " + line)
865 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
866 if not self.refreshed:
867 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
868 if p.returncode != 0:
869 raise VCSException("Hg pull failed", p.output)
870 self.refreshed = True
872 rev = rev or 'default'
875 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
876 if p.returncode != 0:
877 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
878 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
879 # Also delete untracked files, we have to enable purge extension for that:
880 if "'purge' is provided by the following extension" in p.output:
881 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
882 myfile.write("\n[extensions]\nhgext.purge=\n")
883 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
884 if p.returncode != 0:
885 raise VCSException("HG purge failed", p.output)
886 elif p.returncode != 0:
887 raise VCSException("HG purge failed", p.output)
890 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
891 return p.output.splitlines()[1:]
899 def gotorevisionx(self, rev):
900 if not os.path.exists(self.local):
901 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
902 if p.returncode != 0:
903 self.clone_failed = True
904 raise VCSException("Bzr branch failed", p.output)
906 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
907 if p.returncode != 0:
908 raise VCSException("Bzr revert failed", p.output)
909 if not self.refreshed:
910 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
911 if p.returncode != 0:
912 raise VCSException("Bzr update failed", p.output)
913 self.refreshed = True
915 revargs = list(['-r', rev] if rev else [])
916 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
917 if p.returncode != 0:
918 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
921 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
922 return [tag.split(' ')[0].strip() for tag in
923 p.output.splitlines()]
926 def unescape_string(string):
929 if string[0] == '"' and string[-1] == '"':
932 return string.replace("\\'", "'")
935 def retrieve_string(app_dir, string, xmlfiles=None):
937 if not string.startswith('@string/'):
938 return unescape_string(string)
943 os.path.join(app_dir, 'res'),
944 os.path.join(app_dir, 'src', 'main', 'res'),
946 for r, d, f in os.walk(res_dir):
947 if os.path.basename(r) == 'values':
948 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
950 name = string[len('@string/'):]
952 def element_content(element):
953 if element.text is None:
955 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
958 for path in xmlfiles:
959 if not os.path.isfile(path):
961 xml = parse_xml(path)
962 element = xml.find('string[@name="' + name + '"]')
963 if element is not None:
964 content = element_content(element)
965 return retrieve_string(app_dir, content, xmlfiles)
970 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
971 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
974 # Return list of existing files that will be used to find the highest vercode
975 def manifest_paths(app_dir, flavours):
977 possible_manifests = \
978 [os.path.join(app_dir, 'AndroidManifest.xml'),
979 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
980 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
981 os.path.join(app_dir, 'build.gradle')]
983 for flavour in flavours:
986 possible_manifests.append(
987 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
989 return [path for path in possible_manifests if os.path.isfile(path)]
992 # Retrieve the package name. Returns the name, or None if not found.
993 def fetch_real_name(app_dir, flavours):
994 for path in manifest_paths(app_dir, flavours):
995 if not has_extension(path, 'xml') or not os.path.isfile(path):
997 logging.debug("fetch_real_name: Checking manifest at " + path)
998 xml = parse_xml(path)
999 app = xml.find('application')
1002 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1004 label = app.attrib["{http://schemas.android.com/apk/res/android}label"].encode('utf-8')
1005 result = retrieve_string_singleline(app_dir, label)
1007 result = result.strip()
1012 def get_library_references(root_dir):
1014 proppath = os.path.join(root_dir, 'project.properties')
1015 if not os.path.isfile(proppath):
1017 for line in file(proppath):
1018 if not line.startswith('android.library.reference.'):
1020 path = line.split('=')[1].strip()
1021 relpath = os.path.join(root_dir, path)
1022 if not os.path.isdir(relpath):
1024 logging.debug("Found subproject at %s" % path)
1025 libraries.append(path)
1029 def ant_subprojects(root_dir):
1030 subprojects = get_library_references(root_dir)
1031 for subpath in subprojects:
1032 subrelpath = os.path.join(root_dir, subpath)
1033 for p in get_library_references(subrelpath):
1034 relp = os.path.normpath(os.path.join(subpath, p))
1035 if relp not in subprojects:
1036 subprojects.insert(0, relp)
1040 def remove_debuggable_flags(root_dir):
1041 # Remove forced debuggable flags
1042 logging.debug("Removing debuggable flags from %s" % root_dir)
1043 for root, dirs, files in os.walk(root_dir):
1044 if 'AndroidManifest.xml' in files:
1045 regsub_file(r'android:debuggable="[^"]*"',
1047 os.path.join(root, 'AndroidManifest.xml'))
1050 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
1051 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1052 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1055 def app_matches_packagename(app, package):
1058 appid = app.UpdateCheckName or app.id
1059 if appid is None or appid == "Ignore":
1061 return appid == package
1064 # Extract some information from the AndroidManifest.xml at the given path.
1065 # Returns (version, vercode, package), any or all of which might be None.
1066 # All values returned are strings.
1067 def parse_androidmanifests(paths, app):
1069 ignoreversions = app.UpdateCheckIgnore
1070 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1073 return (None, None, None)
1081 if not os.path.isfile(path):
1084 logging.debug("Parsing manifest at {0}".format(path))
1085 gradle = has_extension(path, 'gradle')
1091 for line in file(path):
1092 if gradle_comment.match(line):
1094 # Grab first occurence of each to avoid running into
1095 # alternative flavours and builds.
1097 matches = psearch_g(line)
1099 s = matches.group(2)
1100 if app_matches_packagename(app, s):
1103 matches = vnsearch_g(line)
1105 version = matches.group(2)
1107 matches = vcsearch_g(line)
1109 vercode = matches.group(1)
1112 xml = parse_xml(path)
1113 if "package" in xml.attrib:
1114 s = xml.attrib["package"].encode('utf-8')
1115 if app_matches_packagename(app, s):
1117 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1118 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"].encode('utf-8')
1119 base_dir = os.path.dirname(path)
1120 version = retrieve_string_singleline(base_dir, version)
1121 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1122 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"].encode('utf-8')
1123 if string_is_integer(a):
1126 logging.warning("Problem with xml at {0}".format(path))
1128 # Remember package name, may be defined separately from version+vercode
1130 package = max_package
1132 logging.debug("..got package={0}, version={1}, vercode={2}"
1133 .format(package, version, vercode))
1135 # Always grab the package name and version name in case they are not
1136 # together with the highest version code
1137 if max_package is None and package is not None:
1138 max_package = package
1139 if max_version is None and version is not None:
1140 max_version = version
1142 if max_vercode is None or (vercode is not None and vercode > max_vercode):
1143 if not ignoresearch or not ignoresearch(version):
1144 if version is not None:
1145 max_version = version
1146 if vercode is not None:
1147 max_vercode = vercode
1148 if package is not None:
1149 max_package = package
1151 max_version = "Ignore"
1153 if max_version is None:
1154 max_version = "Unknown"
1156 if max_package and not is_valid_package_name(max_package):
1157 raise FDroidException("Invalid package name {0}".format(max_package))
1159 return (max_version, max_vercode, max_package)
1162 def is_valid_package_name(name):
1163 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1166 class FDroidException(Exception):
1168 def __init__(self, value, detail=None):
1170 self.detail = detail
1172 def shortened_detail(self):
1173 if len(self.detail) < 16000:
1175 return '[...]\n' + self.detail[-16000:]
1177 def get_wikitext(self):
1178 ret = repr(self.value) + "\n"
1181 ret += "<pre>\n" + self.shortened_detail() + "</pre>\n"
1187 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1191 class VCSException(FDroidException):
1195 class BuildException(FDroidException):
1199 # Get the specified source library.
1200 # Returns the path to it. Normally this is the path to be used when referencing
1201 # it, which may be a subdirectory of the actual project. If you want the base
1202 # directory of the project, pass 'basepath=True'.
1203 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1204 raw=False, prepare=True, preponly=False, refresh=True):
1212 name, ref = spec.split('@')
1214 number, name = name.split(':', 1)
1216 name, subdir = name.split('/', 1)
1218 if name not in metadata.srclibs:
1219 raise VCSException('srclib ' + name + ' not found.')
1221 srclib = metadata.srclibs[name]
1223 sdir = os.path.join(srclib_dir, name)
1226 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1227 vcs.srclib = (name, number, sdir)
1229 vcs.gotorevision(ref, refresh)
1236 libdir = os.path.join(sdir, subdir)
1237 elif srclib["Subdir"]:
1238 for subdir in srclib["Subdir"]:
1239 libdir_candidate = os.path.join(sdir, subdir)
1240 if os.path.exists(libdir_candidate):
1241 libdir = libdir_candidate
1247 remove_signing_keys(sdir)
1248 remove_debuggable_flags(sdir)
1252 if srclib["Prepare"]:
1253 cmd = replace_config_vars(srclib["Prepare"], None)
1255 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1256 if p.returncode != 0:
1257 raise BuildException("Error running prepare command for srclib %s"
1263 return (name, number, libdir)
1265 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1268 # Prepare the source code for a particular build
1269 # 'vcs' - the appropriate vcs object for the application
1270 # 'app' - the application details from the metadata
1271 # 'build' - the build details from the metadata
1272 # 'build_dir' - the path to the build directory, usually
1274 # 'srclib_dir' - the path to the source libraries directory, usually
1276 # 'extlib_dir' - the path to the external libraries directory, usually
1278 # Returns the (root, srclibpaths) where:
1279 # 'root' is the root directory, which may be the same as 'build_dir' or may
1280 # be a subdirectory of it.
1281 # 'srclibpaths' is information on the srclibs being used
1282 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1284 # Optionally, the actual app source can be in a subdirectory
1286 root_dir = os.path.join(build_dir, build.subdir)
1288 root_dir = build_dir
1290 # Get a working copy of the right revision
1291 logging.info("Getting source for revision " + build.commit)
1292 vcs.gotorevision(build.commit, refresh)
1294 # Initialise submodules if required
1295 if build.submodules:
1296 logging.info("Initialising submodules")
1297 vcs.initsubmodules()
1299 # Check that a subdir (if we're using one) exists. This has to happen
1300 # after the checkout, since it might not exist elsewhere
1301 if not os.path.exists(root_dir):
1302 raise BuildException('Missing subdir ' + root_dir)
1304 # Run an init command if one is required
1306 cmd = replace_config_vars(build.init, build)
1307 logging.info("Running 'init' commands in %s" % root_dir)
1309 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1310 if p.returncode != 0:
1311 raise BuildException("Error running init command for %s:%s" %
1312 (app.id, build.version), p.output)
1314 # Apply patches if any
1316 logging.info("Applying patches")
1317 for patch in build.patch:
1318 patch = patch.strip()
1319 logging.info("Applying " + patch)
1320 patch_path = os.path.join('metadata', app.id, patch)
1321 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1322 if p.returncode != 0:
1323 raise BuildException("Failed to apply patch %s" % patch_path)
1325 # Get required source libraries
1328 logging.info("Collecting source libraries")
1329 for lib in build.srclibs:
1330 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
1332 for name, number, libpath in srclibpaths:
1333 place_srclib(root_dir, int(number) if number else None, libpath)
1335 basesrclib = vcs.getsrclib()
1336 # If one was used for the main source, add that too.
1338 srclibpaths.append(basesrclib)
1340 # Update the local.properties file
1341 localprops = [os.path.join(build_dir, 'local.properties')]
1343 parts = build.subdir.split(os.sep)
1346 cur = os.path.join(cur, d)
1347 localprops += [os.path.join(cur, 'local.properties')]
1348 for path in localprops:
1350 if os.path.isfile(path):
1351 logging.info("Updating local.properties file at %s" % path)
1352 with open(path, 'r') as f:
1356 logging.info("Creating local.properties file at %s" % path)
1357 # Fix old-fashioned 'sdk-location' by copying
1358 # from sdk.dir, if necessary
1360 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1361 re.S | re.M).group(1)
1362 props += "sdk-location=%s\n" % sdkloc
1364 props += "sdk.dir=%s\n" % config['sdk_path']
1365 props += "sdk-location=%s\n" % config['sdk_path']
1366 ndk_path = build.ndk_path()
1369 props += "ndk.dir=%s\n" % ndk_path
1370 props += "ndk-location=%s\n" % ndk_path
1371 # Add java.encoding if necessary
1373 props += "java.encoding=%s\n" % build.encoding
1374 with open(path, 'w') as f:
1378 if build.build_method() == 'gradle':
1379 flavours = build.gradle
1382 n = build.target.split('-')[1]
1383 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1384 r'compileSdkVersion %s' % n,
1385 os.path.join(root_dir, 'build.gradle'))
1387 # Remove forced debuggable flags
1388 remove_debuggable_flags(root_dir)
1390 # Insert version code and number into the manifest if necessary
1391 if build.forceversion:
1392 logging.info("Changing the version name")
1393 for path in manifest_paths(root_dir, flavours):
1394 if not os.path.isfile(path):
1396 if has_extension(path, 'xml'):
1397 regsub_file(r'android:versionName="[^"]*"',
1398 r'android:versionName="%s"' % build.version,
1400 elif has_extension(path, 'gradle'):
1401 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1402 r"""\1versionName '%s'""" % build.version,
1405 if build.forcevercode:
1406 logging.info("Changing the version code")
1407 for path in manifest_paths(root_dir, flavours):
1408 if not os.path.isfile(path):
1410 if has_extension(path, 'xml'):
1411 regsub_file(r'android:versionCode="[^"]*"',
1412 r'android:versionCode="%s"' % build.vercode,
1414 elif has_extension(path, 'gradle'):
1415 regsub_file(r'versionCode[ =]+[0-9]+',
1416 r'versionCode %s' % build.vercode,
1419 # Delete unwanted files
1421 logging.info("Removing specified files")
1422 for part in getpaths(build_dir, build.rm):
1423 dest = os.path.join(build_dir, part)
1424 logging.info("Removing {0}".format(part))
1425 if os.path.lexists(dest):
1426 if os.path.islink(dest):
1427 FDroidPopen(['unlink', dest], output=False)
1429 FDroidPopen(['rm', '-rf', dest], output=False)
1431 logging.info("...but it didn't exist")
1433 remove_signing_keys(build_dir)
1435 # Add required external libraries
1437 logging.info("Collecting prebuilt libraries")
1438 libsdir = os.path.join(root_dir, 'libs')
1439 if not os.path.exists(libsdir):
1441 for lib in build.extlibs:
1443 logging.info("...installing extlib {0}".format(lib))
1444 libf = os.path.basename(lib)
1445 libsrc = os.path.join(extlib_dir, lib)
1446 if not os.path.exists(libsrc):
1447 raise BuildException("Missing extlib file {0}".format(libsrc))
1448 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1450 # Run a pre-build command if one is required
1452 logging.info("Running 'prebuild' commands in %s" % root_dir)
1454 cmd = replace_config_vars(build.prebuild, build)
1456 # Substitute source library paths into prebuild commands
1457 for name, number, libpath in srclibpaths:
1458 libpath = os.path.relpath(libpath, root_dir)
1459 cmd = cmd.replace('$$' + name + '$$', libpath)
1461 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1462 if p.returncode != 0:
1463 raise BuildException("Error running prebuild command for %s:%s" %
1464 (app.id, build.version), p.output)
1466 # Generate (or update) the ant build file, build.xml...
1467 if build.build_method() == 'ant' and build.update != ['no']:
1468 parms = ['android', 'update', 'lib-project']
1469 lparms = ['android', 'update', 'project']
1472 parms += ['-t', build.target]
1473 lparms += ['-t', build.target]
1475 update_dirs = build.update
1477 update_dirs = ant_subprojects(root_dir) + ['.']
1479 for d in update_dirs:
1480 subdir = os.path.join(root_dir, d)
1482 logging.debug("Updating main project")
1483 cmd = parms + ['-p', d]
1485 logging.debug("Updating subproject %s" % d)
1486 cmd = lparms + ['-p', d]
1487 p = SdkToolsPopen(cmd, cwd=root_dir)
1488 # Check to see whether an error was returned without a proper exit
1489 # code (this is the case for the 'no target set or target invalid'
1491 if p.returncode != 0 or p.output.startswith("Error: "):
1492 raise BuildException("Failed to update project at %s" % d, p.output)
1493 # Clean update dirs via ant
1495 logging.info("Cleaning subproject %s" % d)
1496 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1498 return (root_dir, srclibpaths)
1501 # Extend via globbing the paths from a field and return them as a map from
1502 # original path to resulting paths
1503 def getpaths_map(build_dir, globpaths):
1507 full_path = os.path.join(build_dir, p)
1508 full_path = os.path.normpath(full_path)
1509 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1511 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1515 # Extend via globbing the paths from a field and return them as a set
1516 def getpaths(build_dir, globpaths):
1517 paths_map = getpaths_map(build_dir, globpaths)
1519 for k, v in paths_map.iteritems():
1526 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1532 self.path = os.path.join('stats', 'known_apks.txt')
1534 if os.path.isfile(self.path):
1535 for line in file(self.path):
1536 t = line.rstrip().split(' ')
1538 self.apks[t[0]] = (t[1], None)
1540 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1541 self.changed = False
1543 def writeifchanged(self):
1544 if not self.changed:
1547 if not os.path.exists('stats'):
1551 for apk, app in self.apks.iteritems():
1553 line = apk + ' ' + appid
1555 line += ' ' + time.strftime('%Y-%m-%d', added)
1558 with open(self.path, 'w') as f:
1559 for line in sorted(lst, key=natural_key):
1560 f.write(line + '\n')
1562 # Record an apk (if it's new, otherwise does nothing)
1563 # Returns the date it was added.
1564 def recordapk(self, apk, app):
1565 if apk not in self.apks:
1566 self.apks[apk] = (app, time.gmtime(time.time()))
1568 _, added = self.apks[apk]
1571 # Look up information - given the 'apkname', returns (app id, date added/None).
1572 # Or returns None for an unknown apk.
1573 def getapp(self, apkname):
1574 if apkname in self.apks:
1575 return self.apks[apkname]
1578 # Get the most recent 'num' apps added to the repo, as a list of package ids
1579 # with the most recent first.
1580 def getlatest(self, num):
1582 for apk, app in self.apks.iteritems():
1586 if apps[appid] > added:
1590 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1591 lst = [app for app, _ in sortedapps]
1596 def isApkDebuggable(apkfile, config):
1597 """Returns True if the given apk file is debuggable
1599 :param apkfile: full path to the apk to check"""
1601 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1603 if p.returncode != 0:
1604 logging.critical("Failed to get apk manifest information")
1606 for line in p.output.splitlines():
1607 if 'android:debuggable' in line and not line.endswith('0x0'):
1617 def SdkToolsPopen(commands, cwd=None, output=True):
1619 if cmd not in config:
1620 config[cmd] = find_sdk_tools_cmd(commands[0])
1621 abscmd = config[cmd]
1623 logging.critical("Could not find '%s' on your system" % cmd)
1625 return FDroidPopen([abscmd] + commands[1:],
1626 cwd=cwd, output=output)
1629 def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
1631 Run a command and capture the possibly huge output.
1633 :param commands: command and argument list like in subprocess.Popen
1634 :param cwd: optionally specifies a working directory
1635 :returns: A PopenResult.
1641 cwd = os.path.normpath(cwd)
1642 logging.debug("Directory: %s" % cwd)
1643 logging.debug("> %s" % ' '.join(commands))
1645 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
1646 result = PopenResult()
1649 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
1650 stdout=subprocess.PIPE, stderr=stderr_param)
1651 except OSError as e:
1652 raise BuildException("OSError while trying to execute " +
1653 ' '.join(commands) + ': ' + str(e))
1655 if not stderr_to_stdout and options.verbose:
1656 stderr_queue = Queue()
1657 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
1659 while not stderr_reader.eof():
1660 while not stderr_queue.empty():
1661 line = stderr_queue.get()
1662 sys.stderr.write(line)
1667 stdout_queue = Queue()
1668 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1670 # Check the queue for output (until there is no more to get)
1671 while not stdout_reader.eof():
1672 while not stdout_queue.empty():
1673 line = stdout_queue.get()
1674 if output and options.verbose:
1675 # Output directly to console
1676 sys.stderr.write(line)
1678 result.output += line
1682 result.returncode = p.wait()
1686 gradle_comment = re.compile(r'[ ]*//')
1687 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1688 gradle_line_matches = [
1689 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1690 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1691 re.compile(r'.*\.readLine\(.*'),
1695 def remove_signing_keys(build_dir):
1696 for root, dirs, files in os.walk(build_dir):
1697 if 'build.gradle' in files:
1698 path = os.path.join(root, 'build.gradle')
1700 with open(path, "r") as o:
1701 lines = o.readlines()
1707 with open(path, "w") as o:
1708 while i < len(lines):
1711 while line.endswith('\\\n'):
1712 line = line.rstrip('\\\n') + lines[i]
1715 if gradle_comment.match(line):
1720 opened += line.count('{')
1721 opened -= line.count('}')
1724 if gradle_signing_configs.match(line):
1729 if any(s.match(line) for s in gradle_line_matches):
1737 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1740 'project.properties',
1742 'default.properties',
1743 'ant.properties', ]:
1744 if propfile in files:
1745 path = os.path.join(root, propfile)
1747 with open(path, "r") as o:
1748 lines = o.readlines()
1752 with open(path, "w") as o:
1754 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1761 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1764 def reset_env_path():
1765 global env, orig_path
1766 env['PATH'] = orig_path
1769 def add_to_env_path(path):
1771 paths = env['PATH'].split(os.pathsep)
1775 env['PATH'] = os.pathsep.join(paths)
1778 def replace_config_vars(cmd, build):
1780 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1781 # env['ANDROID_NDK'] is set in build_local right before prepare_source
1782 cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
1783 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1784 if build is not None:
1785 cmd = cmd.replace('$$COMMIT$$', build.commit)
1786 cmd = cmd.replace('$$VERSION$$', build.version)
1787 cmd = cmd.replace('$$VERCODE$$', build.vercode)
1791 def place_srclib(root_dir, number, libpath):
1794 relpath = os.path.relpath(libpath, root_dir)
1795 proppath = os.path.join(root_dir, 'project.properties')
1798 if os.path.isfile(proppath):
1799 with open(proppath, "r") as o:
1800 lines = o.readlines()
1802 with open(proppath, "w") as o:
1805 if line.startswith('android.library.reference.%d=' % number):
1806 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1811 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1813 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
1816 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
1817 """Verify that two apks are the same
1819 One of the inputs is signed, the other is unsigned. The signature metadata
1820 is transferred from the signed to the unsigned apk, and then jarsigner is
1821 used to verify that the signature from the signed apk is also varlid for
1823 :param signed_apk: Path to a signed apk file
1824 :param unsigned_apk: Path to an unsigned apk file expected to match it
1825 :param tmp_dir: Path to directory for temporary files
1826 :returns: None if the verification is successful, otherwise a string
1827 describing what went wrong.
1829 with ZipFile(signed_apk) as signed_apk_as_zip:
1830 meta_inf_files = ['META-INF/MANIFEST.MF']
1831 for f in signed_apk_as_zip.namelist():
1832 if apk_sigfile.match(f):
1833 meta_inf_files.append(f)
1834 if len(meta_inf_files) < 3:
1835 return "Signature files missing from {0}".format(signed_apk)
1836 signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
1837 with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
1838 for meta_inf_file in meta_inf_files:
1839 unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
1841 if subprocess.call([config['jarsigner'], '-verify', unsigned_apk]) != 0:
1842 logging.info("...NOT verified - {0}".format(signed_apk))
1843 return compare_apks(signed_apk, unsigned_apk, tmp_dir)
1844 logging.info("...successfully verified")
1847 apk_badchars = re.compile('''[/ :;'"]''')
1850 def compare_apks(apk1, apk2, tmp_dir):
1853 Returns None if the apk content is the same (apart from the signing key),
1854 otherwise a string describing what's different, or what went wrong when
1855 trying to do the comparison.
1858 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
1859 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
1860 for d in [apk1dir, apk2dir]:
1861 if os.path.exists(d):
1864 os.mkdir(os.path.join(d, 'jar-xf'))
1866 if subprocess.call(['jar', 'xf',
1867 os.path.abspath(apk1)],
1868 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
1869 return("Failed to unpack " + apk1)
1870 if subprocess.call(['jar', 'xf',
1871 os.path.abspath(apk2)],
1872 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
1873 return("Failed to unpack " + apk2)
1875 # try to find apktool in the path, if it hasn't been manually configed
1876 if 'apktool' not in config:
1877 tmp = find_command('apktool')
1879 config['apktool'] = tmp
1880 if 'apktool' in config:
1881 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
1883 return("Failed to unpack " + apk1)
1884 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
1886 return("Failed to unpack " + apk2)
1888 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
1889 lines = p.output.splitlines()
1890 if len(lines) != 1 or 'META-INF' not in lines[0]:
1891 meld = find_command('meld')
1892 if meld is not None:
1893 p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
1894 return("Unexpected diff output - " + p.output)
1896 # since everything verifies, delete the comparison to keep cruft down
1897 shutil.rmtree(apk1dir)
1898 shutil.rmtree(apk2dir)
1900 # If we get here, it seems like they're the same!
1904 def find_command(command):
1905 '''find the full path of a command, or None if it can't be found in the PATH'''
1908 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
1910 fpath, fname = os.path.split(command)
1915 for path in os.environ["PATH"].split(os.pathsep):
1916 path = path.strip('"')
1917 exe_file = os.path.join(path, command)
1918 if is_exe(exe_file):
1925 '''generate a random password for when generating keys'''
1926 h = hashlib.sha256()
1927 h.update(os.urandom(16)) # salt
1928 h.update(bytes(socket.getfqdn()))
1929 return h.digest().encode('base64').strip()
1932 def genkeystore(localconfig):
1933 '''Generate a new key with random passwords and add it to new keystore'''
1934 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
1935 keystoredir = os.path.dirname(localconfig['keystore'])
1936 if keystoredir is None or keystoredir == '':
1937 keystoredir = os.path.join(os.getcwd(), keystoredir)
1938 if not os.path.exists(keystoredir):
1939 os.makedirs(keystoredir, mode=0o700)
1941 write_password_file("keystorepass", localconfig['keystorepass'])
1942 write_password_file("keypass", localconfig['keypass'])
1943 p = FDroidPopen([config['keytool'], '-genkey',
1944 '-keystore', localconfig['keystore'],
1945 '-alias', localconfig['repo_keyalias'],
1946 '-keyalg', 'RSA', '-keysize', '4096',
1947 '-sigalg', 'SHA256withRSA',
1948 '-validity', '10000',
1949 '-storepass:file', config['keystorepassfile'],
1950 '-keypass:file', config['keypassfile'],
1951 '-dname', localconfig['keydname']])
1952 # TODO keypass should be sent via stdin
1953 if p.returncode != 0:
1954 raise BuildException("Failed to generate key", p.output)
1955 os.chmod(localconfig['keystore'], 0o0600)
1956 # now show the lovely key that was just generated
1957 p = FDroidPopen([config['keytool'], '-list', '-v',
1958 '-keystore', localconfig['keystore'],
1959 '-alias', localconfig['repo_keyalias'],
1960 '-storepass:file', config['keystorepassfile']])
1961 logging.info(p.output.strip() + '\n\n')
1964 def write_to_config(thisconfig, key, value=None):
1965 '''write a key/value to the local config.py'''
1967 origkey = key + '_orig'
1968 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
1969 with open('config.py', 'r') as f:
1971 pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
1972 repl = '\n' + key + ' = "' + value + '"'
1973 data = re.sub(pattern, repl, data)
1974 # if this key is not in the file, append it
1975 if not re.match('\s*' + key + '\s*=\s*"', data):
1977 # make sure the file ends with a carraige return
1978 if not re.match('\n$', data):
1980 with open('config.py', 'w') as f:
1984 def parse_xml(path):
1985 return XMLElementTree.parse(path).getroot()
1988 def string_is_integer(string):
1996 def get_per_app_repos():
1997 '''per-app repos are dirs named with the packageName of a single app'''
1999 # Android packageNames are Java packages, they may contain uppercase or
2000 # lowercase letters ('A' through 'Z'), numbers, and underscores
2001 # ('_'). However, individual package name parts may only start with
2002 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2003 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2006 for root, dirs, files in os.walk(os.getcwd()):
2008 print('checking', root, 'for', d)
2009 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2010 # standard parts of an fdroid repo, so never packageNames
2013 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):