1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
35 import xml.etree.ElementTree as XMLElementTree
39 from Queue import Queue
42 from queue import Queue
44 from zipfile import ZipFile
47 from fdroidserver.asynchronousfilereader import AsynchronousFileReader
50 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
59 'sdk_path': "$ANDROID_HOME",
62 'r10e': "$ANDROID_NDK",
64 'build_tools': "23.0.2",
69 'accepted_formats': ['txt', 'yaml'],
70 'sync_from_local_copy_dir': False,
71 'per_app_repos': False,
72 'make_current_version_link': True,
73 'current_version_name_source': 'Name',
74 'update_stats': False,
78 'stats_to_carbon': False,
80 'build_server_always': False,
81 'keystore': 'keystore.jks',
82 'smartcardoptions': [],
88 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
89 'repo_name': "My First FDroid Repo Demo",
90 'repo_icon': "fdroid-icon.png",
91 'repo_description': '''
92 This is a repository of apps to be used with FDroid. Applications in this
93 repository are either official binaries built by the original application
94 developers, or are binaries built from source by the admin of f-droid.org
95 using the tools on https://gitlab.com/u/fdroid.
101 def setup_global_opts(parser):
102 parser.add_argument("-v", "--verbose", action="store_true", default=False,
103 help="Spew out even more information than normal")
104 parser.add_argument("-q", "--quiet", action="store_true", default=False,
105 help="Restrict output to warnings and errors")
108 def fill_config_defaults(thisconfig):
109 for k, v in default_config.items():
110 if k not in thisconfig:
113 # Expand paths (~users and $vars)
114 def expand_path(path):
118 path = os.path.expanduser(path)
119 path = os.path.expandvars(path)
124 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
129 thisconfig[k + '_orig'] = v
131 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
132 if thisconfig['java_paths'] is None:
133 thisconfig['java_paths'] = dict()
134 for d in sorted(glob.glob('/usr/lib/jvm/j*[6-9]*')
135 + glob.glob('/usr/java/jdk1.[6-9]*')
136 + glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
137 + glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')):
138 if os.path.islink(d):
140 j = os.path.basename(d)
141 # the last one found will be the canonical one, so order appropriately
142 for regex in (r'1\.([6-9])\.0\.jdk', # OSX
143 r'jdk1\.([6-9])\.0_[0-9]+.jdk', # OSX and Oracle tarball
144 r'jdk([6-9])-openjdk', # Arch
145 r'java-([6-9])-openjdk', # Arch
146 r'java-([6-9])-jdk', # Arch (oracle)
147 r'java-1\.([6-9])\.0-.*', # RedHat
148 r'java-([6-9])-oracle', # Debian WebUpd8
149 r'jdk-([6-9])-oracle-.*', # Debian make-jpkg
150 r'java-([6-9])-openjdk-[^c][^o][^m].*'): # Debian
151 m = re.match(regex, j)
153 osxhome = os.path.join(d, 'Contents', 'Home')
154 if os.path.exists(osxhome):
155 thisconfig['java_paths'][m.group(1)] = osxhome
157 thisconfig['java_paths'][m.group(1)] = d
159 for java_version in ('7', '8', '9'):
160 if java_version not in thisconfig['java_paths']:
162 java_home = thisconfig['java_paths'][java_version]
163 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
164 if os.path.exists(jarsigner):
165 thisconfig['jarsigner'] = jarsigner
166 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
167 break # Java7 is preferred, so quit if found
169 for k in ['ndk_paths', 'java_paths']:
175 thisconfig[k][k2] = exp
176 thisconfig[k][k2 + '_orig'] = v
179 def regsub_file(pattern, repl, path):
180 with open(path, 'r') as f:
182 text = re.sub(pattern, repl, text)
183 with open(path, 'w') as f:
187 def read_config(opts, config_file='config.py'):
188 """Read the repository config
190 The config is read from config_file, which is in the current directory when
191 any of the repo management commands are used.
193 global config, options, env, orig_path
195 if config is not None:
197 if not os.path.isfile(config_file):
198 logging.critical("Missing config file - is this a repo directory?")
205 logging.debug("Reading %s" % config_file)
206 execfile(config_file, config)
208 # smartcardoptions must be a list since its command line args for Popen
209 if 'smartcardoptions' in config:
210 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
211 elif 'keystore' in config and config['keystore'] == 'NONE':
212 # keystore='NONE' means use smartcard, these are required defaults
213 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
214 'SunPKCS11-OpenSC', '-providerClass',
215 'sun.security.pkcs11.SunPKCS11',
216 '-providerArg', 'opensc-fdroid.cfg']
218 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
219 st = os.stat(config_file)
220 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
221 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
223 fill_config_defaults(config)
225 # There is no standard, so just set up the most common environment
228 orig_path = env['PATH']
229 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
230 env[n] = config['sdk_path']
232 for k, v in config['java_paths'].items():
233 env['JAVA%s_HOME' % k] = v
235 for k in ["keystorepass", "keypass"]:
237 write_password_file(k)
239 for k in ["repo_description", "archive_description"]:
241 config[k] = clean_description(config[k])
243 if 'serverwebroot' in config:
244 if isinstance(config['serverwebroot'], basestring):
245 roots = [config['serverwebroot']]
246 elif all(isinstance(item, basestring) for item in config['serverwebroot']):
247 roots = config['serverwebroot']
249 raise TypeError('only accepts strings, lists, and tuples')
251 for rootstr in roots:
252 # since this is used with rsync, where trailing slashes have
253 # meaning, ensure there is always a trailing slash
254 if rootstr[-1] != '/':
256 rootlist.append(rootstr.replace('//', '/'))
257 config['serverwebroot'] = rootlist
262 def find_sdk_tools_cmd(cmd):
263 '''find a working path to a tool from the Android SDK'''
266 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
267 # try to find a working path to this command, in all the recent possible paths
268 if 'build_tools' in config:
269 build_tools = os.path.join(config['sdk_path'], 'build-tools')
270 # if 'build_tools' was manually set and exists, check only that one
271 configed_build_tools = os.path.join(build_tools, config['build_tools'])
272 if os.path.exists(configed_build_tools):
273 tooldirs.append(configed_build_tools)
275 # no configed version, so hunt known paths for it
276 for f in sorted(os.listdir(build_tools), reverse=True):
277 if os.path.isdir(os.path.join(build_tools, f)):
278 tooldirs.append(os.path.join(build_tools, f))
279 tooldirs.append(build_tools)
280 sdk_tools = os.path.join(config['sdk_path'], 'tools')
281 if os.path.exists(sdk_tools):
282 tooldirs.append(sdk_tools)
283 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
284 if os.path.exists(sdk_platform_tools):
285 tooldirs.append(sdk_platform_tools)
286 tooldirs.append('/usr/bin')
288 if os.path.isfile(os.path.join(d, cmd)):
289 return os.path.join(d, cmd)
290 # did not find the command, exit with error message
291 ensure_build_tools_exists(config)
294 def test_sdk_exists(thisconfig):
295 if 'sdk_path' not in thisconfig:
296 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
299 logging.error("'sdk_path' not set in config.py!")
301 if thisconfig['sdk_path'] == default_config['sdk_path']:
302 logging.error('No Android SDK found!')
303 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
304 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
306 if not os.path.exists(thisconfig['sdk_path']):
307 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
309 if not os.path.isdir(thisconfig['sdk_path']):
310 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
312 for d in ['build-tools', 'platform-tools', 'tools']:
313 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
314 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
315 thisconfig['sdk_path'], d))
320 def ensure_build_tools_exists(thisconfig):
321 if not test_sdk_exists(thisconfig):
323 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
324 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
325 if not os.path.isdir(versioned_build_tools):
326 logging.critical('Android Build Tools path "'
327 + versioned_build_tools + '" does not exist!')
331 def write_password_file(pwtype, password=None):
333 writes out passwords to a protected file instead of passing passwords as
334 command line argments
336 filename = '.fdroid.' + pwtype + '.txt'
337 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
339 os.write(fd, config[pwtype])
341 os.write(fd, password)
343 config[pwtype + 'file'] = filename
346 # Given the arguments in the form of multiple appid:[vc] strings, this returns
347 # a dictionary with the set of vercodes specified for each package.
348 def read_pkg_args(args, allow_vercodes=False):
355 if allow_vercodes and ':' in p:
356 package, vercode = p.split(':')
358 package, vercode = p, None
359 if package not in vercodes:
360 vercodes[package] = [vercode] if vercode else []
362 elif vercode and vercode not in vercodes[package]:
363 vercodes[package] += [vercode] if vercode else []
368 # On top of what read_pkg_args does, this returns the whole app metadata, but
369 # limiting the builds list to the builds matching the vercodes specified.
370 def read_app_args(args, allapps, allow_vercodes=False):
372 vercodes = read_pkg_args(args, allow_vercodes)
378 for appid, app in allapps.iteritems():
379 if appid in vercodes:
382 if len(apps) != len(vercodes):
385 logging.critical("No such package: %s" % p)
386 raise FDroidException("Found invalid app ids in arguments")
388 raise FDroidException("No packages specified")
391 for appid, app in apps.iteritems():
395 app.builds = [b for b in app.builds if b.vercode in vc]
396 if len(app.builds) != len(vercodes[appid]):
398 allvcs = [b.vercode for b in app.builds]
399 for v in vercodes[appid]:
401 logging.critical("No such vercode %s for app %s" % (v, appid))
404 raise FDroidException("Found invalid vercodes for some apps")
409 def get_extension(filename):
410 base, ext = os.path.splitext(filename)
413 return base, ext.lower()[1:]
416 def has_extension(filename, ext):
417 _, f_ext = get_extension(filename)
421 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
424 def clean_description(description):
425 'Remove unneeded newlines and spaces from a block of description text'
427 # this is split up by paragraph to make removing the newlines easier
428 for paragraph in re.split(r'\n\n', description):
429 paragraph = re.sub('\r', '', paragraph)
430 paragraph = re.sub('\n', ' ', paragraph)
431 paragraph = re.sub(' {2,}', ' ', paragraph)
432 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
433 returnstring += paragraph + '\n\n'
434 return returnstring.rstrip('\n')
437 def apknameinfo(filename):
438 filename = os.path.basename(filename)
439 m = apk_regex.match(filename)
441 result = (m.group(1), m.group(2))
442 except AttributeError:
443 raise FDroidException("Invalid apk name: %s" % filename)
447 def getapkname(app, build):
448 return "%s_%s.apk" % (app.id, build.vercode)
451 def getsrcname(app, build):
452 return "%s_%s_src.tar.gz" % (app.id, build.vercode)
464 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
467 def getvcs(vcstype, remote, local):
469 return vcs_git(remote, local)
470 if vcstype == 'git-svn':
471 return vcs_gitsvn(remote, local)
473 return vcs_hg(remote, local)
475 return vcs_bzr(remote, local)
476 if vcstype == 'srclib':
477 if local != os.path.join('build', 'srclib', remote):
478 raise VCSException("Error: srclib paths are hard-coded!")
479 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
481 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
482 raise VCSException("Invalid vcs type " + vcstype)
485 def getsrclibvcs(name):
486 if name not in metadata.srclibs:
487 raise VCSException("Missing srclib " + name)
488 return metadata.srclibs[name]['Repo Type']
493 def __init__(self, remote, local):
495 # svn, git-svn and bzr may require auth
497 if self.repotype() in ('git-svn', 'bzr'):
499 if self.repotype == 'git-svn':
500 raise VCSException("Authentication is not supported for git-svn")
501 self.username, remote = remote.split('@')
502 if ':' not in self.username:
503 raise VCSException("Password required with username")
504 self.username, self.password = self.username.split(':')
508 self.clone_failed = False
509 self.refreshed = False
515 # Take the local repository to a clean version of the given revision, which
516 # is specificed in the VCS's native format. Beforehand, the repository can
517 # be dirty, or even non-existent. If the repository does already exist
518 # locally, it will be updated from the origin, but only once in the
519 # lifetime of the vcs object.
520 # None is acceptable for 'rev' if you know you are cloning a clean copy of
521 # the repo - otherwise it must specify a valid revision.
522 def gotorevision(self, rev, refresh=True):
524 if self.clone_failed:
525 raise VCSException("Downloading the repository already failed once, not trying again.")
527 # The .fdroidvcs-id file for a repo tells us what VCS type
528 # and remote that directory was created from, allowing us to drop it
529 # automatically if either of those things changes.
530 fdpath = os.path.join(self.local, '..',
531 '.fdroidvcs-' + os.path.basename(self.local))
532 cdata = self.repotype() + ' ' + self.remote
535 if os.path.exists(self.local):
536 if os.path.exists(fdpath):
537 with open(fdpath, 'r') as f:
538 fsdata = f.read().strip()
543 logging.info("Repository details for %s changed - deleting" % (
547 logging.info("Repository details for %s missing - deleting" % (
550 shutil.rmtree(self.local)
554 self.refreshed = True
557 self.gotorevisionx(rev)
558 except FDroidException as e:
561 # If necessary, write the .fdroidvcs file.
562 if writeback and not self.clone_failed:
563 with open(fdpath, 'w') as f:
569 # Derived classes need to implement this. It's called once basic checking
570 # has been performend.
571 def gotorevisionx(self, rev):
572 raise VCSException("This VCS type doesn't define gotorevisionx")
574 # Initialise and update submodules
575 def initsubmodules(self):
576 raise VCSException('Submodules not supported for this vcs type')
578 # Get a list of all known tags
580 if not self._gettags:
581 raise VCSException('gettags not supported for this vcs type')
583 for tag in self._gettags():
584 if re.match('[-A-Za-z0-9_. /]+$', tag):
588 def latesttags(self, tags, number):
589 """Get the most recent tags in a given list.
591 :param tags: a list of tags
592 :param number: the number to return
593 :returns: A list containing the most recent tags in the provided
594 list, up to the maximum number given.
596 raise VCSException('latesttags not supported for this vcs type')
598 # Get current commit reference (hash, revision, etc)
600 raise VCSException('getref not supported for this vcs type')
602 # Returns the srclib (name, path) used in setting up the current
613 # If the local directory exists, but is somehow not a git repository, git
614 # will traverse up the directory tree until it finds one that is (i.e.
615 # fdroidserver) and then we'll proceed to destroy it! This is called as
618 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
619 result = p.output.rstrip()
620 if not result.endswith(self.local):
621 raise VCSException('Repository mismatch')
623 def gotorevisionx(self, rev):
624 if not os.path.exists(self.local):
626 p = FDroidPopen(['git', 'clone', self.remote, self.local])
627 if p.returncode != 0:
628 self.clone_failed = True
629 raise VCSException("Git clone failed", p.output)
633 # Discard any working tree changes
634 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
635 'git', 'reset', '--hard'], cwd=self.local, output=False)
636 if p.returncode != 0:
637 raise VCSException("Git reset failed", p.output)
638 # Remove untracked files now, in case they're tracked in the target
639 # revision (it happens!)
640 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
641 'git', 'clean', '-dffx'], cwd=self.local, output=False)
642 if p.returncode != 0:
643 raise VCSException("Git clean failed", p.output)
644 if not self.refreshed:
645 # Get latest commits and tags from remote
646 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
647 if p.returncode != 0:
648 raise VCSException("Git fetch failed", p.output)
649 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
650 if p.returncode != 0:
651 raise VCSException("Git fetch failed", p.output)
652 # Recreate origin/HEAD as git clone would do it, in case it disappeared
653 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
654 if p.returncode != 0:
655 lines = p.output.splitlines()
656 if 'Multiple remote HEAD branches' not in lines[0]:
657 raise VCSException("Git remote set-head failed", p.output)
658 branch = lines[1].split(' ')[-1]
659 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
660 if p2.returncode != 0:
661 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
662 self.refreshed = True
663 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
664 # a github repo. Most of the time this is the same as origin/master.
665 rev = rev or 'origin/HEAD'
666 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
667 if p.returncode != 0:
668 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
669 # Get rid of any uncontrolled files left behind
670 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
671 if p.returncode != 0:
672 raise VCSException("Git clean failed", p.output)
674 def initsubmodules(self):
676 submfile = os.path.join(self.local, '.gitmodules')
677 if not os.path.isfile(submfile):
678 raise VCSException("No git submodules available")
680 # fix submodules not accessible without an account and public key auth
681 with open(submfile, 'r') as f:
682 lines = f.readlines()
683 with open(submfile, 'w') as f:
685 if 'git@github.com' in line:
686 line = line.replace('git@github.com:', 'https://github.com/')
687 if 'git@gitlab.com' in line:
688 line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
691 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
692 if p.returncode != 0:
693 raise VCSException("Git submodule sync failed", p.output)
694 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
695 if p.returncode != 0:
696 raise VCSException("Git submodule update failed", p.output)
700 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
701 return p.output.splitlines()
703 def latesttags(self, tags, number):
708 ['git', 'show', '--format=format:%ct', '-s', tag],
709 cwd=self.local, output=False)
710 # Timestamp is on the last line. For a normal tag, it's the only
711 # line, but for annotated tags, the rest of the info precedes it.
712 ts = int(p.output.splitlines()[-1])
715 for _, t in sorted(tl)[-number:]:
720 class vcs_gitsvn(vcs):
725 # If the local directory exists, but is somehow not a git repository, git
726 # will traverse up the directory tree until it finds one that is (i.e.
727 # fdroidserver) and then we'll proceed to destory it! This is called as
730 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
731 result = p.output.rstrip()
732 if not result.endswith(self.local):
733 raise VCSException('Repository mismatch')
735 def gotorevisionx(self, rev):
736 if not os.path.exists(self.local):
738 gitsvn_args = ['git', 'svn', 'clone']
739 if ';' in self.remote:
740 remote_split = self.remote.split(';')
741 for i in remote_split[1:]:
742 if i.startswith('trunk='):
743 gitsvn_args.extend(['-T', i[6:]])
744 elif i.startswith('tags='):
745 gitsvn_args.extend(['-t', i[5:]])
746 elif i.startswith('branches='):
747 gitsvn_args.extend(['-b', i[9:]])
748 gitsvn_args.extend([remote_split[0], self.local])
749 p = FDroidPopen(gitsvn_args, output=False)
750 if p.returncode != 0:
751 self.clone_failed = True
752 raise VCSException("Git svn clone failed", p.output)
754 gitsvn_args.extend([self.remote, self.local])
755 p = FDroidPopen(gitsvn_args, output=False)
756 if p.returncode != 0:
757 self.clone_failed = True
758 raise VCSException("Git svn clone failed", p.output)
762 # Discard any working tree changes
763 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
764 if p.returncode != 0:
765 raise VCSException("Git reset failed", p.output)
766 # Remove untracked files now, in case they're tracked in the target
767 # revision (it happens!)
768 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
769 if p.returncode != 0:
770 raise VCSException("Git clean failed", p.output)
771 if not self.refreshed:
772 # Get new commits, branches and tags from repo
773 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
774 if p.returncode != 0:
775 raise VCSException("Git svn fetch failed")
776 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
777 if p.returncode != 0:
778 raise VCSException("Git svn rebase failed", p.output)
779 self.refreshed = True
781 rev = rev or 'master'
783 nospaces_rev = rev.replace(' ', '%20')
784 # Try finding a svn tag
785 for treeish in ['origin/', '']:
786 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
787 if p.returncode == 0:
789 if p.returncode != 0:
790 # No tag found, normal svn rev translation
791 # Translate svn rev into git format
792 rev_split = rev.split('/')
795 for treeish in ['origin/', '']:
796 if len(rev_split) > 1:
797 treeish += rev_split[0]
798 svn_rev = rev_split[1]
801 # if no branch is specified, then assume trunk (i.e. 'master' branch):
805 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
807 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
808 git_rev = p.output.rstrip()
810 if p.returncode == 0 and git_rev:
813 if p.returncode != 0 or not git_rev:
814 # Try a plain git checkout as a last resort
815 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
816 if p.returncode != 0:
817 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
819 # Check out the git rev equivalent to the svn rev
820 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
821 if p.returncode != 0:
822 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
824 # Get rid of any uncontrolled files left behind
825 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
826 if p.returncode != 0:
827 raise VCSException("Git clean failed", p.output)
831 for treeish in ['origin/', '']:
832 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
838 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
839 if p.returncode != 0:
841 return p.output.strip()
849 def gotorevisionx(self, rev):
850 if not os.path.exists(self.local):
851 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
852 if p.returncode != 0:
853 self.clone_failed = True
854 raise VCSException("Hg clone failed", p.output)
856 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
857 if p.returncode != 0:
858 raise VCSException("Hg status failed", p.output)
859 for line in p.output.splitlines():
860 if not line.startswith('? '):
861 raise VCSException("Unexpected output from hg status -uS: " + line)
862 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
863 if not self.refreshed:
864 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
865 if p.returncode != 0:
866 raise VCSException("Hg pull failed", p.output)
867 self.refreshed = True
869 rev = rev or 'default'
872 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
873 if p.returncode != 0:
874 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
875 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
876 # Also delete untracked files, we have to enable purge extension for that:
877 if "'purge' is provided by the following extension" in p.output:
878 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
879 myfile.write("\n[extensions]\nhgext.purge=\n")
880 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
881 if p.returncode != 0:
882 raise VCSException("HG purge failed", p.output)
883 elif p.returncode != 0:
884 raise VCSException("HG purge failed", p.output)
887 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
888 return p.output.splitlines()[1:]
896 def gotorevisionx(self, rev):
897 if not os.path.exists(self.local):
898 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
899 if p.returncode != 0:
900 self.clone_failed = True
901 raise VCSException("Bzr branch failed", p.output)
903 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
904 if p.returncode != 0:
905 raise VCSException("Bzr revert failed", p.output)
906 if not self.refreshed:
907 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
908 if p.returncode != 0:
909 raise VCSException("Bzr update failed", p.output)
910 self.refreshed = True
912 revargs = list(['-r', rev] if rev else [])
913 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
914 if p.returncode != 0:
915 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
918 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
919 return [tag.split(' ')[0].strip() for tag in
920 p.output.splitlines()]
923 def unescape_string(string):
926 if string[0] == '"' and string[-1] == '"':
929 return string.replace("\\'", "'")
932 def retrieve_string(app_dir, string, xmlfiles=None):
934 if not string.startswith('@string/'):
935 return unescape_string(string)
940 os.path.join(app_dir, 'res'),
941 os.path.join(app_dir, 'src', 'main', 'res'),
943 for r, d, f in os.walk(res_dir):
944 if os.path.basename(r) == 'values':
945 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
947 name = string[len('@string/'):]
949 def element_content(element):
950 if element.text is None:
952 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
955 for path in xmlfiles:
956 if not os.path.isfile(path):
958 xml = parse_xml(path)
959 element = xml.find('string[@name="' + name + '"]')
960 if element is not None:
961 content = element_content(element)
962 return retrieve_string(app_dir, content, xmlfiles)
967 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
968 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
971 # Return list of existing files that will be used to find the highest vercode
972 def manifest_paths(app_dir, flavours):
974 possible_manifests = \
975 [os.path.join(app_dir, 'AndroidManifest.xml'),
976 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
977 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
978 os.path.join(app_dir, 'build.gradle')]
980 for flavour in flavours:
983 possible_manifests.append(
984 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
986 return [path for path in possible_manifests if os.path.isfile(path)]
989 # Retrieve the package name. Returns the name, or None if not found.
990 def fetch_real_name(app_dir, flavours):
991 for path in manifest_paths(app_dir, flavours):
992 if not has_extension(path, 'xml') or not os.path.isfile(path):
994 logging.debug("fetch_real_name: Checking manifest at " + path)
995 xml = parse_xml(path)
996 app = xml.find('application')
999 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1001 label = app.attrib["{http://schemas.android.com/apk/res/android}label"].encode('utf-8')
1002 result = retrieve_string_singleline(app_dir, label)
1004 result = result.strip()
1009 def get_library_references(root_dir):
1011 proppath = os.path.join(root_dir, 'project.properties')
1012 if not os.path.isfile(proppath):
1014 for line in file(proppath):
1015 if not line.startswith('android.library.reference.'):
1017 path = line.split('=')[1].strip()
1018 relpath = os.path.join(root_dir, path)
1019 if not os.path.isdir(relpath):
1021 logging.debug("Found subproject at %s" % path)
1022 libraries.append(path)
1026 def ant_subprojects(root_dir):
1027 subprojects = get_library_references(root_dir)
1028 for subpath in subprojects:
1029 subrelpath = os.path.join(root_dir, subpath)
1030 for p in get_library_references(subrelpath):
1031 relp = os.path.normpath(os.path.join(subpath, p))
1032 if relp not in subprojects:
1033 subprojects.insert(0, relp)
1037 def remove_debuggable_flags(root_dir):
1038 # Remove forced debuggable flags
1039 logging.debug("Removing debuggable flags from %s" % root_dir)
1040 for root, dirs, files in os.walk(root_dir):
1041 if 'AndroidManifest.xml' in files:
1042 regsub_file(r'android:debuggable="[^"]*"',
1044 os.path.join(root, 'AndroidManifest.xml'))
1047 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
1048 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1049 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1052 def app_matches_packagename(app, package):
1055 appid = app.UpdateCheckName or app.id
1056 if appid is None or appid == "Ignore":
1058 return appid == package
1061 # Extract some information from the AndroidManifest.xml at the given path.
1062 # Returns (version, vercode, package), any or all of which might be None.
1063 # All values returned are strings.
1064 def parse_androidmanifests(paths, app):
1066 ignoreversions = app.UpdateCheckIgnore
1067 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1070 return (None, None, None)
1078 if not os.path.isfile(path):
1081 logging.debug("Parsing manifest at {0}".format(path))
1082 gradle = has_extension(path, 'gradle')
1088 for line in file(path):
1089 if gradle_comment.match(line):
1091 # Grab first occurence of each to avoid running into
1092 # alternative flavours and builds.
1094 matches = psearch_g(line)
1096 s = matches.group(2)
1097 if app_matches_packagename(app, s):
1100 matches = vnsearch_g(line)
1102 version = matches.group(2)
1104 matches = vcsearch_g(line)
1106 vercode = matches.group(1)
1109 xml = parse_xml(path)
1110 if "package" in xml.attrib:
1111 s = xml.attrib["package"].encode('utf-8')
1112 if app_matches_packagename(app, s):
1114 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1115 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"].encode('utf-8')
1116 base_dir = os.path.dirname(path)
1117 version = retrieve_string_singleline(base_dir, version)
1118 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1119 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"].encode('utf-8')
1120 if string_is_integer(a):
1123 logging.warning("Problem with xml at {0}".format(path))
1125 # Remember package name, may be defined separately from version+vercode
1127 package = max_package
1129 logging.debug("..got package={0}, version={1}, vercode={2}"
1130 .format(package, version, vercode))
1132 # Always grab the package name and version name in case they are not
1133 # together with the highest version code
1134 if max_package is None and package is not None:
1135 max_package = package
1136 if max_version is None and version is not None:
1137 max_version = version
1139 if max_vercode is None or (vercode is not None and vercode > max_vercode):
1140 if not ignoresearch or not ignoresearch(version):
1141 if version is not None:
1142 max_version = version
1143 if vercode is not None:
1144 max_vercode = vercode
1145 if package is not None:
1146 max_package = package
1148 max_version = "Ignore"
1150 if max_version is None:
1151 max_version = "Unknown"
1153 if max_package and not is_valid_package_name(max_package):
1154 raise FDroidException("Invalid package name {0}".format(max_package))
1156 return (max_version, max_vercode, max_package)
1159 def is_valid_package_name(name):
1160 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1163 class FDroidException(Exception):
1165 def __init__(self, value, detail=None):
1167 self.detail = detail
1169 def shortened_detail(self):
1170 if len(self.detail) < 16000:
1172 return '[...]\n' + self.detail[-16000:]
1174 def get_wikitext(self):
1175 ret = repr(self.value) + "\n"
1178 ret += "<pre>\n" + self.shortened_detail() + "</pre>\n"
1184 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1188 class VCSException(FDroidException):
1192 class BuildException(FDroidException):
1196 # Get the specified source library.
1197 # Returns the path to it. Normally this is the path to be used when referencing
1198 # it, which may be a subdirectory of the actual project. If you want the base
1199 # directory of the project, pass 'basepath=True'.
1200 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1201 raw=False, prepare=True, preponly=False, refresh=True):
1209 name, ref = spec.split('@')
1211 number, name = name.split(':', 1)
1213 name, subdir = name.split('/', 1)
1215 if name not in metadata.srclibs:
1216 raise VCSException('srclib ' + name + ' not found.')
1218 srclib = metadata.srclibs[name]
1220 sdir = os.path.join(srclib_dir, name)
1223 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1224 vcs.srclib = (name, number, sdir)
1226 vcs.gotorevision(ref, refresh)
1233 libdir = os.path.join(sdir, subdir)
1234 elif srclib["Subdir"]:
1235 for subdir in srclib["Subdir"]:
1236 libdir_candidate = os.path.join(sdir, subdir)
1237 if os.path.exists(libdir_candidate):
1238 libdir = libdir_candidate
1244 remove_signing_keys(sdir)
1245 remove_debuggable_flags(sdir)
1249 if srclib["Prepare"]:
1250 cmd = replace_config_vars(srclib["Prepare"], None)
1252 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1253 if p.returncode != 0:
1254 raise BuildException("Error running prepare command for srclib %s"
1260 return (name, number, libdir)
1262 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1265 # Prepare the source code for a particular build
1266 # 'vcs' - the appropriate vcs object for the application
1267 # 'app' - the application details from the metadata
1268 # 'build' - the build details from the metadata
1269 # 'build_dir' - the path to the build directory, usually
1271 # 'srclib_dir' - the path to the source libraries directory, usually
1273 # 'extlib_dir' - the path to the external libraries directory, usually
1275 # Returns the (root, srclibpaths) where:
1276 # 'root' is the root directory, which may be the same as 'build_dir' or may
1277 # be a subdirectory of it.
1278 # 'srclibpaths' is information on the srclibs being used
1279 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1281 # Optionally, the actual app source can be in a subdirectory
1283 root_dir = os.path.join(build_dir, build.subdir)
1285 root_dir = build_dir
1287 # Get a working copy of the right revision
1288 logging.info("Getting source for revision " + build.commit)
1289 vcs.gotorevision(build.commit, refresh)
1291 # Initialise submodules if required
1292 if build.submodules:
1293 logging.info("Initialising submodules")
1294 vcs.initsubmodules()
1296 # Check that a subdir (if we're using one) exists. This has to happen
1297 # after the checkout, since it might not exist elsewhere
1298 if not os.path.exists(root_dir):
1299 raise BuildException('Missing subdir ' + root_dir)
1301 # Run an init command if one is required
1303 cmd = replace_config_vars(build.init, build)
1304 logging.info("Running 'init' commands in %s" % root_dir)
1306 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1307 if p.returncode != 0:
1308 raise BuildException("Error running init command for %s:%s" %
1309 (app.id, build.version), p.output)
1311 # Apply patches if any
1313 logging.info("Applying patches")
1314 for patch in build.patch:
1315 patch = patch.strip()
1316 logging.info("Applying " + patch)
1317 patch_path = os.path.join('metadata', app.id, patch)
1318 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1319 if p.returncode != 0:
1320 raise BuildException("Failed to apply patch %s" % patch_path)
1322 # Get required source libraries
1325 logging.info("Collecting source libraries")
1326 for lib in build.srclibs:
1327 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
1329 for name, number, libpath in srclibpaths:
1330 place_srclib(root_dir, int(number) if number else None, libpath)
1332 basesrclib = vcs.getsrclib()
1333 # If one was used for the main source, add that too.
1335 srclibpaths.append(basesrclib)
1337 # Update the local.properties file
1338 localprops = [os.path.join(build_dir, 'local.properties')]
1340 parts = build.subdir.split(os.sep)
1343 cur = os.path.join(cur, d)
1344 localprops += [os.path.join(cur, 'local.properties')]
1345 for path in localprops:
1347 if os.path.isfile(path):
1348 logging.info("Updating local.properties file at %s" % path)
1349 with open(path, 'r') as f:
1353 logging.info("Creating local.properties file at %s" % path)
1354 # Fix old-fashioned 'sdk-location' by copying
1355 # from sdk.dir, if necessary
1357 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1358 re.S | re.M).group(1)
1359 props += "sdk-location=%s\n" % sdkloc
1361 props += "sdk.dir=%s\n" % config['sdk_path']
1362 props += "sdk-location=%s\n" % config['sdk_path']
1363 ndk_path = build.ndk_path()
1366 props += "ndk.dir=%s\n" % ndk_path
1367 props += "ndk-location=%s\n" % ndk_path
1368 # Add java.encoding if necessary
1370 props += "java.encoding=%s\n" % build.encoding
1371 with open(path, 'w') as f:
1375 if build.build_method() == 'gradle':
1376 flavours = build.gradle
1379 n = build.target.split('-')[1]
1380 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1381 r'compileSdkVersion %s' % n,
1382 os.path.join(root_dir, 'build.gradle'))
1384 # Remove forced debuggable flags
1385 remove_debuggable_flags(root_dir)
1387 # Insert version code and number into the manifest if necessary
1388 if build.forceversion:
1389 logging.info("Changing the version name")
1390 for path in manifest_paths(root_dir, flavours):
1391 if not os.path.isfile(path):
1393 if has_extension(path, 'xml'):
1394 regsub_file(r'android:versionName="[^"]*"',
1395 r'android:versionName="%s"' % build.version,
1397 elif has_extension(path, 'gradle'):
1398 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1399 r"""\1versionName '%s'""" % build.version,
1402 if build.forcevercode:
1403 logging.info("Changing the version code")
1404 for path in manifest_paths(root_dir, flavours):
1405 if not os.path.isfile(path):
1407 if has_extension(path, 'xml'):
1408 regsub_file(r'android:versionCode="[^"]*"',
1409 r'android:versionCode="%s"' % build.vercode,
1411 elif has_extension(path, 'gradle'):
1412 regsub_file(r'versionCode[ =]+[0-9]+',
1413 r'versionCode %s' % build.vercode,
1416 # Delete unwanted files
1418 logging.info("Removing specified files")
1419 for part in getpaths(build_dir, build.rm):
1420 dest = os.path.join(build_dir, part)
1421 logging.info("Removing {0}".format(part))
1422 if os.path.lexists(dest):
1423 if os.path.islink(dest):
1424 FDroidPopen(['unlink', dest], output=False)
1426 FDroidPopen(['rm', '-rf', dest], output=False)
1428 logging.info("...but it didn't exist")
1430 remove_signing_keys(build_dir)
1432 # Add required external libraries
1434 logging.info("Collecting prebuilt libraries")
1435 libsdir = os.path.join(root_dir, 'libs')
1436 if not os.path.exists(libsdir):
1438 for lib in build.extlibs:
1440 logging.info("...installing extlib {0}".format(lib))
1441 libf = os.path.basename(lib)
1442 libsrc = os.path.join(extlib_dir, lib)
1443 if not os.path.exists(libsrc):
1444 raise BuildException("Missing extlib file {0}".format(libsrc))
1445 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1447 # Run a pre-build command if one is required
1449 logging.info("Running 'prebuild' commands in %s" % root_dir)
1451 cmd = replace_config_vars(build.prebuild, build)
1453 # Substitute source library paths into prebuild commands
1454 for name, number, libpath in srclibpaths:
1455 libpath = os.path.relpath(libpath, root_dir)
1456 cmd = cmd.replace('$$' + name + '$$', libpath)
1458 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1459 if p.returncode != 0:
1460 raise BuildException("Error running prebuild command for %s:%s" %
1461 (app.id, build.version), p.output)
1463 # Generate (or update) the ant build file, build.xml...
1464 if build.build_method() == 'ant' and build.update != ['no']:
1465 parms = ['android', 'update', 'lib-project']
1466 lparms = ['android', 'update', 'project']
1469 parms += ['-t', build.target]
1470 lparms += ['-t', build.target]
1472 update_dirs = build.update
1474 update_dirs = ant_subprojects(root_dir) + ['.']
1476 for d in update_dirs:
1477 subdir = os.path.join(root_dir, d)
1479 logging.debug("Updating main project")
1480 cmd = parms + ['-p', d]
1482 logging.debug("Updating subproject %s" % d)
1483 cmd = lparms + ['-p', d]
1484 p = SdkToolsPopen(cmd, cwd=root_dir)
1485 # Check to see whether an error was returned without a proper exit
1486 # code (this is the case for the 'no target set or target invalid'
1488 if p.returncode != 0 or p.output.startswith("Error: "):
1489 raise BuildException("Failed to update project at %s" % d, p.output)
1490 # Clean update dirs via ant
1492 logging.info("Cleaning subproject %s" % d)
1493 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1495 return (root_dir, srclibpaths)
1498 # Extend via globbing the paths from a field and return them as a map from
1499 # original path to resulting paths
1500 def getpaths_map(build_dir, globpaths):
1504 full_path = os.path.join(build_dir, p)
1505 full_path = os.path.normpath(full_path)
1506 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1508 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1512 # Extend via globbing the paths from a field and return them as a set
1513 def getpaths(build_dir, globpaths):
1514 paths_map = getpaths_map(build_dir, globpaths)
1516 for k, v in paths_map.iteritems():
1523 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1529 self.path = os.path.join('stats', 'known_apks.txt')
1531 if os.path.isfile(self.path):
1532 for line in file(self.path):
1533 t = line.rstrip().split(' ')
1535 self.apks[t[0]] = (t[1], None)
1537 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1538 self.changed = False
1540 def writeifchanged(self):
1541 if not self.changed:
1544 if not os.path.exists('stats'):
1548 for apk, app in self.apks.iteritems():
1550 line = apk + ' ' + appid
1552 line += ' ' + time.strftime('%Y-%m-%d', added)
1555 with open(self.path, 'w') as f:
1556 for line in sorted(lst, key=natural_key):
1557 f.write(line + '\n')
1559 # Record an apk (if it's new, otherwise does nothing)
1560 # Returns the date it was added.
1561 def recordapk(self, apk, app):
1562 if apk not in self.apks:
1563 self.apks[apk] = (app, time.gmtime(time.time()))
1565 _, added = self.apks[apk]
1568 # Look up information - given the 'apkname', returns (app id, date added/None).
1569 # Or returns None for an unknown apk.
1570 def getapp(self, apkname):
1571 if apkname in self.apks:
1572 return self.apks[apkname]
1575 # Get the most recent 'num' apps added to the repo, as a list of package ids
1576 # with the most recent first.
1577 def getlatest(self, num):
1579 for apk, app in self.apks.iteritems():
1583 if apps[appid] > added:
1587 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1588 lst = [app for app, _ in sortedapps]
1593 def isApkDebuggable(apkfile, config):
1594 """Returns True if the given apk file is debuggable
1596 :param apkfile: full path to the apk to check"""
1598 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1600 if p.returncode != 0:
1601 logging.critical("Failed to get apk manifest information")
1603 for line in p.output.splitlines():
1604 if 'android:debuggable' in line and not line.endswith('0x0'):
1614 def SdkToolsPopen(commands, cwd=None, output=True):
1616 if cmd not in config:
1617 config[cmd] = find_sdk_tools_cmd(commands[0])
1618 abscmd = config[cmd]
1620 logging.critical("Could not find '%s' on your system" % cmd)
1622 return FDroidPopen([abscmd] + commands[1:],
1623 cwd=cwd, output=output)
1626 def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
1628 Run a command and capture the possibly huge output.
1630 :param commands: command and argument list like in subprocess.Popen
1631 :param cwd: optionally specifies a working directory
1632 :returns: A PopenResult.
1638 cwd = os.path.normpath(cwd)
1639 logging.debug("Directory: %s" % cwd)
1640 logging.debug("> %s" % ' '.join(commands))
1642 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
1643 result = PopenResult()
1646 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
1647 stdout=subprocess.PIPE, stderr=stderr_param)
1648 except OSError as e:
1649 raise BuildException("OSError while trying to execute " +
1650 ' '.join(commands) + ': ' + str(e))
1652 if not stderr_to_stdout and options.verbose:
1653 stderr_queue = Queue()
1654 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
1656 while not stderr_reader.eof():
1657 while not stderr_queue.empty():
1658 line = stderr_queue.get()
1659 sys.stderr.write(line)
1664 stdout_queue = Queue()
1665 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1667 # Check the queue for output (until there is no more to get)
1668 while not stdout_reader.eof():
1669 while not stdout_queue.empty():
1670 line = stdout_queue.get()
1671 if output and options.verbose:
1672 # Output directly to console
1673 sys.stderr.write(line)
1675 result.output += line
1679 result.returncode = p.wait()
1683 gradle_comment = re.compile(r'[ ]*//')
1684 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1685 gradle_line_matches = [
1686 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1687 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1688 re.compile(r'.*\.readLine\(.*'),
1692 def remove_signing_keys(build_dir):
1693 for root, dirs, files in os.walk(build_dir):
1694 if 'build.gradle' in files:
1695 path = os.path.join(root, 'build.gradle')
1697 with open(path, "r") as o:
1698 lines = o.readlines()
1704 with open(path, "w") as o:
1705 while i < len(lines):
1708 while line.endswith('\\\n'):
1709 line = line.rstrip('\\\n') + lines[i]
1712 if gradle_comment.match(line):
1717 opened += line.count('{')
1718 opened -= line.count('}')
1721 if gradle_signing_configs.match(line):
1726 if any(s.match(line) for s in gradle_line_matches):
1734 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1737 'project.properties',
1739 'default.properties',
1740 'ant.properties', ]:
1741 if propfile in files:
1742 path = os.path.join(root, propfile)
1744 with open(path, "r") as o:
1745 lines = o.readlines()
1749 with open(path, "w") as o:
1751 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1758 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1761 def reset_env_path():
1762 global env, orig_path
1763 env['PATH'] = orig_path
1766 def add_to_env_path(path):
1768 paths = env['PATH'].split(os.pathsep)
1772 env['PATH'] = os.pathsep.join(paths)
1775 def replace_config_vars(cmd, build):
1777 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1778 # env['ANDROID_NDK'] is set in build_local right before prepare_source
1779 cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
1780 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1781 if build is not None:
1782 cmd = cmd.replace('$$COMMIT$$', build.commit)
1783 cmd = cmd.replace('$$VERSION$$', build.version)
1784 cmd = cmd.replace('$$VERCODE$$', build.vercode)
1788 def place_srclib(root_dir, number, libpath):
1791 relpath = os.path.relpath(libpath, root_dir)
1792 proppath = os.path.join(root_dir, 'project.properties')
1795 if os.path.isfile(proppath):
1796 with open(proppath, "r") as o:
1797 lines = o.readlines()
1799 with open(proppath, "w") as o:
1802 if line.startswith('android.library.reference.%d=' % number):
1803 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1808 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1810 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
1813 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
1814 """Verify that two apks are the same
1816 One of the inputs is signed, the other is unsigned. The signature metadata
1817 is transferred from the signed to the unsigned apk, and then jarsigner is
1818 used to verify that the signature from the signed apk is also varlid for
1820 :param signed_apk: Path to a signed apk file
1821 :param unsigned_apk: Path to an unsigned apk file expected to match it
1822 :param tmp_dir: Path to directory for temporary files
1823 :returns: None if the verification is successful, otherwise a string
1824 describing what went wrong.
1826 with ZipFile(signed_apk) as signed_apk_as_zip:
1827 meta_inf_files = ['META-INF/MANIFEST.MF']
1828 for f in signed_apk_as_zip.namelist():
1829 if apk_sigfile.match(f):
1830 meta_inf_files.append(f)
1831 if len(meta_inf_files) < 3:
1832 return "Signature files missing from {0}".format(signed_apk)
1833 signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
1834 with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
1835 for meta_inf_file in meta_inf_files:
1836 unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
1838 if subprocess.call([config['jarsigner'], '-verify', unsigned_apk]) != 0:
1839 logging.info("...NOT verified - {0}".format(signed_apk))
1840 return compare_apks(signed_apk, unsigned_apk, tmp_dir)
1841 logging.info("...successfully verified")
1844 apk_badchars = re.compile('''[/ :;'"]''')
1847 def compare_apks(apk1, apk2, tmp_dir):
1850 Returns None if the apk content is the same (apart from the signing key),
1851 otherwise a string describing what's different, or what went wrong when
1852 trying to do the comparison.
1855 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
1856 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
1857 for d in [apk1dir, apk2dir]:
1858 if os.path.exists(d):
1861 os.mkdir(os.path.join(d, 'jar-xf'))
1863 if subprocess.call(['jar', 'xf',
1864 os.path.abspath(apk1)],
1865 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
1866 return("Failed to unpack " + apk1)
1867 if subprocess.call(['jar', 'xf',
1868 os.path.abspath(apk2)],
1869 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
1870 return("Failed to unpack " + apk2)
1872 # try to find apktool in the path, if it hasn't been manually configed
1873 if 'apktool' not in config:
1874 tmp = find_command('apktool')
1876 config['apktool'] = tmp
1877 if 'apktool' in config:
1878 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
1880 return("Failed to unpack " + apk1)
1881 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
1883 return("Failed to unpack " + apk2)
1885 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
1886 lines = p.output.splitlines()
1887 if len(lines) != 1 or 'META-INF' not in lines[0]:
1888 meld = find_command('meld')
1889 if meld is not None:
1890 p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
1891 return("Unexpected diff output - " + p.output)
1893 # since everything verifies, delete the comparison to keep cruft down
1894 shutil.rmtree(apk1dir)
1895 shutil.rmtree(apk2dir)
1897 # If we get here, it seems like they're the same!
1901 def find_command(command):
1902 '''find the full path of a command, or None if it can't be found in the PATH'''
1905 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
1907 fpath, fname = os.path.split(command)
1912 for path in os.environ["PATH"].split(os.pathsep):
1913 path = path.strip('"')
1914 exe_file = os.path.join(path, command)
1915 if is_exe(exe_file):
1922 '''generate a random password for when generating keys'''
1923 h = hashlib.sha256()
1924 h.update(os.urandom(16)) # salt
1925 h.update(bytes(socket.getfqdn()))
1926 return h.digest().encode('base64').strip()
1929 def genkeystore(localconfig):
1930 '''Generate a new key with random passwords and add it to new keystore'''
1931 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
1932 keystoredir = os.path.dirname(localconfig['keystore'])
1933 if keystoredir is None or keystoredir == '':
1934 keystoredir = os.path.join(os.getcwd(), keystoredir)
1935 if not os.path.exists(keystoredir):
1936 os.makedirs(keystoredir, mode=0o700)
1938 write_password_file("keystorepass", localconfig['keystorepass'])
1939 write_password_file("keypass", localconfig['keypass'])
1940 p = FDroidPopen([config['keytool'], '-genkey',
1941 '-keystore', localconfig['keystore'],
1942 '-alias', localconfig['repo_keyalias'],
1943 '-keyalg', 'RSA', '-keysize', '4096',
1944 '-sigalg', 'SHA256withRSA',
1945 '-validity', '10000',
1946 '-storepass:file', config['keystorepassfile'],
1947 '-keypass:file', config['keypassfile'],
1948 '-dname', localconfig['keydname']])
1949 # TODO keypass should be sent via stdin
1950 if p.returncode != 0:
1951 raise BuildException("Failed to generate key", p.output)
1952 os.chmod(localconfig['keystore'], 0o0600)
1953 # now show the lovely key that was just generated
1954 p = FDroidPopen([config['keytool'], '-list', '-v',
1955 '-keystore', localconfig['keystore'],
1956 '-alias', localconfig['repo_keyalias'],
1957 '-storepass:file', config['keystorepassfile']])
1958 logging.info(p.output.strip() + '\n\n')
1961 def write_to_config(thisconfig, key, value=None):
1962 '''write a key/value to the local config.py'''
1964 origkey = key + '_orig'
1965 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
1966 with open('config.py', 'r') as f:
1968 pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
1969 repl = '\n' + key + ' = "' + value + '"'
1970 data = re.sub(pattern, repl, data)
1971 # if this key is not in the file, append it
1972 if not re.match('\s*' + key + '\s*=\s*"', data):
1974 # make sure the file ends with a carraige return
1975 if not re.match('\n$', data):
1977 with open('config.py', 'w') as f:
1981 def parse_xml(path):
1982 return XMLElementTree.parse(path).getroot()
1985 def string_is_integer(string):
1993 def get_per_app_repos():
1994 '''per-app repos are dirs named with the packageName of a single app'''
1996 # Android packageNames are Java packages, they may contain uppercase or
1997 # lowercase letters ('A' through 'Z'), numbers, and underscores
1998 # ('_'). However, individual package name parts may only start with
1999 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2000 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2003 for root, dirs, files in os.walk(os.getcwd()):
2005 print('checking', root, 'for', d)
2006 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2007 # standard parts of an fdroid repo, so never packageNames
2010 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):