1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
33 from distutils.version import LooseVersion
34 from zipfile import ZipFile
45 'sdk_path': "$ANDROID_HOME",
48 'r10d': "$ANDROID_NDK"
50 'build_tools': "22.0.0",
54 'sync_from_local_copy_dir': False,
55 'make_current_version_link': True,
56 'current_version_name_source': 'Name',
57 'update_stats': False,
61 'stats_to_carbon': False,
63 'build_server_always': False,
64 'keystore': os.path.join("$HOME", '.local', 'share', 'fdroidserver', 'keystore.jks'),
65 'smartcardoptions': [],
71 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
72 'repo_name': "My First FDroid Repo Demo",
73 'repo_icon': "fdroid-icon.png",
74 'repo_description': '''
75 This is a repository of apps to be used with FDroid. Applications in this
76 repository are either official binaries built by the original application
77 developers, or are binaries built from source by the admin of f-droid.org
78 using the tools on https://gitlab.com/u/fdroid.
84 def fill_config_defaults(thisconfig):
85 for k, v in default_config.items():
86 if k not in thisconfig:
89 # Expand paths (~users and $vars)
90 def expand_path(path):
94 path = os.path.expanduser(path)
95 path = os.path.expandvars(path)
100 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
105 thisconfig[k + '_orig'] = v
107 for k in ['ndk_paths']:
113 thisconfig[k][k2] = exp
114 thisconfig[k][k2 + '_orig'] = v
117 def read_config(opts, config_file='config.py'):
118 """Read the repository config
120 The config is read from config_file, which is in the current directory when
121 any of the repo management commands are used.
123 global config, options, env, orig_path
125 if config is not None:
127 if not os.path.isfile(config_file):
128 logging.critical("Missing config file - is this a repo directory?")
135 logging.debug("Reading %s" % config_file)
136 execfile(config_file, config)
138 # smartcardoptions must be a list since its command line args for Popen
139 if 'smartcardoptions' in config:
140 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
141 elif 'keystore' in config and config['keystore'] == 'NONE':
142 # keystore='NONE' means use smartcard, these are required defaults
143 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
144 'SunPKCS11-OpenSC', '-providerClass',
145 'sun.security.pkcs11.SunPKCS11',
146 '-providerArg', 'opensc-fdroid.cfg']
148 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
149 st = os.stat(config_file)
150 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
151 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
153 fill_config_defaults(config)
155 # There is no standard, so just set up the most common environment
158 orig_path = env['PATH']
159 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
160 env[n] = config['sdk_path']
162 for k in ["keystorepass", "keypass"]:
164 write_password_file(k)
166 for k in ["repo_description", "archive_description"]:
168 config[k] = clean_description(config[k])
170 if 'serverwebroot' in config:
171 if isinstance(config['serverwebroot'], basestring):
172 roots = [config['serverwebroot']]
173 elif all(isinstance(item, basestring) for item in config['serverwebroot']):
174 roots = config['serverwebroot']
176 raise TypeError('only accepts strings, lists, and tuples')
178 for rootstr in roots:
179 # since this is used with rsync, where trailing slashes have
180 # meaning, ensure there is always a trailing slash
181 if rootstr[-1] != '/':
183 rootlist.append(rootstr.replace('//', '/'))
184 config['serverwebroot'] = rootlist
189 def get_ndk_path(version):
191 version = 'r10d' # latest
192 paths = config['ndk_paths']
193 if version not in paths:
195 return paths[version] or ''
198 def find_sdk_tools_cmd(cmd):
199 '''find a working path to a tool from the Android SDK'''
202 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
203 # try to find a working path to this command, in all the recent possible paths
204 if 'build_tools' in config:
205 build_tools = os.path.join(config['sdk_path'], 'build-tools')
206 # if 'build_tools' was manually set and exists, check only that one
207 configed_build_tools = os.path.join(build_tools, config['build_tools'])
208 if os.path.exists(configed_build_tools):
209 tooldirs.append(configed_build_tools)
211 # no configed version, so hunt known paths for it
212 for f in sorted(os.listdir(build_tools), reverse=True):
213 if os.path.isdir(os.path.join(build_tools, f)):
214 tooldirs.append(os.path.join(build_tools, f))
215 tooldirs.append(build_tools)
216 sdk_tools = os.path.join(config['sdk_path'], 'tools')
217 if os.path.exists(sdk_tools):
218 tooldirs.append(sdk_tools)
219 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
220 if os.path.exists(sdk_platform_tools):
221 tooldirs.append(sdk_platform_tools)
222 tooldirs.append('/usr/bin')
224 if os.path.isfile(os.path.join(d, cmd)):
225 return os.path.join(d, cmd)
226 # did not find the command, exit with error message
227 ensure_build_tools_exists(config)
230 def test_sdk_exists(thisconfig):
231 if 'sdk_path' not in thisconfig:
232 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
235 logging.error("'sdk_path' not set in config.py!")
237 if thisconfig['sdk_path'] == default_config['sdk_path']:
238 logging.error('No Android SDK found!')
239 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
240 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
242 if not os.path.exists(thisconfig['sdk_path']):
243 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
245 if not os.path.isdir(thisconfig['sdk_path']):
246 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
248 for d in ['build-tools', 'platform-tools', 'tools']:
249 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
250 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
251 thisconfig['sdk_path'], d))
256 def ensure_build_tools_exists(thisconfig):
257 if not test_sdk_exists(thisconfig):
259 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
260 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
261 if not os.path.isdir(versioned_build_tools):
262 logging.critical('Android Build Tools path "'
263 + versioned_build_tools + '" does not exist!')
267 def write_password_file(pwtype, password=None):
269 writes out passwords to a protected file instead of passing passwords as
270 command line argments
272 filename = '.fdroid.' + pwtype + '.txt'
273 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
275 os.write(fd, config[pwtype])
277 os.write(fd, password)
279 config[pwtype + 'file'] = filename
282 # Given the arguments in the form of multiple appid:[vc] strings, this returns
283 # a dictionary with the set of vercodes specified for each package.
284 def read_pkg_args(args, allow_vercodes=False):
291 if allow_vercodes and ':' in p:
292 package, vercode = p.split(':')
294 package, vercode = p, None
295 if package not in vercodes:
296 vercodes[package] = [vercode] if vercode else []
298 elif vercode and vercode not in vercodes[package]:
299 vercodes[package] += [vercode] if vercode else []
304 # On top of what read_pkg_args does, this returns the whole app metadata, but
305 # limiting the builds list to the builds matching the vercodes specified.
306 def read_app_args(args, allapps, allow_vercodes=False):
308 vercodes = read_pkg_args(args, allow_vercodes)
314 for appid, app in allapps.iteritems():
315 if appid in vercodes:
318 if len(apps) != len(vercodes):
321 logging.critical("No such package: %s" % p)
322 raise FDroidException("Found invalid app ids in arguments")
324 raise FDroidException("No packages specified")
327 for appid, app in apps.iteritems():
331 app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
332 if len(app['builds']) != len(vercodes[appid]):
334 allvcs = [b['vercode'] for b in app['builds']]
335 for v in vercodes[appid]:
337 logging.critical("No such vercode %s for app %s" % (v, appid))
340 raise FDroidException("Found invalid vercodes for some apps")
345 def has_extension(filename, extension):
346 name, ext = os.path.splitext(filename)
347 ext = ext.lower()[1:]
348 return ext == extension
353 def clean_description(description):
354 'Remove unneeded newlines and spaces from a block of description text'
356 # this is split up by paragraph to make removing the newlines easier
357 for paragraph in re.split(r'\n\n', description):
358 paragraph = re.sub('\r', '', paragraph)
359 paragraph = re.sub('\n', ' ', paragraph)
360 paragraph = re.sub(' {2,}', ' ', paragraph)
361 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
362 returnstring += paragraph + '\n\n'
363 return returnstring.rstrip('\n')
366 def apknameinfo(filename):
368 filename = os.path.basename(filename)
369 if apk_regex is None:
370 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
371 m = apk_regex.match(filename)
373 result = (m.group(1), m.group(2))
374 except AttributeError:
375 raise FDroidException("Invalid apk name: %s" % filename)
379 def getapkname(app, build):
380 return "%s_%s.apk" % (app['id'], build['vercode'])
383 def getsrcname(app, build):
384 return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
391 return app['Auto Name']
396 return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
399 def getvcs(vcstype, remote, local):
401 return vcs_git(remote, local)
402 if vcstype == 'git-svn':
403 return vcs_gitsvn(remote, local)
405 return vcs_hg(remote, local)
407 return vcs_bzr(remote, local)
408 if vcstype == 'srclib':
409 if local != os.path.join('build', 'srclib', remote):
410 raise VCSException("Error: srclib paths are hard-coded!")
411 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
413 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
414 raise VCSException("Invalid vcs type " + vcstype)
417 def getsrclibvcs(name):
418 if name not in metadata.srclibs:
419 raise VCSException("Missing srclib " + name)
420 return metadata.srclibs[name]['Repo Type']
425 def __init__(self, remote, local):
427 # svn, git-svn and bzr may require auth
429 if self.repotype() in ('git-svn', 'bzr'):
431 if self.repotype == 'git-svn':
432 raise VCSException("Authentication is not supported for git-svn")
433 self.username, remote = remote.split('@')
434 if ':' not in self.username:
435 raise VCSException("Password required with username")
436 self.username, self.password = self.username.split(':')
440 self.clone_failed = False
441 self.refreshed = False
447 # Take the local repository to a clean version of the given revision, which
448 # is specificed in the VCS's native format. Beforehand, the repository can
449 # be dirty, or even non-existent. If the repository does already exist
450 # locally, it will be updated from the origin, but only once in the
451 # lifetime of the vcs object.
452 # None is acceptable for 'rev' if you know you are cloning a clean copy of
453 # the repo - otherwise it must specify a valid revision.
454 def gotorevision(self, rev):
456 if self.clone_failed:
457 raise VCSException("Downloading the repository already failed once, not trying again.")
459 # The .fdroidvcs-id file for a repo tells us what VCS type
460 # and remote that directory was created from, allowing us to drop it
461 # automatically if either of those things changes.
462 fdpath = os.path.join(self.local, '..',
463 '.fdroidvcs-' + os.path.basename(self.local))
464 cdata = self.repotype() + ' ' + self.remote
467 if os.path.exists(self.local):
468 if os.path.exists(fdpath):
469 with open(fdpath, 'r') as f:
470 fsdata = f.read().strip()
475 logging.info("Repository details for %s changed - deleting" % (
479 logging.info("Repository details for %s missing - deleting" % (
482 shutil.rmtree(self.local)
487 self.gotorevisionx(rev)
488 except FDroidException, e:
491 # If necessary, write the .fdroidvcs file.
492 if writeback and not self.clone_failed:
493 with open(fdpath, 'w') as f:
499 # Derived classes need to implement this. It's called once basic checking
500 # has been performend.
501 def gotorevisionx(self, rev):
502 raise VCSException("This VCS type doesn't define gotorevisionx")
504 # Initialise and update submodules
505 def initsubmodules(self):
506 raise VCSException('Submodules not supported for this vcs type')
508 # Get a list of all known tags
510 if not self._gettags:
511 raise VCSException('gettags not supported for this vcs type')
513 for tag in self._gettags():
514 if re.match('[-A-Za-z0-9_. ]+$', tag):
518 def latesttags(self, tags, number):
519 """Get the most recent tags in a given list.
521 :param tags: a list of tags
522 :param number: the number to return
523 :returns: A list containing the most recent tags in the provided
524 list, up to the maximum number given.
526 raise VCSException('latesttags not supported for this vcs type')
528 # Get current commit reference (hash, revision, etc)
530 raise VCSException('getref not supported for this vcs type')
532 # Returns the srclib (name, path) used in setting up the current
543 # If the local directory exists, but is somehow not a git repository, git
544 # will traverse up the directory tree until it finds one that is (i.e.
545 # fdroidserver) and then we'll proceed to destroy it! This is called as
548 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
549 result = p.output.rstrip()
550 if not result.endswith(self.local):
551 raise VCSException('Repository mismatch')
553 def gotorevisionx(self, rev):
554 if not os.path.exists(self.local):
556 p = FDroidPopen(['git', 'clone', self.remote, self.local])
557 if p.returncode != 0:
558 self.clone_failed = True
559 raise VCSException("Git clone failed", p.output)
563 # Discard any working tree changes
564 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
565 if p.returncode != 0:
566 raise VCSException("Git reset failed", p.output)
567 # Remove untracked files now, in case they're tracked in the target
568 # revision (it happens!)
569 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
570 if p.returncode != 0:
571 raise VCSException("Git clean failed", p.output)
572 if not self.refreshed:
573 # Get latest commits and tags from remote
574 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
575 if p.returncode != 0:
576 raise VCSException("Git fetch failed", p.output)
577 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
578 if p.returncode != 0:
579 raise VCSException("Git fetch failed", p.output)
580 # Recreate origin/HEAD as git clone would do it, in case it disappeared
581 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
582 if p.returncode != 0:
583 lines = p.output.splitlines()
584 if 'Multiple remote HEAD branches' not in lines[0]:
585 raise VCSException("Git remote set-head failed", p.output)
586 branch = lines[1].split(' ')[-1]
587 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
588 if p2.returncode != 0:
589 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
590 self.refreshed = True
591 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
592 # a github repo. Most of the time this is the same as origin/master.
593 rev = rev or 'origin/HEAD'
594 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
595 if p.returncode != 0:
596 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
597 # Get rid of any uncontrolled files left behind
598 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
599 if p.returncode != 0:
600 raise VCSException("Git clean failed", p.output)
602 def initsubmodules(self):
604 submfile = os.path.join(self.local, '.gitmodules')
605 if not os.path.isfile(submfile):
606 raise VCSException("No git submodules available")
608 # fix submodules not accessible without an account and public key auth
609 with open(submfile, 'r') as f:
610 lines = f.readlines()
611 with open(submfile, 'w') as f:
613 if 'git@github.com' in line:
614 line = line.replace('git@github.com:', 'https://github.com/')
618 ['git', 'reset', '--hard'],
619 ['git', 'clean', '-dffx'],
621 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive'] + cmd, cwd=self.local, output=False)
622 if p.returncode != 0:
623 raise VCSException("Git submodule reset failed", p.output)
624 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
625 if p.returncode != 0:
626 raise VCSException("Git submodule sync failed", p.output)
627 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
628 if p.returncode != 0:
629 raise VCSException("Git submodule update failed", p.output)
633 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
634 return p.output.splitlines()
636 def latesttags(self, tags, number):
641 ['git', 'show', '--format=format:%ct', '-s', tag],
642 cwd=self.local, output=False)
643 # Timestamp is on the last line. For a normal tag, it's the only
644 # line, but for annotated tags, the rest of the info precedes it.
645 ts = int(p.output.splitlines()[-1])
648 for _, t in sorted(tl)[-number:]:
653 class vcs_gitsvn(vcs):
658 # If the local directory exists, but is somehow not a git repository, git
659 # will traverse up the directory tree until it finds one that is (i.e.
660 # fdroidserver) and then we'll proceed to destory it! This is called as
663 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
664 result = p.output.rstrip()
665 if not result.endswith(self.local):
666 raise VCSException('Repository mismatch')
668 def gotorevisionx(self, rev):
669 if not os.path.exists(self.local):
671 gitsvn_args = ['git', 'svn', 'clone']
672 if ';' in self.remote:
673 remote_split = self.remote.split(';')
674 for i in remote_split[1:]:
675 if i.startswith('trunk='):
676 gitsvn_args.extend(['-T', i[6:]])
677 elif i.startswith('tags='):
678 gitsvn_args.extend(['-t', i[5:]])
679 elif i.startswith('branches='):
680 gitsvn_args.extend(['-b', i[9:]])
681 gitsvn_args.extend([remote_split[0], self.local])
682 p = FDroidPopen(gitsvn_args, output=False)
683 if p.returncode != 0:
684 self.clone_failed = True
685 raise VCSException("Git svn clone failed", p.output)
687 gitsvn_args.extend([self.remote, self.local])
688 p = FDroidPopen(gitsvn_args, output=False)
689 if p.returncode != 0:
690 self.clone_failed = True
691 raise VCSException("Git svn clone failed", p.output)
695 # Discard any working tree changes
696 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
697 if p.returncode != 0:
698 raise VCSException("Git reset failed", p.output)
699 # Remove untracked files now, in case they're tracked in the target
700 # revision (it happens!)
701 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
702 if p.returncode != 0:
703 raise VCSException("Git clean failed", p.output)
704 if not self.refreshed:
705 # Get new commits, branches and tags from repo
706 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
707 if p.returncode != 0:
708 raise VCSException("Git svn fetch failed")
709 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
710 if p.returncode != 0:
711 raise VCSException("Git svn rebase failed", p.output)
712 self.refreshed = True
714 rev = rev or 'master'
716 nospaces_rev = rev.replace(' ', '%20')
717 # Try finding a svn tag
718 for treeish in ['origin/', '']:
719 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
720 if p.returncode == 0:
722 if p.returncode != 0:
723 # No tag found, normal svn rev translation
724 # Translate svn rev into git format
725 rev_split = rev.split('/')
728 for treeish in ['origin/', '']:
729 if len(rev_split) > 1:
730 treeish += rev_split[0]
731 svn_rev = rev_split[1]
734 # if no branch is specified, then assume trunk (i.e. 'master' branch):
738 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
740 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
741 git_rev = p.output.rstrip()
743 if p.returncode == 0 and git_rev:
746 if p.returncode != 0 or not git_rev:
747 # Try a plain git checkout as a last resort
748 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
749 if p.returncode != 0:
750 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
752 # Check out the git rev equivalent to the svn rev
753 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
754 if p.returncode != 0:
755 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
757 # Get rid of any uncontrolled files left behind
758 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
759 if p.returncode != 0:
760 raise VCSException("Git clean failed", p.output)
764 for treeish in ['origin/', '']:
765 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
771 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
772 if p.returncode != 0:
774 return p.output.strip()
782 def gotorevisionx(self, rev):
783 if not os.path.exists(self.local):
784 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
785 if p.returncode != 0:
786 self.clone_failed = True
787 raise VCSException("Hg clone failed", p.output)
789 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
790 if p.returncode != 0:
791 raise VCSException("Hg status failed", p.output)
792 for line in p.output.splitlines():
793 if not line.startswith('? '):
794 raise VCSException("Unexpected output from hg status -uS: " + line)
795 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
796 if not self.refreshed:
797 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
798 if p.returncode != 0:
799 raise VCSException("Hg pull failed", p.output)
800 self.refreshed = True
802 rev = rev or 'default'
805 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
806 if p.returncode != 0:
807 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
808 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
809 # Also delete untracked files, we have to enable purge extension for that:
810 if "'purge' is provided by the following extension" in p.output:
811 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
812 myfile.write("\n[extensions]\nhgext.purge=\n")
813 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
814 if p.returncode != 0:
815 raise VCSException("HG purge failed", p.output)
816 elif p.returncode != 0:
817 raise VCSException("HG purge failed", p.output)
820 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
821 return p.output.splitlines()[1:]
829 def gotorevisionx(self, rev):
830 if not os.path.exists(self.local):
831 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
832 if p.returncode != 0:
833 self.clone_failed = True
834 raise VCSException("Bzr branch failed", p.output)
836 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
837 if p.returncode != 0:
838 raise VCSException("Bzr revert failed", p.output)
839 if not self.refreshed:
840 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
841 if p.returncode != 0:
842 raise VCSException("Bzr update failed", p.output)
843 self.refreshed = True
845 revargs = list(['-r', rev] if rev else [])
846 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
847 if p.returncode != 0:
848 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
851 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
852 return [tag.split(' ')[0].strip() for tag in
853 p.output.splitlines()]
856 def retrieve_string(app_dir, string, xmlfiles=None):
859 os.path.join(app_dir, 'res'),
860 os.path.join(app_dir, 'src', 'main'),
865 for res_dir in res_dirs:
866 for r, d, f in os.walk(res_dir):
867 if os.path.basename(r) == 'values':
868 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
871 if string.startswith('@string/'):
872 string_search = re.compile(r'.*name="' + string[8:] + '".*?>"?([^<]+?)"?<.*').search
873 elif string.startswith('&') and string.endswith(';'):
874 string_search = re.compile(r'.*<!ENTITY.*' + string[1:-1] + '.*?"([^"]+?)".*>').search
876 if string_search is not None:
877 for xmlfile in xmlfiles:
878 if not os.path.isfile(xmlfile):
880 for line in file(xmlfile):
881 matches = string_search(line)
883 return retrieve_string(app_dir, matches.group(1), xmlfiles)
886 return string.replace("\\'", "'")
889 # Return list of existing files that will be used to find the highest vercode
890 def manifest_paths(app_dir, flavours):
892 possible_manifests = \
893 [os.path.join(app_dir, 'AndroidManifest.xml'),
894 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
895 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
896 os.path.join(app_dir, 'build.gradle')]
898 for flavour in flavours:
901 possible_manifests.append(
902 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
904 return [path for path in possible_manifests if os.path.isfile(path)]
907 # Retrieve the package name. Returns the name, or None if not found.
908 def fetch_real_name(app_dir, flavours):
909 app_search = re.compile(r'.*<application.*').search
910 name_search = re.compile(r'.*android:label="([^"]+)".*').search
912 for f in manifest_paths(app_dir, flavours):
913 if not has_extension(f, 'xml') or not os.path.isfile(f):
915 logging.debug("fetch_real_name: Checking manifest at " + f)
921 matches = name_search(line)
923 stringname = matches.group(1)
924 logging.debug("fetch_real_name: using string " + stringname)
925 result = retrieve_string(app_dir, stringname)
927 result = result.strip()
932 # Retrieve the version name
933 def version_name(original, app_dir, flavours):
934 for f in manifest_paths(app_dir, flavours):
935 if not has_extension(f, 'xml'):
937 string = retrieve_string(app_dir, original)
943 def get_library_references(root_dir):
945 proppath = os.path.join(root_dir, 'project.properties')
946 if not os.path.isfile(proppath):
948 for line in file(proppath):
949 if not line.startswith('android.library.reference.'):
951 path = line.split('=')[1].strip()
952 relpath = os.path.join(root_dir, path)
953 if not os.path.isdir(relpath):
955 logging.debug("Found subproject at %s" % path)
956 libraries.append(path)
960 def ant_subprojects(root_dir):
961 subprojects = get_library_references(root_dir)
962 for subpath in subprojects:
963 subrelpath = os.path.join(root_dir, subpath)
964 for p in get_library_references(subrelpath):
965 relp = os.path.normpath(os.path.join(subpath, p))
966 if relp not in subprojects:
967 subprojects.insert(0, relp)
971 def remove_debuggable_flags(root_dir):
972 # Remove forced debuggable flags
973 logging.debug("Removing debuggable flags from %s" % root_dir)
974 for root, dirs, files in os.walk(root_dir):
975 if 'AndroidManifest.xml' in files:
976 path = os.path.join(root, 'AndroidManifest.xml')
977 p = FDroidPopen(['sed', '-i', 's/android:debuggable="[^"]*"//g', path], output=False)
978 if p.returncode != 0:
979 raise BuildException("Failed to remove debuggable flags of %s" % path)
982 # Extract some information from the AndroidManifest.xml at the given path.
983 # Returns (version, vercode, package), any or all of which might be None.
984 # All values returned are strings.
985 def parse_androidmanifests(paths, ignoreversions=None):
988 return (None, None, None)
990 vcsearch = re.compile(r'.*:versionCode="([0-9]+?)".*').search
991 vnsearch = re.compile(r'.*:versionName="([^"]+?)".*').search
992 psearch = re.compile(r'.*package="([^"]+)".*').search
994 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
995 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
996 psearch_g = re.compile(r'.*packageName *=* *["\']([^"]+)["\'].*').search
998 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1006 if not os.path.isfile(path):
1009 logging.debug("Parsing manifest at {0}".format(path))
1010 gradle = has_extension(path, 'gradle')
1013 # Remember package name, may be defined separately from version+vercode
1014 package = max_package
1016 for line in file(path):
1019 matches = psearch_g(line)
1021 matches = psearch(line)
1023 package = matches.group(1)
1026 matches = vnsearch_g(line)
1028 matches = vnsearch(line)
1030 version = matches.group(2 if gradle else 1)
1033 matches = vcsearch_g(line)
1035 matches = vcsearch(line)
1037 vercode = matches.group(1)
1039 logging.debug("..got package={0}, version={1}, vercode={2}"
1040 .format(package, version, vercode))
1042 # Always grab the package name and version name in case they are not
1043 # together with the highest version code
1044 if max_package is None and package is not None:
1045 max_package = package
1046 if max_version is None and version is not None:
1047 max_version = version
1049 if max_vercode is None or (vercode is not None and vercode > max_vercode):
1050 if not ignoresearch or not ignoresearch(version):
1051 if version is not None:
1052 max_version = version
1053 if vercode is not None:
1054 max_vercode = vercode
1055 if package is not None:
1056 max_package = package
1058 max_version = "Ignore"
1060 if max_version is None:
1061 max_version = "Unknown"
1063 if max_package and not is_valid_package_name(max_package):
1064 raise FDroidException("Invalid package name {0}".format(max_package))
1066 return (max_version, max_vercode, max_package)
1069 def is_valid_package_name(name):
1070 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1073 class FDroidException(Exception):
1075 def __init__(self, value, detail=None):
1077 self.detail = detail
1079 def get_wikitext(self):
1080 ret = repr(self.value) + "\n"
1084 txt = self.detail[-8192:] if len(self.detail) > 8192 else self.detail
1092 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1096 class VCSException(FDroidException):
1100 class BuildException(FDroidException):
1104 # Get the specified source library.
1105 # Returns the path to it. Normally this is the path to be used when referencing
1106 # it, which may be a subdirectory of the actual project. If you want the base
1107 # directory of the project, pass 'basepath=True'.
1108 def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
1109 basepath=False, raw=False, prepare=True, preponly=False):
1117 name, ref = spec.split('@')
1119 number, name = name.split(':', 1)
1121 name, subdir = name.split('/', 1)
1123 if name not in metadata.srclibs:
1124 raise VCSException('srclib ' + name + ' not found.')
1126 srclib = metadata.srclibs[name]
1128 sdir = os.path.join(srclib_dir, name)
1131 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1132 vcs.srclib = (name, number, sdir)
1134 vcs.gotorevision(ref)
1141 libdir = os.path.join(sdir, subdir)
1142 elif srclib["Subdir"]:
1143 for subdir in srclib["Subdir"]:
1144 libdir_candidate = os.path.join(sdir, subdir)
1145 if os.path.exists(libdir_candidate):
1146 libdir = libdir_candidate
1152 if srclib["Srclibs"]:
1154 for lib in srclib["Srclibs"].replace(';', ',').split(','):
1156 for t in srclibpaths:
1161 raise VCSException('Missing recursive srclib %s for %s' % (
1163 place_srclib(libdir, n, s_tuple[2])
1166 remove_signing_keys(sdir)
1167 remove_debuggable_flags(sdir)
1171 if srclib["Prepare"]:
1172 cmd = replace_config_vars(srclib["Prepare"])
1174 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1175 if p.returncode != 0:
1176 raise BuildException("Error running prepare command for srclib %s"
1182 return (name, number, libdir)
1185 # Prepare the source code for a particular build
1186 # 'vcs' - the appropriate vcs object for the application
1187 # 'app' - the application details from the metadata
1188 # 'build' - the build details from the metadata
1189 # 'build_dir' - the path to the build directory, usually
1191 # 'srclib_dir' - the path to the source libraries directory, usually
1193 # 'extlib_dir' - the path to the external libraries directory, usually
1195 # Returns the (root, srclibpaths) where:
1196 # 'root' is the root directory, which may be the same as 'build_dir' or may
1197 # be a subdirectory of it.
1198 # 'srclibpaths' is information on the srclibs being used
1199 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False):
1201 # Optionally, the actual app source can be in a subdirectory
1203 root_dir = os.path.join(build_dir, build['subdir'])
1205 root_dir = build_dir
1207 # Get a working copy of the right revision
1208 logging.info("Getting source for revision " + build['commit'])
1209 vcs.gotorevision(build['commit'])
1211 # Initialise submodules if requred
1212 if build['submodules']:
1213 logging.info("Initialising submodules")
1214 vcs.initsubmodules()
1216 # Check that a subdir (if we're using one) exists. This has to happen
1217 # after the checkout, since it might not exist elsewhere
1218 if not os.path.exists(root_dir):
1219 raise BuildException('Missing subdir ' + root_dir)
1221 # Run an init command if one is required
1223 cmd = replace_config_vars(build['init'])
1224 logging.info("Running 'init' commands in %s" % root_dir)
1226 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1227 if p.returncode != 0:
1228 raise BuildException("Error running init command for %s:%s" %
1229 (app['id'], build['version']), p.output)
1231 # Apply patches if any
1233 logging.info("Applying patches")
1234 for patch in build['patch']:
1235 patch = patch.strip()
1236 logging.info("Applying " + patch)
1237 patch_path = os.path.join('metadata', app['id'], patch)
1238 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1239 if p.returncode != 0:
1240 raise BuildException("Failed to apply patch %s" % patch_path)
1242 # Get required source libraries
1244 if build['srclibs']:
1245 logging.info("Collecting source libraries")
1246 for lib in build['srclibs']:
1247 srclibpaths.append(getsrclib(lib, srclib_dir, build, srclibpaths,
1250 for name, number, libpath in srclibpaths:
1251 place_srclib(root_dir, int(number) if number else None, libpath)
1253 basesrclib = vcs.getsrclib()
1254 # If one was used for the main source, add that too.
1256 srclibpaths.append(basesrclib)
1258 # Update the local.properties file
1259 localprops = [os.path.join(build_dir, 'local.properties')]
1261 localprops += [os.path.join(root_dir, 'local.properties')]
1262 for path in localprops:
1264 if os.path.isfile(path):
1265 logging.info("Updating local.properties file at %s" % path)
1271 logging.info("Creating local.properties file at %s" % path)
1272 # Fix old-fashioned 'sdk-location' by copying
1273 # from sdk.dir, if necessary
1274 if build['oldsdkloc']:
1275 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1276 re.S | re.M).group(1)
1277 props += "sdk-location=%s\n" % sdkloc
1279 props += "sdk.dir=%s\n" % config['sdk_path']
1280 props += "sdk-location=%s\n" % config['sdk_path']
1281 if build['ndk_path']:
1283 props += "ndk.dir=%s\n" % build['ndk_path']
1284 props += "ndk-location=%s\n" % build['ndk_path']
1285 # Add java.encoding if necessary
1286 if build['encoding']:
1287 props += "java.encoding=%s\n" % build['encoding']
1293 if build['type'] == 'gradle':
1294 flavours = build['gradle']
1296 version_regex = re.compile(r".*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1297 gradlepluginver = None
1299 gradle_dirs = [root_dir]
1301 # Parent dir build.gradle
1302 parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
1303 if parent_dir.startswith(build_dir):
1304 gradle_dirs.append(parent_dir)
1306 for dir_path in gradle_dirs:
1309 if not os.path.isdir(dir_path):
1311 for filename in os.listdir(dir_path):
1312 if not filename.endswith('.gradle'):
1314 path = os.path.join(dir_path, filename)
1315 if not os.path.isfile(path):
1317 for line in file(path):
1318 match = version_regex.match(line)
1320 gradlepluginver = match.group(1)
1324 build['gradlepluginver'] = LooseVersion(gradlepluginver)
1326 logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
1327 build['gradlepluginver'] = LooseVersion('0.11')
1330 n = build["target"].split('-')[1]
1331 FDroidPopen(['sed', '-i',
1332 's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g',
1333 'build.gradle'], cwd=root_dir, output=False)
1335 # Remove forced debuggable flags
1336 remove_debuggable_flags(root_dir)
1338 # Insert version code and number into the manifest if necessary
1339 if build['forceversion']:
1340 logging.info("Changing the version name")
1341 for path in manifest_paths(root_dir, flavours):
1342 if not os.path.isfile(path):
1344 if has_extension(path, 'xml'):
1345 p = FDroidPopen(['sed', '-i',
1346 's/android:versionName="[^"]*"/android:versionName="' + build['version'] + '"/g',
1347 path], output=False)
1348 if p.returncode != 0:
1349 raise BuildException("Failed to amend manifest")
1350 elif has_extension(path, 'gradle'):
1351 p = FDroidPopen(['sed', '-i',
1352 's/versionName *=* *.*/versionName = "' + build['version'] + '"/g',
1353 path], output=False)
1354 if p.returncode != 0:
1355 raise BuildException("Failed to amend build.gradle")
1356 if build['forcevercode']:
1357 logging.info("Changing the version code")
1358 for path in manifest_paths(root_dir, flavours):
1359 if not os.path.isfile(path):
1361 if has_extension(path, 'xml'):
1362 p = FDroidPopen(['sed', '-i',
1363 's/android:versionCode="[^"]*"/android:versionCode="' + build['vercode'] + '"/g',
1364 path], output=False)
1365 if p.returncode != 0:
1366 raise BuildException("Failed to amend manifest")
1367 elif has_extension(path, 'gradle'):
1368 p = FDroidPopen(['sed', '-i',
1369 's/versionCode *=* *[0-9]*/versionCode = ' + build['vercode'] + '/g',
1370 path], output=False)
1371 if p.returncode != 0:
1372 raise BuildException("Failed to amend build.gradle")
1374 # Delete unwanted files
1376 logging.info("Removing specified files")
1377 for part in getpaths(build_dir, build, 'rm'):
1378 dest = os.path.join(build_dir, part)
1379 logging.info("Removing {0}".format(part))
1380 if os.path.lexists(dest):
1381 if os.path.islink(dest):
1382 FDroidPopen(['unlink', dest], output=False)
1384 FDroidPopen(['rm', '-rf', dest], output=False)
1386 logging.info("...but it didn't exist")
1388 remove_signing_keys(build_dir)
1390 # Add required external libraries
1391 if build['extlibs']:
1392 logging.info("Collecting prebuilt libraries")
1393 libsdir = os.path.join(root_dir, 'libs')
1394 if not os.path.exists(libsdir):
1396 for lib in build['extlibs']:
1398 logging.info("...installing extlib {0}".format(lib))
1399 libf = os.path.basename(lib)
1400 libsrc = os.path.join(extlib_dir, lib)
1401 if not os.path.exists(libsrc):
1402 raise BuildException("Missing extlib file {0}".format(libsrc))
1403 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1405 # Run a pre-build command if one is required
1406 if build['prebuild']:
1407 logging.info("Running 'prebuild' commands in %s" % root_dir)
1409 cmd = replace_config_vars(build['prebuild'])
1411 # Substitute source library paths into prebuild commands
1412 for name, number, libpath in srclibpaths:
1413 libpath = os.path.relpath(libpath, root_dir)
1414 cmd = cmd.replace('$$' + name + '$$', libpath)
1416 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1417 if p.returncode != 0:
1418 raise BuildException("Error running prebuild command for %s:%s" %
1419 (app['id'], build['version']), p.output)
1421 # Generate (or update) the ant build file, build.xml...
1422 if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
1423 parms = ['android', 'update', 'lib-project']
1424 lparms = ['android', 'update', 'project']
1427 parms += ['-t', build['target']]
1428 lparms += ['-t', build['target']]
1429 if build['update'] == ['auto']:
1430 update_dirs = ant_subprojects(root_dir) + ['.']
1432 update_dirs = build['update']
1434 for d in update_dirs:
1435 subdir = os.path.join(root_dir, d)
1437 logging.debug("Updating main project")
1438 cmd = parms + ['-p', d]
1440 logging.debug("Updating subproject %s" % d)
1441 cmd = lparms + ['-p', d]
1442 p = SdkToolsPopen(cmd, cwd=root_dir)
1443 # Check to see whether an error was returned without a proper exit
1444 # code (this is the case for the 'no target set or target invalid'
1446 if p.returncode != 0 or p.output.startswith("Error: "):
1447 raise BuildException("Failed to update project at %s" % d, p.output)
1448 # Clean update dirs via ant
1450 logging.info("Cleaning subproject %s" % d)
1451 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1453 return (root_dir, srclibpaths)
1456 # Split and extend via globbing the paths from a field
1457 def getpaths(build_dir, build, field):
1459 for p in build[field]:
1461 full_path = os.path.join(build_dir, p)
1462 full_path = os.path.normpath(full_path)
1463 paths += [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1467 # Scan the source code in the given directory (and all subdirectories)
1468 # and return the number of fatal problems encountered
1469 def scan_source(build_dir, root_dir, thisbuild):
1473 # Common known non-free blobs (always lower case):
1475 re.compile(r'flurryagent', re.IGNORECASE),
1476 re.compile(r'paypal.*mpl', re.IGNORECASE),
1477 re.compile(r'google.*analytics', re.IGNORECASE),
1478 re.compile(r'admob.*sdk.*android', re.IGNORECASE),
1479 re.compile(r'google.*ad.*view', re.IGNORECASE),
1480 re.compile(r'google.*admob', re.IGNORECASE),
1481 re.compile(r'google.*play.*services', re.IGNORECASE),
1482 re.compile(r'crittercism', re.IGNORECASE),
1483 re.compile(r'heyzap', re.IGNORECASE),
1484 re.compile(r'jpct.*ae', re.IGNORECASE),
1485 re.compile(r'youtube.*android.*player.*api', re.IGNORECASE),
1486 re.compile(r'bugsense', re.IGNORECASE),
1487 re.compile(r'crashlytics', re.IGNORECASE),
1488 re.compile(r'ouya.*sdk', re.IGNORECASE),
1489 re.compile(r'libspen23', re.IGNORECASE),
1492 scanignore = getpaths(build_dir, thisbuild, 'scanignore')
1493 scandelete = getpaths(build_dir, thisbuild, 'scandelete')
1495 scanignore_worked = set()
1496 scandelete_worked = set()
1499 ms = magic.open(magic.MIME_TYPE)
1501 except AttributeError:
1505 for p in scanignore:
1506 if fd.startswith(p):
1507 scanignore_worked.add(p)
1512 for p in scandelete:
1513 if fd.startswith(p):
1514 scandelete_worked.add(p)
1518 def ignoreproblem(what, fd, fp):
1519 logging.info('Ignoring %s at %s' % (what, fd))
1522 def removeproblem(what, fd, fp):
1523 logging.info('Removing %s at %s' % (what, fd))
1527 def warnproblem(what, fd):
1528 logging.warn('Found %s at %s' % (what, fd))
1530 def handleproblem(what, fd, fp):
1532 return ignoreproblem(what, fd, fp)
1534 return removeproblem(what, fd, fp)
1535 logging.error('Found %s at %s' % (what, fd))
1538 # Iterate through all files in the source code
1539 for r, d, f in os.walk(build_dir, topdown=True):
1541 # It's topdown, so checking the basename is enough
1542 for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
1548 # Path (relative) to the file
1549 fp = os.path.join(r, curfile)
1550 fd = fp[len(build_dir) + 1:]
1553 mime = magic.from_file(fp, mime=True) if ms is None else ms.file(fp)
1554 except UnicodeError:
1555 warnproblem('malformed magic number', fd)
1557 if mime == 'application/x-sharedlib':
1558 count += handleproblem('shared library', fd, fp)
1560 elif mime == 'application/x-archive':
1561 count += handleproblem('static library', fd, fp)
1563 elif mime == 'application/x-executable':
1564 count += handleproblem('binary executable', fd, fp)
1566 elif mime == 'application/x-java-applet':
1567 count += handleproblem('Java compiled class', fd, fp)
1572 'application/java-archive',
1573 'application/octet-stream',
1576 if has_extension(fp, 'apk'):
1577 removeproblem('APK file', fd, fp)
1579 elif has_extension(fp, 'jar'):
1581 if any(suspect.match(curfile) for suspect in usual_suspects):
1582 count += handleproblem('usual supect', fd, fp)
1584 warnproblem('JAR file', fd)
1586 elif has_extension(fp, 'zip'):
1587 warnproblem('ZIP file', fd)
1590 warnproblem('unknown compressed or binary file', fd)
1592 elif has_extension(fp, 'java') and os.path.isfile(fp):
1593 if not os.path.isfile(fp):
1595 for line in file(fp):
1596 if 'DexClassLoader' in line:
1597 count += handleproblem('DexClassLoader', fd, fp)
1602 for p in scanignore:
1603 if p not in scanignore_worked:
1604 logging.error('Unused scanignore path: %s' % p)
1607 for p in scandelete:
1608 if p not in scandelete_worked:
1609 logging.error('Unused scandelete path: %s' % p)
1612 # Presence of a jni directory without buildjni=yes might
1613 # indicate a problem (if it's not a problem, explicitly use
1614 # buildjni=no to bypass this check)
1615 if (os.path.exists(os.path.join(root_dir, 'jni')) and
1616 not thisbuild['buildjni']):
1617 logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
1626 self.path = os.path.join('stats', 'known_apks.txt')
1628 if os.path.isfile(self.path):
1629 for line in file(self.path):
1630 t = line.rstrip().split(' ')
1632 self.apks[t[0]] = (t[1], None)
1634 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1635 self.changed = False
1637 def writeifchanged(self):
1639 if not os.path.exists('stats'):
1641 f = open(self.path, 'w')
1643 for apk, app in self.apks.iteritems():
1645 line = apk + ' ' + appid
1647 line += ' ' + time.strftime('%Y-%m-%d', added)
1649 for line in sorted(lst):
1650 f.write(line + '\n')
1653 # Record an apk (if it's new, otherwise does nothing)
1654 # Returns the date it was added.
1655 def recordapk(self, apk, app):
1656 if apk not in self.apks:
1657 self.apks[apk] = (app, time.gmtime(time.time()))
1659 _, added = self.apks[apk]
1662 # Look up information - given the 'apkname', returns (app id, date added/None).
1663 # Or returns None for an unknown apk.
1664 def getapp(self, apkname):
1665 if apkname in self.apks:
1666 return self.apks[apkname]
1669 # Get the most recent 'num' apps added to the repo, as a list of package ids
1670 # with the most recent first.
1671 def getlatest(self, num):
1673 for apk, app in self.apks.iteritems():
1677 if apps[appid] > added:
1681 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1682 lst = [app for app, _ in sortedapps]
1687 def isApkDebuggable(apkfile, config):
1688 """Returns True if the given apk file is debuggable
1690 :param apkfile: full path to the apk to check"""
1692 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1694 if p.returncode != 0:
1695 logging.critical("Failed to get apk manifest information")
1697 for line in p.output.splitlines():
1698 if 'android:debuggable' in line and not line.endswith('0x0'):
1703 class AsynchronousFileReader(threading.Thread):
1706 Helper class to implement asynchronous reading of a file
1707 in a separate thread. Pushes read lines on a queue to
1708 be consumed in another thread.
1711 def __init__(self, fd, queue):
1712 assert isinstance(queue, Queue.Queue)
1713 assert callable(fd.readline)
1714 threading.Thread.__init__(self)
1719 '''The body of the tread: read lines and put them on the queue.'''
1720 for line in iter(self._fd.readline, ''):
1721 self._queue.put(line)
1724 '''Check whether there is no more content to expect.'''
1725 return not self.is_alive() and self._queue.empty()
1733 def SdkToolsPopen(commands, cwd=None, output=True):
1735 if cmd not in config:
1736 config[cmd] = find_sdk_tools_cmd(commands[0])
1737 return FDroidPopen([config[cmd]] + commands[1:],
1738 cwd=cwd, output=output)
1741 def FDroidPopen(commands, cwd=None, output=True):
1743 Run a command and capture the possibly huge output.
1745 :param commands: command and argument list like in subprocess.Popen
1746 :param cwd: optionally specifies a working directory
1747 :returns: A PopenResult.
1753 cwd = os.path.normpath(cwd)
1754 logging.debug("Directory: %s" % cwd)
1755 logging.debug("> %s" % ' '.join(commands))
1757 result = PopenResult()
1760 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
1761 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1763 raise BuildException("OSError while trying to execute " +
1764 ' '.join(commands) + ': ' + str(e))
1766 stdout_queue = Queue.Queue()
1767 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1768 stdout_reader.start()
1770 # Check the queue for output (until there is no more to get)
1771 while not stdout_reader.eof():
1772 while not stdout_queue.empty():
1773 line = stdout_queue.get()
1774 if output and options.verbose:
1775 # Output directly to console
1776 sys.stderr.write(line)
1778 result.output += line
1782 result.returncode = p.wait()
1786 def remove_signing_keys(build_dir):
1787 comment = re.compile(r'[ ]*//')
1788 signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1790 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1791 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1792 re.compile(r'.*variant\.outputFile = .*'),
1793 re.compile(r'.*output\.outputFile = .*'),
1794 re.compile(r'.*\.readLine\(.*'),
1796 for root, dirs, files in os.walk(build_dir):
1797 if 'build.gradle' in files:
1798 path = os.path.join(root, 'build.gradle')
1800 with open(path, "r") as o:
1801 lines = o.readlines()
1807 with open(path, "w") as o:
1808 while i < len(lines):
1811 while line.endswith('\\\n'):
1812 line = line.rstrip('\\\n') + lines[i]
1815 if comment.match(line):
1819 opened += line.count('{')
1820 opened -= line.count('}')
1823 if signing_configs.match(line):
1828 if any(s.match(line) for s in line_matches):
1836 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1839 'project.properties',
1841 'default.properties',
1842 'ant.properties', ]:
1843 if propfile in files:
1844 path = os.path.join(root, propfile)
1846 with open(path, "r") as o:
1847 lines = o.readlines()
1851 with open(path, "w") as o:
1853 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1860 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1863 def reset_env_path():
1864 global env, orig_path
1865 env['PATH'] = orig_path
1868 def add_to_env_path(path):
1870 paths = env['PATH'].split(os.pathsep)
1874 env['PATH'] = os.pathsep.join(paths)
1877 def replace_config_vars(cmd):
1879 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1880 # env['ANDROID_NDK'] is set in build_local right before prepare_source
1881 cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
1882 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1886 def place_srclib(root_dir, number, libpath):
1889 relpath = os.path.relpath(libpath, root_dir)
1890 proppath = os.path.join(root_dir, 'project.properties')
1893 if not os.path.isfile(proppath):
1896 with open(proppath, "r") as o:
1897 lines = o.readlines()
1899 with open(proppath, "w") as o:
1902 if line.startswith('android.library.reference.%d=' % number):
1903 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1908 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1911 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
1912 """Verify that two apks are the same
1914 One of the inputs is signed, the other is unsigned. The signature metadata
1915 is transferred from the signed to the unsigned apk, and then jarsigner is
1916 used to verify that the signature from the signed apk is also varlid for
1918 :param signed_apk: Path to a signed apk file
1919 :param unsigned_apk: Path to an unsigned apk file expected to match it
1920 :param tmp_dir: Path to directory for temporary files
1921 :returns: None if the verification is successful, otherwise a string
1922 describing what went wrong.
1924 sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
1925 with ZipFile(signed_apk) as signed_apk_as_zip:
1926 meta_inf_files = ['META-INF/MANIFEST.MF']
1927 for f in signed_apk_as_zip.namelist():
1928 if sigfile.match(f):
1929 meta_inf_files.append(f)
1930 if len(meta_inf_files) < 3:
1931 return "Signature files missing from {0}".format(signed_apk)
1932 signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
1933 with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
1934 for meta_inf_file in meta_inf_files:
1935 unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
1937 if subprocess.call(['jarsigner', '-verify', unsigned_apk]) != 0:
1938 logging.info("...NOT verified - {0}".format(signed_apk))
1939 return compare_apks(signed_apk, unsigned_apk, tmp_dir)
1940 logging.info("...successfully verified")
1944 def compare_apks(apk1, apk2, tmp_dir):
1947 Returns None if the apk content is the same (apart from the signing key),
1948 otherwise a string describing what's different, or what went wrong when
1949 trying to do the comparison.
1952 badchars = re.compile('''[/ :;'"]''')
1953 apk1dir = os.path.join(tmp_dir, badchars.sub('_', apk1[0:-4])) # trim .apk
1954 apk2dir = os.path.join(tmp_dir, badchars.sub('_', apk2[0:-4])) # trim .apk
1955 for d in [apk1dir, apk2dir]:
1956 if os.path.exists(d):
1959 os.mkdir(os.path.join(d, 'jar-xf'))
1961 if subprocess.call(['jar', 'xf',
1962 os.path.abspath(apk1)],
1963 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
1964 return("Failed to unpack " + apk1)
1965 if subprocess.call(['jar', 'xf',
1966 os.path.abspath(apk2)],
1967 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
1968 return("Failed to unpack " + apk2)
1970 # try to find apktool in the path, if it hasn't been manually configed
1971 if 'apktool' not in config:
1972 tmp = find_command('apktool')
1974 config['apktool'] = tmp
1975 if 'apktool' in config:
1976 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
1978 return("Failed to unpack " + apk1)
1979 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
1981 return("Failed to unpack " + apk2)
1983 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
1984 lines = p.output.splitlines()
1985 if len(lines) != 1 or 'META-INF' not in lines[0]:
1986 meld = find_command('meld')
1987 if meld is not None:
1988 p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
1989 return("Unexpected diff output - " + p.output)
1991 # since everything verifies, delete the comparison to keep cruft down
1992 shutil.rmtree(apk1dir)
1993 shutil.rmtree(apk2dir)
1995 # If we get here, it seems like they're the same!
1999 def find_command(command):
2000 '''find the full path of a command, or None if it can't be found in the PATH'''
2003 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2005 fpath, fname = os.path.split(command)
2010 for path in os.environ["PATH"].split(os.pathsep):
2011 path = path.strip('"')
2012 exe_file = os.path.join(path, command)
2013 if is_exe(exe_file):