3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
37 import xml.etree.ElementTree as XMLElementTree
39 from queue import Queue
41 from zipfile import ZipFile
43 import fdroidserver.metadata
44 from .asynchronousfilereader import AsynchronousFileReader
47 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
56 'sdk_path': "$ANDROID_HOME",
59 'r10e': "$ANDROID_NDK",
61 'build_tools': "23.0.2",
66 'accepted_formats': ['txt', 'yaml'],
67 'sync_from_local_copy_dir': False,
68 'per_app_repos': False,
69 'make_current_version_link': True,
70 'current_version_name_source': 'Name',
71 'update_stats': False,
75 'stats_to_carbon': False,
77 'build_server_always': False,
78 'keystore': 'keystore.jks',
79 'smartcardoptions': [],
85 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
86 'repo_name': "My First FDroid Repo Demo",
87 'repo_icon': "fdroid-icon.png",
88 'repo_description': '''
89 This is a repository of apps to be used with FDroid. Applications in this
90 repository are either official binaries built by the original application
91 developers, or are binaries built from source by the admin of f-droid.org
92 using the tools on https://gitlab.com/u/fdroid.
98 def setup_global_opts(parser):
99 parser.add_argument("-v", "--verbose", action="store_true", default=False,
100 help="Spew out even more information than normal")
101 parser.add_argument("-q", "--quiet", action="store_true", default=False,
102 help="Restrict output to warnings and errors")
105 def fill_config_defaults(thisconfig):
106 for k, v in default_config.items():
107 if k not in thisconfig:
110 # Expand paths (~users and $vars)
111 def expand_path(path):
115 path = os.path.expanduser(path)
116 path = os.path.expandvars(path)
121 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
126 thisconfig[k + '_orig'] = v
128 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
129 if thisconfig['java_paths'] is None:
130 thisconfig['java_paths'] = dict()
132 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
133 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
134 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
135 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
136 if os.getenv('JAVA_HOME') is not None:
137 pathlist += os.getenv('JAVA_HOME')
138 if os.getenv('PROGRAMFILES') is not None:
139 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
140 for d in sorted(pathlist):
141 if os.path.islink(d):
143 j = os.path.basename(d)
144 # the last one found will be the canonical one, so order appropriately
146 r'^1\.([6-9])\.0\.jdk$', # OSX
147 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
148 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
149 r'^jdk([6-9])-openjdk$', # Arch
150 r'^java-([6-9])-openjdk$', # Arch
151 r'^java-([6-9])-jdk$', # Arch (oracle)
152 r'^java-1\.([6-9])\.0-.*$', # RedHat
153 r'^java-([6-9])-oracle$', # Debian WebUpd8
154 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
155 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
157 m = re.match(regex, j)
160 osxhome = os.path.join(d, 'Contents', 'Home')
161 if os.path.exists(osxhome):
162 thisconfig['java_paths'][m.group(1)] = osxhome
164 thisconfig['java_paths'][m.group(1)] = d
166 for java_version in ('7', '8', '9'):
167 if java_version not in thisconfig['java_paths']:
169 java_home = thisconfig['java_paths'][java_version]
170 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
171 if os.path.exists(jarsigner):
172 thisconfig['jarsigner'] = jarsigner
173 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
174 break # Java7 is preferred, so quit if found
176 for k in ['ndk_paths', 'java_paths']:
182 thisconfig[k][k2] = exp
183 thisconfig[k][k2 + '_orig'] = v
186 def regsub_file(pattern, repl, path):
187 with open(path, 'r') as f:
189 text = re.sub(pattern, repl, text)
190 with open(path, 'w') as f:
194 def read_config(opts, config_file='config.py'):
195 """Read the repository config
197 The config is read from config_file, which is in the current directory when
198 any of the repo management commands are used.
200 global config, options, env, orig_path
202 if config is not None:
204 if not os.path.isfile(config_file):
205 logging.critical("Missing config file - is this a repo directory?")
212 logging.debug("Reading %s" % config_file)
213 with io.open(config_file, "rb") as f:
214 code = compile(f.read(), config_file, 'exec')
215 exec(code, None, config)
217 # smartcardoptions must be a list since its command line args for Popen
218 if 'smartcardoptions' in config:
219 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
220 elif 'keystore' in config and config['keystore'] == 'NONE':
221 # keystore='NONE' means use smartcard, these are required defaults
222 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
223 'SunPKCS11-OpenSC', '-providerClass',
224 'sun.security.pkcs11.SunPKCS11',
225 '-providerArg', 'opensc-fdroid.cfg']
227 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
228 st = os.stat(config_file)
229 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
230 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
232 fill_config_defaults(config)
234 # There is no standard, so just set up the most common environment
237 orig_path = env['PATH']
238 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
239 env[n] = config['sdk_path']
241 for k, v in config['java_paths'].items():
242 env['JAVA%s_HOME' % k] = v
244 for k in ["keystorepass", "keypass"]:
246 write_password_file(k)
248 for k in ["repo_description", "archive_description"]:
250 config[k] = clean_description(config[k])
252 if 'serverwebroot' in config:
253 if isinstance(config['serverwebroot'], str):
254 roots = [config['serverwebroot']]
255 elif all(isinstance(item, str) for item in config['serverwebroot']):
256 roots = config['serverwebroot']
258 raise TypeError('only accepts strings, lists, and tuples')
260 for rootstr in roots:
261 # since this is used with rsync, where trailing slashes have
262 # meaning, ensure there is always a trailing slash
263 if rootstr[-1] != '/':
265 rootlist.append(rootstr.replace('//', '/'))
266 config['serverwebroot'] = rootlist
271 def find_sdk_tools_cmd(cmd):
272 '''find a working path to a tool from the Android SDK'''
275 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
276 # try to find a working path to this command, in all the recent possible paths
277 if 'build_tools' in config:
278 build_tools = os.path.join(config['sdk_path'], 'build-tools')
279 # if 'build_tools' was manually set and exists, check only that one
280 configed_build_tools = os.path.join(build_tools, config['build_tools'])
281 if os.path.exists(configed_build_tools):
282 tooldirs.append(configed_build_tools)
284 # no configed version, so hunt known paths for it
285 for f in sorted(os.listdir(build_tools), reverse=True):
286 if os.path.isdir(os.path.join(build_tools, f)):
287 tooldirs.append(os.path.join(build_tools, f))
288 tooldirs.append(build_tools)
289 sdk_tools = os.path.join(config['sdk_path'], 'tools')
290 if os.path.exists(sdk_tools):
291 tooldirs.append(sdk_tools)
292 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
293 if os.path.exists(sdk_platform_tools):
294 tooldirs.append(sdk_platform_tools)
295 tooldirs.append('/usr/bin')
297 if os.path.isfile(os.path.join(d, cmd)):
298 return os.path.join(d, cmd)
299 # did not find the command, exit with error message
300 ensure_build_tools_exists(config)
303 def test_sdk_exists(thisconfig):
304 if 'sdk_path' not in thisconfig:
305 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
308 logging.error("'sdk_path' not set in config.py!")
310 if thisconfig['sdk_path'] == default_config['sdk_path']:
311 logging.error('No Android SDK found!')
312 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
313 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
315 if not os.path.exists(thisconfig['sdk_path']):
316 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
318 if not os.path.isdir(thisconfig['sdk_path']):
319 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
321 for d in ['build-tools', 'platform-tools', 'tools']:
322 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
323 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
324 thisconfig['sdk_path'], d))
329 def ensure_build_tools_exists(thisconfig):
330 if not test_sdk_exists(thisconfig):
332 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
333 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
334 if not os.path.isdir(versioned_build_tools):
335 logging.critical('Android Build Tools path "'
336 + versioned_build_tools + '" does not exist!')
340 def write_password_file(pwtype, password=None):
342 writes out passwords to a protected file instead of passing passwords as
343 command line argments
345 filename = '.fdroid.' + pwtype + '.txt'
346 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
348 os.write(fd, config[pwtype].encode('utf-8'))
350 os.write(fd, password.encode('utf-8'))
352 config[pwtype + 'file'] = filename
355 # Given the arguments in the form of multiple appid:[vc] strings, this returns
356 # a dictionary with the set of vercodes specified for each package.
357 def read_pkg_args(args, allow_vercodes=False):
364 if allow_vercodes and ':' in p:
365 package, vercode = p.split(':')
367 package, vercode = p, None
368 if package not in vercodes:
369 vercodes[package] = [vercode] if vercode else []
371 elif vercode and vercode not in vercodes[package]:
372 vercodes[package] += [vercode] if vercode else []
377 # On top of what read_pkg_args does, this returns the whole app metadata, but
378 # limiting the builds list to the builds matching the vercodes specified.
379 def read_app_args(args, allapps, allow_vercodes=False):
381 vercodes = read_pkg_args(args, allow_vercodes)
387 for appid, app in allapps.items():
388 if appid in vercodes:
391 if len(apps) != len(vercodes):
394 logging.critical("No such package: %s" % p)
395 raise FDroidException("Found invalid app ids in arguments")
397 raise FDroidException("No packages specified")
400 for appid, app in apps.items():
404 app.builds = [b for b in app.builds if b.vercode in vc]
405 if len(app.builds) != len(vercodes[appid]):
407 allvcs = [b.vercode for b in app.builds]
408 for v in vercodes[appid]:
410 logging.critical("No such vercode %s for app %s" % (v, appid))
413 raise FDroidException("Found invalid vercodes for some apps")
418 def get_extension(filename):
419 base, ext = os.path.splitext(filename)
422 return base, ext.lower()[1:]
425 def has_extension(filename, ext):
426 _, f_ext = get_extension(filename)
430 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
433 def clean_description(description):
434 'Remove unneeded newlines and spaces from a block of description text'
436 # this is split up by paragraph to make removing the newlines easier
437 for paragraph in re.split(r'\n\n', description):
438 paragraph = re.sub('\r', '', paragraph)
439 paragraph = re.sub('\n', ' ', paragraph)
440 paragraph = re.sub(' {2,}', ' ', paragraph)
441 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
442 returnstring += paragraph + '\n\n'
443 return returnstring.rstrip('\n')
446 def apknameinfo(filename):
447 filename = os.path.basename(filename)
448 m = apk_regex.match(filename)
450 result = (m.group(1), m.group(2))
451 except AttributeError:
452 raise FDroidException("Invalid apk name: %s" % filename)
456 def getapkname(app, build):
457 return "%s_%s.apk" % (app.id, build.vercode)
460 def getsrcname(app, build):
461 return "%s_%s_src.tar.gz" % (app.id, build.vercode)
473 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
476 def getvcs(vcstype, remote, local):
478 return vcs_git(remote, local)
479 if vcstype == 'git-svn':
480 return vcs_gitsvn(remote, local)
482 return vcs_hg(remote, local)
484 return vcs_bzr(remote, local)
485 if vcstype == 'srclib':
486 if local != os.path.join('build', 'srclib', remote):
487 raise VCSException("Error: srclib paths are hard-coded!")
488 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
490 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
491 raise VCSException("Invalid vcs type " + vcstype)
494 def getsrclibvcs(name):
495 if name not in fdroidserver.metadata.srclibs:
496 raise VCSException("Missing srclib " + name)
497 return fdroidserver.metadata.srclibs[name]['Repo Type']
502 def __init__(self, remote, local):
504 # svn, git-svn and bzr may require auth
506 if self.repotype() in ('git-svn', 'bzr'):
508 if self.repotype == 'git-svn':
509 raise VCSException("Authentication is not supported for git-svn")
510 self.username, remote = remote.split('@')
511 if ':' not in self.username:
512 raise VCSException("Password required with username")
513 self.username, self.password = self.username.split(':')
517 self.clone_failed = False
518 self.refreshed = False
524 # Take the local repository to a clean version of the given revision, which
525 # is specificed in the VCS's native format. Beforehand, the repository can
526 # be dirty, or even non-existent. If the repository does already exist
527 # locally, it will be updated from the origin, but only once in the
528 # lifetime of the vcs object.
529 # None is acceptable for 'rev' if you know you are cloning a clean copy of
530 # the repo - otherwise it must specify a valid revision.
531 def gotorevision(self, rev, refresh=True):
533 if self.clone_failed:
534 raise VCSException("Downloading the repository already failed once, not trying again.")
536 # The .fdroidvcs-id file for a repo tells us what VCS type
537 # and remote that directory was created from, allowing us to drop it
538 # automatically if either of those things changes.
539 fdpath = os.path.join(self.local, '..',
540 '.fdroidvcs-' + os.path.basename(self.local))
541 fdpath = os.path.normpath(fdpath)
542 cdata = self.repotype() + ' ' + self.remote
545 if os.path.exists(self.local):
546 if os.path.exists(fdpath):
547 with open(fdpath, 'r') as f:
548 fsdata = f.read().strip()
553 logging.info("Repository details for %s changed - deleting" % (
557 logging.info("Repository details for %s missing - deleting" % (
560 shutil.rmtree(self.local)
564 self.refreshed = True
567 self.gotorevisionx(rev)
568 except FDroidException as e:
571 # If necessary, write the .fdroidvcs file.
572 if writeback and not self.clone_failed:
573 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
574 with open(fdpath, 'w+') as f:
580 # Derived classes need to implement this. It's called once basic checking
581 # has been performend.
582 def gotorevisionx(self, rev):
583 raise VCSException("This VCS type doesn't define gotorevisionx")
585 # Initialise and update submodules
586 def initsubmodules(self):
587 raise VCSException('Submodules not supported for this vcs type')
589 # Get a list of all known tags
591 if not self._gettags:
592 raise VCSException('gettags not supported for this vcs type')
594 for tag in self._gettags():
595 if re.match('[-A-Za-z0-9_. /]+$', tag):
599 # Get a list of all the known tags, sorted from newest to oldest
600 def latesttags(self):
601 raise VCSException('latesttags not supported for this vcs type')
603 # Get current commit reference (hash, revision, etc)
605 raise VCSException('getref not supported for this vcs type')
607 # Returns the srclib (name, path) used in setting up the current
618 # If the local directory exists, but is somehow not a git repository, git
619 # will traverse up the directory tree until it finds one that is (i.e.
620 # fdroidserver) and then we'll proceed to destroy it! This is called as
623 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
624 result = p.output.rstrip()
625 if not result.endswith(self.local):
626 raise VCSException('Repository mismatch')
628 def gotorevisionx(self, rev):
629 if not os.path.exists(self.local):
631 p = FDroidPopen(['git', 'clone', self.remote, self.local])
632 if p.returncode != 0:
633 self.clone_failed = True
634 raise VCSException("Git clone failed", p.output)
638 # Discard any working tree changes
639 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
640 'git', 'reset', '--hard'], cwd=self.local, output=False)
641 if p.returncode != 0:
642 raise VCSException("Git reset failed", p.output)
643 # Remove untracked files now, in case they're tracked in the target
644 # revision (it happens!)
645 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
646 'git', 'clean', '-dffx'], cwd=self.local, output=False)
647 if p.returncode != 0:
648 raise VCSException("Git clean failed", p.output)
649 if not self.refreshed:
650 # Get latest commits and tags from remote
651 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
652 if p.returncode != 0:
653 raise VCSException("Git fetch failed", p.output)
654 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
655 if p.returncode != 0:
656 raise VCSException("Git fetch failed", p.output)
657 # Recreate origin/HEAD as git clone would do it, in case it disappeared
658 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
659 if p.returncode != 0:
660 lines = p.output.splitlines()
661 if 'Multiple remote HEAD branches' not in lines[0]:
662 raise VCSException("Git remote set-head failed", p.output)
663 branch = lines[1].split(' ')[-1]
664 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
665 if p2.returncode != 0:
666 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
667 self.refreshed = True
668 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
669 # a github repo. Most of the time this is the same as origin/master.
670 rev = rev or 'origin/HEAD'
671 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
672 if p.returncode != 0:
673 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
674 # Get rid of any uncontrolled files left behind
675 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
676 if p.returncode != 0:
677 raise VCSException("Git clean failed", p.output)
679 def initsubmodules(self):
681 submfile = os.path.join(self.local, '.gitmodules')
682 if not os.path.isfile(submfile):
683 raise VCSException("No git submodules available")
685 # fix submodules not accessible without an account and public key auth
686 with open(submfile, 'r') as f:
687 lines = f.readlines()
688 with open(submfile, 'w') as f:
690 if 'git@github.com' in line:
691 line = line.replace('git@github.com:', 'https://github.com/')
692 if 'git@gitlab.com' in line:
693 line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
696 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
697 if p.returncode != 0:
698 raise VCSException("Git submodule sync failed", p.output)
699 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
700 if p.returncode != 0:
701 raise VCSException("Git submodule update failed", p.output)
705 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
706 return p.output.splitlines()
708 tag_format = re.compile(r'.*tag: ([^),]*).*')
710 def latesttags(self):
712 p = FDroidPopen(['git', 'log', '--tags',
713 '--simplify-by-decoration', '--pretty=format:%d'],
714 cwd=self.local, output=False)
716 for line in p.output.splitlines():
717 m = self.tag_format.match(line)
725 class vcs_gitsvn(vcs):
730 # If the local directory exists, but is somehow not a git repository, git
731 # will traverse up the directory tree until it finds one that is (i.e.
732 # fdroidserver) and then we'll proceed to destory it! This is called as
735 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
736 result = p.output.rstrip()
737 if not result.endswith(self.local):
738 raise VCSException('Repository mismatch')
740 def gotorevisionx(self, rev):
741 if not os.path.exists(self.local):
743 gitsvn_args = ['git', 'svn', 'clone']
744 if ';' in self.remote:
745 remote_split = self.remote.split(';')
746 for i in remote_split[1:]:
747 if i.startswith('trunk='):
748 gitsvn_args.extend(['-T', i[6:]])
749 elif i.startswith('tags='):
750 gitsvn_args.extend(['-t', i[5:]])
751 elif i.startswith('branches='):
752 gitsvn_args.extend(['-b', i[9:]])
753 gitsvn_args.extend([remote_split[0], self.local])
754 p = FDroidPopen(gitsvn_args, output=False)
755 if p.returncode != 0:
756 self.clone_failed = True
757 raise VCSException("Git svn clone failed", p.output)
759 gitsvn_args.extend([self.remote, self.local])
760 p = FDroidPopen(gitsvn_args, output=False)
761 if p.returncode != 0:
762 self.clone_failed = True
763 raise VCSException("Git svn clone failed", p.output)
767 # Discard any working tree changes
768 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
769 if p.returncode != 0:
770 raise VCSException("Git reset failed", p.output)
771 # Remove untracked files now, in case they're tracked in the target
772 # revision (it happens!)
773 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
774 if p.returncode != 0:
775 raise VCSException("Git clean failed", p.output)
776 if not self.refreshed:
777 # Get new commits, branches and tags from repo
778 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
779 if p.returncode != 0:
780 raise VCSException("Git svn fetch failed")
781 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
782 if p.returncode != 0:
783 raise VCSException("Git svn rebase failed", p.output)
784 self.refreshed = True
786 rev = rev or 'master'
788 nospaces_rev = rev.replace(' ', '%20')
789 # Try finding a svn tag
790 for treeish in ['origin/', '']:
791 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
792 if p.returncode == 0:
794 if p.returncode != 0:
795 # No tag found, normal svn rev translation
796 # Translate svn rev into git format
797 rev_split = rev.split('/')
800 for treeish in ['origin/', '']:
801 if len(rev_split) > 1:
802 treeish += rev_split[0]
803 svn_rev = rev_split[1]
806 # if no branch is specified, then assume trunk (i.e. 'master' branch):
810 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
812 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
813 git_rev = p.output.rstrip()
815 if p.returncode == 0 and git_rev:
818 if p.returncode != 0 or not git_rev:
819 # Try a plain git checkout as a last resort
820 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
821 if p.returncode != 0:
822 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
824 # Check out the git rev equivalent to the svn rev
825 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
826 if p.returncode != 0:
827 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
829 # Get rid of any uncontrolled files left behind
830 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
831 if p.returncode != 0:
832 raise VCSException("Git clean failed", p.output)
836 for treeish in ['origin/', '']:
837 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
843 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
844 if p.returncode != 0:
846 return p.output.strip()
854 def gotorevisionx(self, rev):
855 if not os.path.exists(self.local):
856 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
857 if p.returncode != 0:
858 self.clone_failed = True
859 raise VCSException("Hg clone failed", p.output)
861 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
862 if p.returncode != 0:
863 raise VCSException("Hg status failed", p.output)
864 for line in p.output.splitlines():
865 if not line.startswith('? '):
866 raise VCSException("Unexpected output from hg status -uS: " + line)
867 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
868 if not self.refreshed:
869 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
870 if p.returncode != 0:
871 raise VCSException("Hg pull failed", p.output)
872 self.refreshed = True
874 rev = rev or 'default'
877 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
878 if p.returncode != 0:
879 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
880 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
881 # Also delete untracked files, we have to enable purge extension for that:
882 if "'purge' is provided by the following extension" in p.output:
883 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
884 myfile.write("\n[extensions]\nhgext.purge=\n")
885 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
886 if p.returncode != 0:
887 raise VCSException("HG purge failed", p.output)
888 elif p.returncode != 0:
889 raise VCSException("HG purge failed", p.output)
892 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
893 return p.output.splitlines()[1:]
901 def gotorevisionx(self, rev):
902 if not os.path.exists(self.local):
903 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
904 if p.returncode != 0:
905 self.clone_failed = True
906 raise VCSException("Bzr branch failed", p.output)
908 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
909 if p.returncode != 0:
910 raise VCSException("Bzr revert failed", p.output)
911 if not self.refreshed:
912 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
913 if p.returncode != 0:
914 raise VCSException("Bzr update failed", p.output)
915 self.refreshed = True
917 revargs = list(['-r', rev] if rev else [])
918 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
919 if p.returncode != 0:
920 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
923 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
924 return [tag.split(' ')[0].strip() for tag in
925 p.output.splitlines()]
928 def unescape_string(string):
931 if string[0] == '"' and string[-1] == '"':
934 return string.replace("\\'", "'")
937 def retrieve_string(app_dir, string, xmlfiles=None):
939 if not string.startswith('@string/'):
940 return unescape_string(string)
945 os.path.join(app_dir, 'res'),
946 os.path.join(app_dir, 'src', 'main', 'res'),
948 for r, d, f in os.walk(res_dir):
949 if os.path.basename(r) == 'values':
950 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
952 name = string[len('@string/'):]
954 def element_content(element):
955 if element.text is None:
957 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
958 return s.decode('utf-8').strip()
960 for path in xmlfiles:
961 if not os.path.isfile(path):
963 xml = parse_xml(path)
964 element = xml.find('string[@name="' + name + '"]')
965 if element is not None:
966 content = element_content(element)
967 return retrieve_string(app_dir, content, xmlfiles)
972 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
973 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
976 # Return list of existing files that will be used to find the highest vercode
977 def manifest_paths(app_dir, flavours):
979 possible_manifests = \
980 [os.path.join(app_dir, 'AndroidManifest.xml'),
981 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
982 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
983 os.path.join(app_dir, 'build.gradle')]
985 for flavour in flavours:
988 possible_manifests.append(
989 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
991 return [path for path in possible_manifests if os.path.isfile(path)]
994 # Retrieve the package name. Returns the name, or None if not found.
995 def fetch_real_name(app_dir, flavours):
996 for path in manifest_paths(app_dir, flavours):
997 if not has_extension(path, 'xml') or not os.path.isfile(path):
999 logging.debug("fetch_real_name: Checking manifest at " + path)
1000 xml = parse_xml(path)
1001 app = xml.find('application')
1004 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1006 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1007 result = retrieve_string_singleline(app_dir, label)
1009 result = result.strip()
1014 def get_library_references(root_dir):
1016 proppath = os.path.join(root_dir, 'project.properties')
1017 if not os.path.isfile(proppath):
1019 with open(proppath, 'r') as f:
1021 if not line.startswith('android.library.reference.'):
1023 path = line.split('=')[1].strip()
1024 relpath = os.path.join(root_dir, path)
1025 if not os.path.isdir(relpath):
1027 logging.debug("Found subproject at %s" % path)
1028 libraries.append(path)
1032 def ant_subprojects(root_dir):
1033 subprojects = get_library_references(root_dir)
1034 for subpath in subprojects:
1035 subrelpath = os.path.join(root_dir, subpath)
1036 for p in get_library_references(subrelpath):
1037 relp = os.path.normpath(os.path.join(subpath, p))
1038 if relp not in subprojects:
1039 subprojects.insert(0, relp)
1043 def remove_debuggable_flags(root_dir):
1044 # Remove forced debuggable flags
1045 logging.debug("Removing debuggable flags from %s" % root_dir)
1046 for root, dirs, files in os.walk(root_dir):
1047 if 'AndroidManifest.xml' in files:
1048 regsub_file(r'android:debuggable="[^"]*"',
1050 os.path.join(root, 'AndroidManifest.xml'))
1053 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
1054 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1055 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1058 def app_matches_packagename(app, package):
1061 appid = app.UpdateCheckName or app.id
1062 if appid is None or appid == "Ignore":
1064 return appid == package
1067 # Extract some information from the AndroidManifest.xml at the given path.
1068 # Returns (version, vercode, package), any or all of which might be None.
1069 # All values returned are strings.
1070 def parse_androidmanifests(paths, app):
1072 ignoreversions = app.UpdateCheckIgnore
1073 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1076 return (None, None, None)
1084 if not os.path.isfile(path):
1087 logging.debug("Parsing manifest at {0}".format(path))
1088 gradle = has_extension(path, 'gradle')
1094 with open(path, 'r') as f:
1096 if gradle_comment.match(line):
1098 # Grab first occurence of each to avoid running into
1099 # alternative flavours and builds.
1101 matches = psearch_g(line)
1103 s = matches.group(2)
1104 if app_matches_packagename(app, s):
1107 matches = vnsearch_g(line)
1109 version = matches.group(2)
1111 matches = vcsearch_g(line)
1113 vercode = matches.group(1)
1116 xml = parse_xml(path)
1117 if "package" in xml.attrib:
1118 s = xml.attrib["package"]
1119 if app_matches_packagename(app, s):
1121 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1122 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1123 base_dir = os.path.dirname(path)
1124 version = retrieve_string_singleline(base_dir, version)
1125 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1126 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1127 if string_is_integer(a):
1130 logging.warning("Problem with xml at {0}".format(path))
1132 # Remember package name, may be defined separately from version+vercode
1134 package = max_package
1136 logging.debug("..got package={0}, version={1}, vercode={2}"
1137 .format(package, version, vercode))
1139 # Always grab the package name and version name in case they are not
1140 # together with the highest version code
1141 if max_package is None and package is not None:
1142 max_package = package
1143 if max_version is None and version is not None:
1144 max_version = version
1146 if max_vercode is None or (vercode is not None and vercode > max_vercode):
1147 if not ignoresearch or not ignoresearch(version):
1148 if version is not None:
1149 max_version = version
1150 if vercode is not None:
1151 max_vercode = vercode
1152 if package is not None:
1153 max_package = package
1155 max_version = "Ignore"
1157 if max_version is None:
1158 max_version = "Unknown"
1160 if max_package and not is_valid_package_name(max_package):
1161 raise FDroidException("Invalid package name {0}".format(max_package))
1163 return (max_version, max_vercode, max_package)
1166 def is_valid_package_name(name):
1167 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1170 class FDroidException(Exception):
1172 def __init__(self, value, detail=None):
1174 self.detail = detail
1176 def shortened_detail(self):
1177 if len(self.detail) < 16000:
1179 return '[...]\n' + self.detail[-16000:]
1181 def get_wikitext(self):
1182 ret = repr(self.value) + "\n"
1185 ret += "<pre>\n" + self.shortened_detail() + "</pre>\n"
1191 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1195 class VCSException(FDroidException):
1199 class BuildException(FDroidException):
1203 # Get the specified source library.
1204 # Returns the path to it. Normally this is the path to be used when referencing
1205 # it, which may be a subdirectory of the actual project. If you want the base
1206 # directory of the project, pass 'basepath=True'.
1207 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1208 raw=False, prepare=True, preponly=False, refresh=True):
1216 name, ref = spec.split('@')
1218 number, name = name.split(':', 1)
1220 name, subdir = name.split('/', 1)
1222 if name not in fdroidserver.metadata.srclibs:
1223 raise VCSException('srclib ' + name + ' not found.')
1225 srclib = fdroidserver.metadata.srclibs[name]
1227 sdir = os.path.join(srclib_dir, name)
1230 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1231 vcs.srclib = (name, number, sdir)
1233 vcs.gotorevision(ref, refresh)
1240 libdir = os.path.join(sdir, subdir)
1241 elif srclib["Subdir"]:
1242 for subdir in srclib["Subdir"]:
1243 libdir_candidate = os.path.join(sdir, subdir)
1244 if os.path.exists(libdir_candidate):
1245 libdir = libdir_candidate
1251 remove_signing_keys(sdir)
1252 remove_debuggable_flags(sdir)
1256 if srclib["Prepare"]:
1257 cmd = replace_config_vars(srclib["Prepare"], None)
1259 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1260 if p.returncode != 0:
1261 raise BuildException("Error running prepare command for srclib %s"
1267 return (name, number, libdir)
1269 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1272 # Prepare the source code for a particular build
1273 # 'vcs' - the appropriate vcs object for the application
1274 # 'app' - the application details from the metadata
1275 # 'build' - the build details from the metadata
1276 # 'build_dir' - the path to the build directory, usually
1278 # 'srclib_dir' - the path to the source libraries directory, usually
1280 # 'extlib_dir' - the path to the external libraries directory, usually
1282 # Returns the (root, srclibpaths) where:
1283 # 'root' is the root directory, which may be the same as 'build_dir' or may
1284 # be a subdirectory of it.
1285 # 'srclibpaths' is information on the srclibs being used
1286 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1288 # Optionally, the actual app source can be in a subdirectory
1290 root_dir = os.path.join(build_dir, build.subdir)
1292 root_dir = build_dir
1294 # Get a working copy of the right revision
1295 logging.info("Getting source for revision " + build.commit)
1296 vcs.gotorevision(build.commit, refresh)
1298 # Initialise submodules if required
1299 if build.submodules:
1300 logging.info("Initialising submodules")
1301 vcs.initsubmodules()
1303 # Check that a subdir (if we're using one) exists. This has to happen
1304 # after the checkout, since it might not exist elsewhere
1305 if not os.path.exists(root_dir):
1306 raise BuildException('Missing subdir ' + root_dir)
1308 # Run an init command if one is required
1310 cmd = replace_config_vars(build.init, build)
1311 logging.info("Running 'init' commands in %s" % root_dir)
1313 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1314 if p.returncode != 0:
1315 raise BuildException("Error running init command for %s:%s" %
1316 (app.id, build.version), p.output)
1318 # Apply patches if any
1320 logging.info("Applying patches")
1321 for patch in build.patch:
1322 patch = patch.strip()
1323 logging.info("Applying " + patch)
1324 patch_path = os.path.join('metadata', app.id, patch)
1325 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1326 if p.returncode != 0:
1327 raise BuildException("Failed to apply patch %s" % patch_path)
1329 # Get required source libraries
1332 logging.info("Collecting source libraries")
1333 for lib in build.srclibs:
1334 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
1336 for name, number, libpath in srclibpaths:
1337 place_srclib(root_dir, int(number) if number else None, libpath)
1339 basesrclib = vcs.getsrclib()
1340 # If one was used for the main source, add that too.
1342 srclibpaths.append(basesrclib)
1344 # Update the local.properties file
1345 localprops = [os.path.join(build_dir, 'local.properties')]
1347 parts = build.subdir.split(os.sep)
1350 cur = os.path.join(cur, d)
1351 localprops += [os.path.join(cur, 'local.properties')]
1352 for path in localprops:
1354 if os.path.isfile(path):
1355 logging.info("Updating local.properties file at %s" % path)
1356 with open(path, 'r') as f:
1360 logging.info("Creating local.properties file at %s" % path)
1361 # Fix old-fashioned 'sdk-location' by copying
1362 # from sdk.dir, if necessary
1364 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1365 re.S | re.M).group(1)
1366 props += "sdk-location=%s\n" % sdkloc
1368 props += "sdk.dir=%s\n" % config['sdk_path']
1369 props += "sdk-location=%s\n" % config['sdk_path']
1370 ndk_path = build.ndk_path()
1373 props += "ndk.dir=%s\n" % ndk_path
1374 props += "ndk-location=%s\n" % ndk_path
1375 # Add java.encoding if necessary
1377 props += "java.encoding=%s\n" % build.encoding
1378 with open(path, 'w') as f:
1382 if build.build_method() == 'gradle':
1383 flavours = build.gradle
1386 n = build.target.split('-')[1]
1387 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1388 r'compileSdkVersion %s' % n,
1389 os.path.join(root_dir, 'build.gradle'))
1391 # Remove forced debuggable flags
1392 remove_debuggable_flags(root_dir)
1394 # Insert version code and number into the manifest if necessary
1395 if build.forceversion:
1396 logging.info("Changing the version name")
1397 for path in manifest_paths(root_dir, flavours):
1398 if not os.path.isfile(path):
1400 if has_extension(path, 'xml'):
1401 regsub_file(r'android:versionName="[^"]*"',
1402 r'android:versionName="%s"' % build.version,
1404 elif has_extension(path, 'gradle'):
1405 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1406 r"""\1versionName '%s'""" % build.version,
1409 if build.forcevercode:
1410 logging.info("Changing the version code")
1411 for path in manifest_paths(root_dir, flavours):
1412 if not os.path.isfile(path):
1414 if has_extension(path, 'xml'):
1415 regsub_file(r'android:versionCode="[^"]*"',
1416 r'android:versionCode="%s"' % build.vercode,
1418 elif has_extension(path, 'gradle'):
1419 regsub_file(r'versionCode[ =]+[0-9]+',
1420 r'versionCode %s' % build.vercode,
1423 # Delete unwanted files
1425 logging.info("Removing specified files")
1426 for part in getpaths(build_dir, build.rm):
1427 dest = os.path.join(build_dir, part)
1428 logging.info("Removing {0}".format(part))
1429 if os.path.lexists(dest):
1430 if os.path.islink(dest):
1431 FDroidPopen(['unlink', dest], output=False)
1433 FDroidPopen(['rm', '-rf', dest], output=False)
1435 logging.info("...but it didn't exist")
1437 remove_signing_keys(build_dir)
1439 # Add required external libraries
1441 logging.info("Collecting prebuilt libraries")
1442 libsdir = os.path.join(root_dir, 'libs')
1443 if not os.path.exists(libsdir):
1445 for lib in build.extlibs:
1447 logging.info("...installing extlib {0}".format(lib))
1448 libf = os.path.basename(lib)
1449 libsrc = os.path.join(extlib_dir, lib)
1450 if not os.path.exists(libsrc):
1451 raise BuildException("Missing extlib file {0}".format(libsrc))
1452 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1454 # Run a pre-build command if one is required
1456 logging.info("Running 'prebuild' commands in %s" % root_dir)
1458 cmd = replace_config_vars(build.prebuild, build)
1460 # Substitute source library paths into prebuild commands
1461 for name, number, libpath in srclibpaths:
1462 libpath = os.path.relpath(libpath, root_dir)
1463 cmd = cmd.replace('$$' + name + '$$', libpath)
1465 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1466 if p.returncode != 0:
1467 raise BuildException("Error running prebuild command for %s:%s" %
1468 (app.id, build.version), p.output)
1470 # Generate (or update) the ant build file, build.xml...
1471 if build.build_method() == 'ant' and build.update != ['no']:
1472 parms = ['android', 'update', 'lib-project']
1473 lparms = ['android', 'update', 'project']
1476 parms += ['-t', build.target]
1477 lparms += ['-t', build.target]
1479 update_dirs = build.update
1481 update_dirs = ant_subprojects(root_dir) + ['.']
1483 for d in update_dirs:
1484 subdir = os.path.join(root_dir, d)
1486 logging.debug("Updating main project")
1487 cmd = parms + ['-p', d]
1489 logging.debug("Updating subproject %s" % d)
1490 cmd = lparms + ['-p', d]
1491 p = SdkToolsPopen(cmd, cwd=root_dir)
1492 # Check to see whether an error was returned without a proper exit
1493 # code (this is the case for the 'no target set or target invalid'
1495 if p.returncode != 0 or p.output.startswith("Error: "):
1496 raise BuildException("Failed to update project at %s" % d, p.output)
1497 # Clean update dirs via ant
1499 logging.info("Cleaning subproject %s" % d)
1500 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1502 return (root_dir, srclibpaths)
1505 # Extend via globbing the paths from a field and return them as a map from
1506 # original path to resulting paths
1507 def getpaths_map(build_dir, globpaths):
1511 full_path = os.path.join(build_dir, p)
1512 full_path = os.path.normpath(full_path)
1513 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1515 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1519 # Extend via globbing the paths from a field and return them as a set
1520 def getpaths(build_dir, globpaths):
1521 paths_map = getpaths_map(build_dir, globpaths)
1523 for k, v in paths_map.items():
1530 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1536 self.path = os.path.join('stats', 'known_apks.txt')
1538 if os.path.isfile(self.path):
1539 with open(self.path, 'r') as f:
1541 t = line.rstrip().split(' ')
1543 self.apks[t[0]] = (t[1], None)
1545 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1546 self.changed = False
1548 def writeifchanged(self):
1549 if not self.changed:
1552 if not os.path.exists('stats'):
1556 for apk, app in self.apks.items():
1558 line = apk + ' ' + appid
1560 line += ' ' + time.strftime('%Y-%m-%d', added)
1563 with open(self.path, 'w') as f:
1564 for line in sorted(lst, key=natural_key):
1565 f.write(line + '\n')
1567 # Record an apk (if it's new, otherwise does nothing)
1568 # Returns the date it was added.
1569 def recordapk(self, apk, app):
1570 if apk not in self.apks:
1571 self.apks[apk] = (app, time.gmtime(time.time()))
1573 _, added = self.apks[apk]
1576 # Look up information - given the 'apkname', returns (app id, date added/None).
1577 # Or returns None for an unknown apk.
1578 def getapp(self, apkname):
1579 if apkname in self.apks:
1580 return self.apks[apkname]
1583 # Get the most recent 'num' apps added to the repo, as a list of package ids
1584 # with the most recent first.
1585 def getlatest(self, num):
1587 for apk, app in self.apks.items():
1591 if apps[appid] > added:
1595 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1596 lst = [app for app, _ in sortedapps]
1601 def isApkDebuggable(apkfile, config):
1602 """Returns True if the given apk file is debuggable
1604 :param apkfile: full path to the apk to check"""
1606 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1608 if p.returncode != 0:
1609 logging.critical("Failed to get apk manifest information")
1611 for line in p.output.splitlines():
1612 if 'android:debuggable' in line and not line.endswith('0x0'):
1619 self.returncode = None
1623 def SdkToolsPopen(commands, cwd=None, output=True):
1625 if cmd not in config:
1626 config[cmd] = find_sdk_tools_cmd(commands[0])
1627 abscmd = config[cmd]
1629 logging.critical("Could not find '%s' on your system" % cmd)
1631 return FDroidPopen([abscmd] + commands[1:],
1632 cwd=cwd, output=output)
1635 def FDroidPopenBytes(commands, cwd=None, output=True, stderr_to_stdout=True):
1637 Run a command and capture the possibly huge output as bytes.
1639 :param commands: command and argument list like in subprocess.Popen
1640 :param cwd: optionally specifies a working directory
1641 :returns: A PopenResult.
1647 cwd = os.path.normpath(cwd)
1648 logging.debug("Directory: %s" % cwd)
1649 logging.debug("> %s" % ' '.join(commands))
1651 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
1652 result = PopenResult()
1655 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
1656 stdout=subprocess.PIPE, stderr=stderr_param)
1657 except OSError as e:
1658 raise BuildException("OSError while trying to execute " +
1659 ' '.join(commands) + ': ' + str(e))
1661 if not stderr_to_stdout and options.verbose:
1662 stderr_queue = Queue()
1663 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
1665 while not stderr_reader.eof():
1666 while not stderr_queue.empty():
1667 line = stderr_queue.get()
1668 sys.stderr.buffer.write(line)
1673 stdout_queue = Queue()
1674 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1677 # Check the queue for output (until there is no more to get)
1678 while not stdout_reader.eof():
1679 while not stdout_queue.empty():
1680 line = stdout_queue.get()
1681 if output and options.verbose:
1682 # Output directly to console
1683 sys.stderr.buffer.write(line)
1689 result.returncode = p.wait()
1690 result.output = buf.getvalue()
1695 def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
1697 Run a command and capture the possibly huge output as a str.
1699 :param commands: command and argument list like in subprocess.Popen
1700 :param cwd: optionally specifies a working directory
1701 :returns: A PopenResult.
1703 result = FDroidPopenBytes(commands, cwd, output, stderr_to_stdout)
1704 result.output = result.output.decode('utf-8')
1708 gradle_comment = re.compile(r'[ ]*//')
1709 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1710 gradle_line_matches = [
1711 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1712 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1713 re.compile(r'.*\.readLine\(.*'),
1717 def remove_signing_keys(build_dir):
1718 for root, dirs, files in os.walk(build_dir):
1719 if 'build.gradle' in files:
1720 path = os.path.join(root, 'build.gradle')
1722 with open(path, "r") as o:
1723 lines = o.readlines()
1729 with open(path, "w") as o:
1730 while i < len(lines):
1733 while line.endswith('\\\n'):
1734 line = line.rstrip('\\\n') + lines[i]
1737 if gradle_comment.match(line):
1742 opened += line.count('{')
1743 opened -= line.count('}')
1746 if gradle_signing_configs.match(line):
1751 if any(s.match(line) for s in gradle_line_matches):
1759 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1762 'project.properties',
1764 'default.properties',
1765 'ant.properties', ]:
1766 if propfile in files:
1767 path = os.path.join(root, propfile)
1769 with open(path, "r") as o:
1770 lines = o.readlines()
1774 with open(path, "w") as o:
1776 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1783 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1786 def reset_env_path():
1787 global env, orig_path
1788 env['PATH'] = orig_path
1791 def add_to_env_path(path):
1793 paths = env['PATH'].split(os.pathsep)
1797 env['PATH'] = os.pathsep.join(paths)
1800 def replace_config_vars(cmd, build):
1802 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1803 # env['ANDROID_NDK'] is set in build_local right before prepare_source
1804 cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
1805 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1806 if build is not None:
1807 cmd = cmd.replace('$$COMMIT$$', build.commit)
1808 cmd = cmd.replace('$$VERSION$$', build.version)
1809 cmd = cmd.replace('$$VERCODE$$', build.vercode)
1813 def place_srclib(root_dir, number, libpath):
1816 relpath = os.path.relpath(libpath, root_dir)
1817 proppath = os.path.join(root_dir, 'project.properties')
1820 if os.path.isfile(proppath):
1821 with open(proppath, "r") as o:
1822 lines = o.readlines()
1824 with open(proppath, "w") as o:
1827 if line.startswith('android.library.reference.%d=' % number):
1828 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1833 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1835 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
1838 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
1839 """Verify that two apks are the same
1841 One of the inputs is signed, the other is unsigned. The signature metadata
1842 is transferred from the signed to the unsigned apk, and then jarsigner is
1843 used to verify that the signature from the signed apk is also varlid for
1845 :param signed_apk: Path to a signed apk file
1846 :param unsigned_apk: Path to an unsigned apk file expected to match it
1847 :param tmp_dir: Path to directory for temporary files
1848 :returns: None if the verification is successful, otherwise a string
1849 describing what went wrong.
1851 with ZipFile(signed_apk) as signed_apk_as_zip:
1852 meta_inf_files = ['META-INF/MANIFEST.MF']
1853 for f in signed_apk_as_zip.namelist():
1854 if apk_sigfile.match(f):
1855 meta_inf_files.append(f)
1856 if len(meta_inf_files) < 3:
1857 return "Signature files missing from {0}".format(signed_apk)
1858 signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
1859 with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
1860 for meta_inf_file in meta_inf_files:
1861 unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
1863 if subprocess.call([config['jarsigner'], '-verify', unsigned_apk]) != 0:
1864 logging.info("...NOT verified - {0}".format(signed_apk))
1865 return compare_apks(signed_apk, unsigned_apk, tmp_dir)
1866 logging.info("...successfully verified")
1869 apk_badchars = re.compile('''[/ :;'"]''')
1872 def compare_apks(apk1, apk2, tmp_dir):
1875 Returns None if the apk content is the same (apart from the signing key),
1876 otherwise a string describing what's different, or what went wrong when
1877 trying to do the comparison.
1880 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
1881 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
1882 for d in [apk1dir, apk2dir]:
1883 if os.path.exists(d):
1886 os.mkdir(os.path.join(d, 'jar-xf'))
1888 if subprocess.call(['jar', 'xf',
1889 os.path.abspath(apk1)],
1890 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
1891 return("Failed to unpack " + apk1)
1892 if subprocess.call(['jar', 'xf',
1893 os.path.abspath(apk2)],
1894 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
1895 return("Failed to unpack " + apk2)
1897 # try to find apktool in the path, if it hasn't been manually configed
1898 if 'apktool' not in config:
1899 tmp = find_command('apktool')
1901 config['apktool'] = tmp
1902 if 'apktool' in config:
1903 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
1905 return("Failed to unpack " + apk1)
1906 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
1908 return("Failed to unpack " + apk2)
1910 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
1911 lines = p.output.splitlines()
1912 if len(lines) != 1 or 'META-INF' not in lines[0]:
1913 meld = find_command('meld')
1914 if meld is not None:
1915 p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
1916 return("Unexpected diff output - " + p.output)
1918 # since everything verifies, delete the comparison to keep cruft down
1919 shutil.rmtree(apk1dir)
1920 shutil.rmtree(apk2dir)
1922 # If we get here, it seems like they're the same!
1926 def find_command(command):
1927 '''find the full path of a command, or None if it can't be found in the PATH'''
1930 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
1932 fpath, fname = os.path.split(command)
1937 for path in os.environ["PATH"].split(os.pathsep):
1938 path = path.strip('"')
1939 exe_file = os.path.join(path, command)
1940 if is_exe(exe_file):
1947 '''generate a random password for when generating keys'''
1948 h = hashlib.sha256()
1949 h.update(os.urandom(16)) # salt
1950 h.update(socket.getfqdn().encode('utf-8'))
1951 passwd = base64.b64encode(h.digest()).strip()
1952 return passwd.decode('utf-8')
1955 def genkeystore(localconfig):
1956 '''Generate a new key with random passwords and add it to new keystore'''
1957 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
1958 keystoredir = os.path.dirname(localconfig['keystore'])
1959 if keystoredir is None or keystoredir == '':
1960 keystoredir = os.path.join(os.getcwd(), keystoredir)
1961 if not os.path.exists(keystoredir):
1962 os.makedirs(keystoredir, mode=0o700)
1964 write_password_file("keystorepass", localconfig['keystorepass'])
1965 write_password_file("keypass", localconfig['keypass'])
1966 p = FDroidPopen([config['keytool'], '-genkey',
1967 '-keystore', localconfig['keystore'],
1968 '-alias', localconfig['repo_keyalias'],
1969 '-keyalg', 'RSA', '-keysize', '4096',
1970 '-sigalg', 'SHA256withRSA',
1971 '-validity', '10000',
1972 '-storepass:file', config['keystorepassfile'],
1973 '-keypass:file', config['keypassfile'],
1974 '-dname', localconfig['keydname']])
1975 # TODO keypass should be sent via stdin
1976 if p.returncode != 0:
1977 raise BuildException("Failed to generate key", p.output)
1978 os.chmod(localconfig['keystore'], 0o0600)
1979 # now show the lovely key that was just generated
1980 p = FDroidPopen([config['keytool'], '-list', '-v',
1981 '-keystore', localconfig['keystore'],
1982 '-alias', localconfig['repo_keyalias'],
1983 '-storepass:file', config['keystorepassfile']])
1984 logging.info(p.output.strip() + '\n\n')
1987 def write_to_config(thisconfig, key, value=None):
1988 '''write a key/value to the local config.py'''
1990 origkey = key + '_orig'
1991 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
1992 with open('config.py', 'r') as f:
1994 pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
1995 repl = '\n' + key + ' = "' + value + '"'
1996 data = re.sub(pattern, repl, data)
1997 # if this key is not in the file, append it
1998 if not re.match('\s*' + key + '\s*=\s*"', data):
2000 # make sure the file ends with a carraige return
2001 if not re.match('\n$', data):
2003 with open('config.py', 'w') as f:
2007 def parse_xml(path):
2008 return XMLElementTree.parse(path).getroot()
2011 def string_is_integer(string):
2019 def get_per_app_repos():
2020 '''per-app repos are dirs named with the packageName of a single app'''
2022 # Android packageNames are Java packages, they may contain uppercase or
2023 # lowercase letters ('A' through 'Z'), numbers, and underscores
2024 # ('_'). However, individual package name parts may only start with
2025 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2026 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2029 for root, dirs, files in os.walk(os.getcwd()):
2031 print('checking', root, 'for', d)
2032 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2033 # standard parts of an fdroid repo, so never packageNames
2036 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):