3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
37 import xml.etree.ElementTree as XMLElementTree
39 from queue import Queue
41 from zipfile import ZipFile
43 import fdroidserver.metadata
44 from .asynchronousfilereader import AsynchronousFileReader
47 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
56 'sdk_path': "$ANDROID_HOME",
59 'r10e': "$ANDROID_NDK",
61 'build_tools': "23.0.2",
66 'accepted_formats': ['txt', 'yaml'],
67 'sync_from_local_copy_dir': False,
68 'per_app_repos': False,
69 'make_current_version_link': True,
70 'current_version_name_source': 'Name',
71 'update_stats': False,
75 'stats_to_carbon': False,
77 'build_server_always': False,
78 'keystore': 'keystore.jks',
79 'smartcardoptions': [],
85 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
86 'repo_name': "My First FDroid Repo Demo",
87 'repo_icon': "fdroid-icon.png",
88 'repo_description': '''
89 This is a repository of apps to be used with FDroid. Applications in this
90 repository are either official binaries built by the original application
91 developers, or are binaries built from source by the admin of f-droid.org
92 using the tools on https://gitlab.com/u/fdroid.
98 def setup_global_opts(parser):
99 parser.add_argument("-v", "--verbose", action="store_true", default=False,
100 help="Spew out even more information than normal")
101 parser.add_argument("-q", "--quiet", action="store_true", default=False,
102 help="Restrict output to warnings and errors")
105 def fill_config_defaults(thisconfig):
106 for k, v in default_config.items():
107 if k not in thisconfig:
110 # Expand paths (~users and $vars)
111 def expand_path(path):
115 path = os.path.expanduser(path)
116 path = os.path.expandvars(path)
121 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
126 thisconfig[k + '_orig'] = v
128 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
129 if thisconfig['java_paths'] is None:
130 thisconfig['java_paths'] = dict()
132 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
133 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
134 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
135 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
136 if os.getenv('JAVA_HOME') is not None:
137 pathlist += os.getenv('JAVA_HOME')
138 if os.getenv('PROGRAMFILES') is not None:
139 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
140 for d in sorted(pathlist):
141 if os.path.islink(d):
143 j = os.path.basename(d)
144 # the last one found will be the canonical one, so order appropriately
146 r'^1\.([6-9])\.0\.jdk$', # OSX
147 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
148 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
149 r'^jdk([6-9])-openjdk$', # Arch
150 r'^java-([6-9])-openjdk$', # Arch
151 r'^java-([6-9])-jdk$', # Arch (oracle)
152 r'^java-1\.([6-9])\.0-.*$', # RedHat
153 r'^java-([6-9])-oracle$', # Debian WebUpd8
154 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
155 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
157 m = re.match(regex, j)
160 osxhome = os.path.join(d, 'Contents', 'Home')
161 if os.path.exists(osxhome):
162 thisconfig['java_paths'][m.group(1)] = osxhome
164 thisconfig['java_paths'][m.group(1)] = d
166 for java_version in ('7', '8', '9'):
167 if java_version not in thisconfig['java_paths']:
169 java_home = thisconfig['java_paths'][java_version]
170 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
171 if os.path.exists(jarsigner):
172 thisconfig['jarsigner'] = jarsigner
173 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
174 break # Java7 is preferred, so quit if found
176 for k in ['ndk_paths', 'java_paths']:
182 thisconfig[k][k2] = exp
183 thisconfig[k][k2 + '_orig'] = v
186 def regsub_file(pattern, repl, path):
187 with open(path, 'r') as f:
189 text = re.sub(pattern, repl, text)
190 with open(path, 'w') as f:
194 def read_config(opts, config_file='config.py'):
195 """Read the repository config
197 The config is read from config_file, which is in the current directory when
198 any of the repo management commands are used.
200 global config, options, orig_path
202 if config is not None:
204 if not os.path.isfile(config_file):
205 logging.critical("Missing config file - is this a repo directory?")
212 logging.debug("Reading %s" % config_file)
213 with io.open(config_file, "rb") as f:
214 code = compile(f.read(), config_file, 'exec')
215 exec(code, None, config)
217 # smartcardoptions must be a list since its command line args for Popen
218 if 'smartcardoptions' in config:
219 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
220 elif 'keystore' in config and config['keystore'] == 'NONE':
221 # keystore='NONE' means use smartcard, these are required defaults
222 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
223 'SunPKCS11-OpenSC', '-providerClass',
224 'sun.security.pkcs11.SunPKCS11',
225 '-providerArg', 'opensc-fdroid.cfg']
227 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
228 st = os.stat(config_file)
229 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
230 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
232 fill_config_defaults(config)
234 # There is no standard, so just set up the most common environment
237 orig_path = env['PATH']
238 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
239 env[n] = config['sdk_path']
241 for k, v in config['java_paths'].items():
242 env['JAVA%s_HOME' % k] = v
244 for k in ["keystorepass", "keypass"]:
246 write_password_file(k)
248 for k in ["repo_description", "archive_description"]:
250 config[k] = clean_description(config[k])
252 if 'serverwebroot' in config:
253 if isinstance(config['serverwebroot'], str):
254 roots = [config['serverwebroot']]
255 elif all(isinstance(item, str) for item in config['serverwebroot']):
256 roots = config['serverwebroot']
258 raise TypeError('only accepts strings, lists, and tuples')
260 for rootstr in roots:
261 # since this is used with rsync, where trailing slashes have
262 # meaning, ensure there is always a trailing slash
263 if rootstr[-1] != '/':
265 rootlist.append(rootstr.replace('//', '/'))
266 config['serverwebroot'] = rootlist
271 def get_ndk_path(version):
272 if config is None or 'ndk_paths' not in config:
273 ndk_path = os.getenv('ANDROID_NDK_HOME')
275 logging.error('No NDK found! Either set ANDROID_NDK_HOME or add ndk_path to your config.py')
279 version = 'r10e' # falls back to latest
280 paths = config['ndk_paths']
281 if version not in paths:
283 return paths[version] or ''
286 def find_sdk_tools_cmd(cmd):
287 '''find a working path to a tool from the Android SDK'''
290 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
291 # try to find a working path to this command, in all the recent possible paths
292 if 'build_tools' in config:
293 build_tools = os.path.join(config['sdk_path'], 'build-tools')
294 # if 'build_tools' was manually set and exists, check only that one
295 configed_build_tools = os.path.join(build_tools, config['build_tools'])
296 if os.path.exists(configed_build_tools):
297 tooldirs.append(configed_build_tools)
299 # no configed version, so hunt known paths for it
300 for f in sorted(os.listdir(build_tools), reverse=True):
301 if os.path.isdir(os.path.join(build_tools, f)):
302 tooldirs.append(os.path.join(build_tools, f))
303 tooldirs.append(build_tools)
304 sdk_tools = os.path.join(config['sdk_path'], 'tools')
305 if os.path.exists(sdk_tools):
306 tooldirs.append(sdk_tools)
307 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
308 if os.path.exists(sdk_platform_tools):
309 tooldirs.append(sdk_platform_tools)
310 tooldirs.append('/usr/bin')
312 if os.path.isfile(os.path.join(d, cmd)):
313 return os.path.join(d, cmd)
314 # did not find the command, exit with error message
315 ensure_build_tools_exists(config)
318 def test_sdk_exists(thisconfig):
319 if 'sdk_path' not in thisconfig:
320 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
323 logging.error("'sdk_path' not set in config.py!")
325 if thisconfig['sdk_path'] == default_config['sdk_path']:
326 logging.error('No Android SDK found!')
327 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
328 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
330 if not os.path.exists(thisconfig['sdk_path']):
331 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
333 if not os.path.isdir(thisconfig['sdk_path']):
334 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
336 for d in ['build-tools', 'platform-tools', 'tools']:
337 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
338 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
339 thisconfig['sdk_path'], d))
344 def ensure_build_tools_exists(thisconfig):
345 if not test_sdk_exists(thisconfig):
347 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
348 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
349 if not os.path.isdir(versioned_build_tools):
350 logging.critical('Android Build Tools path "'
351 + versioned_build_tools + '" does not exist!')
355 def write_password_file(pwtype, password=None):
357 writes out passwords to a protected file instead of passing passwords as
358 command line argments
360 filename = '.fdroid.' + pwtype + '.txt'
361 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
363 os.write(fd, config[pwtype].encode('utf-8'))
365 os.write(fd, password.encode('utf-8'))
367 config[pwtype + 'file'] = filename
370 # Given the arguments in the form of multiple appid:[vc] strings, this returns
371 # a dictionary with the set of vercodes specified for each package.
372 def read_pkg_args(args, allow_vercodes=False):
379 if allow_vercodes and ':' in p:
380 package, vercode = p.split(':')
382 package, vercode = p, None
383 if package not in vercodes:
384 vercodes[package] = [vercode] if vercode else []
386 elif vercode and vercode not in vercodes[package]:
387 vercodes[package] += [vercode] if vercode else []
392 # On top of what read_pkg_args does, this returns the whole app metadata, but
393 # limiting the builds list to the builds matching the vercodes specified.
394 def read_app_args(args, allapps, allow_vercodes=False):
396 vercodes = read_pkg_args(args, allow_vercodes)
402 for appid, app in allapps.items():
403 if appid in vercodes:
406 if len(apps) != len(vercodes):
409 logging.critical("No such package: %s" % p)
410 raise FDroidException("Found invalid app ids in arguments")
412 raise FDroidException("No packages specified")
415 for appid, app in apps.items():
419 app.builds = [b for b in app.builds if b.vercode in vc]
420 if len(app.builds) != len(vercodes[appid]):
422 allvcs = [b.vercode for b in app.builds]
423 for v in vercodes[appid]:
425 logging.critical("No such vercode %s for app %s" % (v, appid))
428 raise FDroidException("Found invalid vercodes for some apps")
433 def get_extension(filename):
434 base, ext = os.path.splitext(filename)
437 return base, ext.lower()[1:]
440 def has_extension(filename, ext):
441 _, f_ext = get_extension(filename)
445 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
448 def clean_description(description):
449 'Remove unneeded newlines and spaces from a block of description text'
451 # this is split up by paragraph to make removing the newlines easier
452 for paragraph in re.split(r'\n\n', description):
453 paragraph = re.sub('\r', '', paragraph)
454 paragraph = re.sub('\n', ' ', paragraph)
455 paragraph = re.sub(' {2,}', ' ', paragraph)
456 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
457 returnstring += paragraph + '\n\n'
458 return returnstring.rstrip('\n')
461 def apknameinfo(filename):
462 filename = os.path.basename(filename)
463 m = apk_regex.match(filename)
465 result = (m.group(1), m.group(2))
466 except AttributeError:
467 raise FDroidException("Invalid apk name: %s" % filename)
471 def getapkname(app, build):
472 return "%s_%s.apk" % (app.id, build.vercode)
475 def getsrcname(app, build):
476 return "%s_%s_src.tar.gz" % (app.id, build.vercode)
488 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
491 def getvcs(vcstype, remote, local):
493 return vcs_git(remote, local)
494 if vcstype == 'git-svn':
495 return vcs_gitsvn(remote, local)
497 return vcs_hg(remote, local)
499 return vcs_bzr(remote, local)
500 if vcstype == 'srclib':
501 if local != os.path.join('build', 'srclib', remote):
502 raise VCSException("Error: srclib paths are hard-coded!")
503 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
505 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
506 raise VCSException("Invalid vcs type " + vcstype)
509 def getsrclibvcs(name):
510 if name not in fdroidserver.metadata.srclibs:
511 raise VCSException("Missing srclib " + name)
512 return fdroidserver.metadata.srclibs[name]['Repo Type']
517 def __init__(self, remote, local):
519 # svn, git-svn and bzr may require auth
521 if self.repotype() in ('git-svn', 'bzr'):
523 if self.repotype == 'git-svn':
524 raise VCSException("Authentication is not supported for git-svn")
525 self.username, remote = remote.split('@')
526 if ':' not in self.username:
527 raise VCSException("Password required with username")
528 self.username, self.password = self.username.split(':')
532 self.clone_failed = False
533 self.refreshed = False
539 # Take the local repository to a clean version of the given revision, which
540 # is specificed in the VCS's native format. Beforehand, the repository can
541 # be dirty, or even non-existent. If the repository does already exist
542 # locally, it will be updated from the origin, but only once in the
543 # lifetime of the vcs object.
544 # None is acceptable for 'rev' if you know you are cloning a clean copy of
545 # the repo - otherwise it must specify a valid revision.
546 def gotorevision(self, rev, refresh=True):
548 if self.clone_failed:
549 raise VCSException("Downloading the repository already failed once, not trying again.")
551 # The .fdroidvcs-id file for a repo tells us what VCS type
552 # and remote that directory was created from, allowing us to drop it
553 # automatically if either of those things changes.
554 fdpath = os.path.join(self.local, '..',
555 '.fdroidvcs-' + os.path.basename(self.local))
556 fdpath = os.path.normpath(fdpath)
557 cdata = self.repotype() + ' ' + self.remote
560 if os.path.exists(self.local):
561 if os.path.exists(fdpath):
562 with open(fdpath, 'r') as f:
563 fsdata = f.read().strip()
568 logging.info("Repository details for %s changed - deleting" % (
572 logging.info("Repository details for %s missing - deleting" % (
575 shutil.rmtree(self.local)
579 self.refreshed = True
582 self.gotorevisionx(rev)
583 except FDroidException as e:
586 # If necessary, write the .fdroidvcs file.
587 if writeback and not self.clone_failed:
588 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
589 with open(fdpath, 'w+') as f:
595 # Derived classes need to implement this. It's called once basic checking
596 # has been performend.
597 def gotorevisionx(self, rev):
598 raise VCSException("This VCS type doesn't define gotorevisionx")
600 # Initialise and update submodules
601 def initsubmodules(self):
602 raise VCSException('Submodules not supported for this vcs type')
604 # Get a list of all known tags
606 if not self._gettags:
607 raise VCSException('gettags not supported for this vcs type')
609 for tag in self._gettags():
610 if re.match('[-A-Za-z0-9_. /]+$', tag):
614 # Get a list of all the known tags, sorted from newest to oldest
615 def latesttags(self):
616 raise VCSException('latesttags not supported for this vcs type')
618 # Get current commit reference (hash, revision, etc)
620 raise VCSException('getref not supported for this vcs type')
622 # Returns the srclib (name, path) used in setting up the current
633 # If the local directory exists, but is somehow not a git repository, git
634 # will traverse up the directory tree until it finds one that is (i.e.
635 # fdroidserver) and then we'll proceed to destroy it! This is called as
638 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
639 result = p.output.rstrip()
640 if not result.endswith(self.local):
641 raise VCSException('Repository mismatch')
643 def gotorevisionx(self, rev):
644 if not os.path.exists(self.local):
646 p = FDroidPopen(['git', 'clone', self.remote, self.local])
647 if p.returncode != 0:
648 self.clone_failed = True
649 raise VCSException("Git clone failed", p.output)
653 # Discard any working tree changes
654 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
655 'git', 'reset', '--hard'], cwd=self.local, output=False)
656 if p.returncode != 0:
657 raise VCSException("Git reset failed", p.output)
658 # Remove untracked files now, in case they're tracked in the target
659 # revision (it happens!)
660 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
661 'git', 'clean', '-dffx'], cwd=self.local, output=False)
662 if p.returncode != 0:
663 raise VCSException("Git clean failed", p.output)
664 if not self.refreshed:
665 # Get latest commits and tags from remote
666 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
667 if p.returncode != 0:
668 raise VCSException("Git fetch failed", p.output)
669 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
670 if p.returncode != 0:
671 raise VCSException("Git fetch failed", p.output)
672 # Recreate origin/HEAD as git clone would do it, in case it disappeared
673 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
674 if p.returncode != 0:
675 lines = p.output.splitlines()
676 if 'Multiple remote HEAD branches' not in lines[0]:
677 raise VCSException("Git remote set-head failed", p.output)
678 branch = lines[1].split(' ')[-1]
679 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
680 if p2.returncode != 0:
681 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
682 self.refreshed = True
683 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
684 # a github repo. Most of the time this is the same as origin/master.
685 rev = rev or 'origin/HEAD'
686 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
687 if p.returncode != 0:
688 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
689 # Get rid of any uncontrolled files left behind
690 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
691 if p.returncode != 0:
692 raise VCSException("Git clean failed", p.output)
694 def initsubmodules(self):
696 submfile = os.path.join(self.local, '.gitmodules')
697 if not os.path.isfile(submfile):
698 raise VCSException("No git submodules available")
700 # fix submodules not accessible without an account and public key auth
701 with open(submfile, 'r') as f:
702 lines = f.readlines()
703 with open(submfile, 'w') as f:
705 if 'git@github.com' in line:
706 line = line.replace('git@github.com:', 'https://github.com/')
707 if 'git@gitlab.com' in line:
708 line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
711 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
712 if p.returncode != 0:
713 raise VCSException("Git submodule sync failed", p.output)
714 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
715 if p.returncode != 0:
716 raise VCSException("Git submodule update failed", p.output)
720 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
721 return p.output.splitlines()
723 tag_format = re.compile(r'tag: ([^),]*)')
725 def latesttags(self):
727 p = FDroidPopen(['git', 'log', '--tags',
728 '--simplify-by-decoration', '--pretty=format:%d'],
729 cwd=self.local, output=False)
731 for line in p.output.splitlines():
732 for tag in self.tag_format.findall(line):
737 class vcs_gitsvn(vcs):
742 # If the local directory exists, but is somehow not a git repository, git
743 # will traverse up the directory tree until it finds one that is (i.e.
744 # fdroidserver) and then we'll proceed to destory it! This is called as
747 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
748 result = p.output.rstrip()
749 if not result.endswith(self.local):
750 raise VCSException('Repository mismatch')
752 def gotorevisionx(self, rev):
753 if not os.path.exists(self.local):
755 gitsvn_args = ['git', 'svn', 'clone']
756 if ';' in self.remote:
757 remote_split = self.remote.split(';')
758 for i in remote_split[1:]:
759 if i.startswith('trunk='):
760 gitsvn_args.extend(['-T', i[6:]])
761 elif i.startswith('tags='):
762 gitsvn_args.extend(['-t', i[5:]])
763 elif i.startswith('branches='):
764 gitsvn_args.extend(['-b', i[9:]])
765 gitsvn_args.extend([remote_split[0], self.local])
766 p = FDroidPopen(gitsvn_args, output=False)
767 if p.returncode != 0:
768 self.clone_failed = True
769 raise VCSException("Git svn clone failed", p.output)
771 gitsvn_args.extend([self.remote, self.local])
772 p = FDroidPopen(gitsvn_args, output=False)
773 if p.returncode != 0:
774 self.clone_failed = True
775 raise VCSException("Git svn clone failed", p.output)
779 # Discard any working tree changes
780 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
781 if p.returncode != 0:
782 raise VCSException("Git reset failed", p.output)
783 # Remove untracked files now, in case they're tracked in the target
784 # revision (it happens!)
785 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
786 if p.returncode != 0:
787 raise VCSException("Git clean failed", p.output)
788 if not self.refreshed:
789 # Get new commits, branches and tags from repo
790 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
791 if p.returncode != 0:
792 raise VCSException("Git svn fetch failed")
793 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
794 if p.returncode != 0:
795 raise VCSException("Git svn rebase failed", p.output)
796 self.refreshed = True
798 rev = rev or 'master'
800 nospaces_rev = rev.replace(' ', '%20')
801 # Try finding a svn tag
802 for treeish in ['origin/', '']:
803 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
804 if p.returncode == 0:
806 if p.returncode != 0:
807 # No tag found, normal svn rev translation
808 # Translate svn rev into git format
809 rev_split = rev.split('/')
812 for treeish in ['origin/', '']:
813 if len(rev_split) > 1:
814 treeish += rev_split[0]
815 svn_rev = rev_split[1]
818 # if no branch is specified, then assume trunk (i.e. 'master' branch):
822 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
824 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
825 git_rev = p.output.rstrip()
827 if p.returncode == 0 and git_rev:
830 if p.returncode != 0 or not git_rev:
831 # Try a plain git checkout as a last resort
832 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
833 if p.returncode != 0:
834 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
836 # Check out the git rev equivalent to the svn rev
837 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
838 if p.returncode != 0:
839 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
841 # Get rid of any uncontrolled files left behind
842 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
843 if p.returncode != 0:
844 raise VCSException("Git clean failed", p.output)
848 for treeish in ['origin/', '']:
849 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
855 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
856 if p.returncode != 0:
858 return p.output.strip()
866 def gotorevisionx(self, rev):
867 if not os.path.exists(self.local):
868 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
869 if p.returncode != 0:
870 self.clone_failed = True
871 raise VCSException("Hg clone failed", p.output)
873 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
874 if p.returncode != 0:
875 raise VCSException("Hg status failed", p.output)
876 for line in p.output.splitlines():
877 if not line.startswith('? '):
878 raise VCSException("Unexpected output from hg status -uS: " + line)
879 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
880 if not self.refreshed:
881 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
882 if p.returncode != 0:
883 raise VCSException("Hg pull failed", p.output)
884 self.refreshed = True
886 rev = rev or 'default'
889 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
890 if p.returncode != 0:
891 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
892 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
893 # Also delete untracked files, we have to enable purge extension for that:
894 if "'purge' is provided by the following extension" in p.output:
895 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
896 myfile.write("\n[extensions]\nhgext.purge=\n")
897 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
898 if p.returncode != 0:
899 raise VCSException("HG purge failed", p.output)
900 elif p.returncode != 0:
901 raise VCSException("HG purge failed", p.output)
904 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
905 return p.output.splitlines()[1:]
913 def gotorevisionx(self, rev):
914 if not os.path.exists(self.local):
915 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
916 if p.returncode != 0:
917 self.clone_failed = True
918 raise VCSException("Bzr branch failed", p.output)
920 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
921 if p.returncode != 0:
922 raise VCSException("Bzr revert failed", p.output)
923 if not self.refreshed:
924 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
925 if p.returncode != 0:
926 raise VCSException("Bzr update failed", p.output)
927 self.refreshed = True
929 revargs = list(['-r', rev] if rev else [])
930 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
931 if p.returncode != 0:
932 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
935 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
936 return [tag.split(' ')[0].strip() for tag in
937 p.output.splitlines()]
940 def unescape_string(string):
943 if string[0] == '"' and string[-1] == '"':
946 return string.replace("\\'", "'")
949 def retrieve_string(app_dir, string, xmlfiles=None):
951 if not string.startswith('@string/'):
952 return unescape_string(string)
957 os.path.join(app_dir, 'res'),
958 os.path.join(app_dir, 'src', 'main', 'res'),
960 for r, d, f in os.walk(res_dir):
961 if os.path.basename(r) == 'values':
962 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
964 name = string[len('@string/'):]
966 def element_content(element):
967 if element.text is None:
969 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
970 return s.decode('utf-8').strip()
972 for path in xmlfiles:
973 if not os.path.isfile(path):
975 xml = parse_xml(path)
976 element = xml.find('string[@name="' + name + '"]')
977 if element is not None:
978 content = element_content(element)
979 return retrieve_string(app_dir, content, xmlfiles)
984 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
985 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
988 # Return list of existing files that will be used to find the highest vercode
989 def manifest_paths(app_dir, flavours):
991 possible_manifests = \
992 [os.path.join(app_dir, 'AndroidManifest.xml'),
993 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
994 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
995 os.path.join(app_dir, 'build.gradle')]
997 for flavour in flavours:
1000 possible_manifests.append(
1001 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1003 return [path for path in possible_manifests if os.path.isfile(path)]
1006 # Retrieve the package name. Returns the name, or None if not found.
1007 def fetch_real_name(app_dir, flavours):
1008 for path in manifest_paths(app_dir, flavours):
1009 if not has_extension(path, 'xml') or not os.path.isfile(path):
1011 logging.debug("fetch_real_name: Checking manifest at " + path)
1012 xml = parse_xml(path)
1013 app = xml.find('application')
1016 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1018 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1019 result = retrieve_string_singleline(app_dir, label)
1021 result = result.strip()
1026 def get_library_references(root_dir):
1028 proppath = os.path.join(root_dir, 'project.properties')
1029 if not os.path.isfile(proppath):
1031 with open(proppath, 'r') as f:
1033 if not line.startswith('android.library.reference.'):
1035 path = line.split('=')[1].strip()
1036 relpath = os.path.join(root_dir, path)
1037 if not os.path.isdir(relpath):
1039 logging.debug("Found subproject at %s" % path)
1040 libraries.append(path)
1044 def ant_subprojects(root_dir):
1045 subprojects = get_library_references(root_dir)
1046 for subpath in subprojects:
1047 subrelpath = os.path.join(root_dir, subpath)
1048 for p in get_library_references(subrelpath):
1049 relp = os.path.normpath(os.path.join(subpath, p))
1050 if relp not in subprojects:
1051 subprojects.insert(0, relp)
1055 def remove_debuggable_flags(root_dir):
1056 # Remove forced debuggable flags
1057 logging.debug("Removing debuggable flags from %s" % root_dir)
1058 for root, dirs, files in os.walk(root_dir):
1059 if 'AndroidManifest.xml' in files:
1060 regsub_file(r'android:debuggable="[^"]*"',
1062 os.path.join(root, 'AndroidManifest.xml'))
1065 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
1066 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1067 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1070 def app_matches_packagename(app, package):
1073 appid = app.UpdateCheckName or app.id
1074 if appid is None or appid == "Ignore":
1076 return appid == package
1079 # Extract some information from the AndroidManifest.xml at the given path.
1080 # Returns (version, vercode, package), any or all of which might be None.
1081 # All values returned are strings.
1082 def parse_androidmanifests(paths, app):
1084 ignoreversions = app.UpdateCheckIgnore
1085 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1088 return (None, None, None)
1096 if not os.path.isfile(path):
1099 logging.debug("Parsing manifest at {0}".format(path))
1100 gradle = has_extension(path, 'gradle')
1106 with open(path, 'r') as f:
1108 if gradle_comment.match(line):
1110 # Grab first occurence of each to avoid running into
1111 # alternative flavours and builds.
1113 matches = psearch_g(line)
1115 s = matches.group(2)
1116 if app_matches_packagename(app, s):
1119 matches = vnsearch_g(line)
1121 version = matches.group(2)
1123 matches = vcsearch_g(line)
1125 vercode = matches.group(1)
1128 xml = parse_xml(path)
1129 if "package" in xml.attrib:
1130 s = xml.attrib["package"]
1131 if app_matches_packagename(app, s):
1133 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1134 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1135 base_dir = os.path.dirname(path)
1136 version = retrieve_string_singleline(base_dir, version)
1137 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1138 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1139 if string_is_integer(a):
1142 logging.warning("Problem with xml at {0}".format(path))
1144 # Remember package name, may be defined separately from version+vercode
1146 package = max_package
1148 logging.debug("..got package={0}, version={1}, vercode={2}"
1149 .format(package, version, vercode))
1151 # Always grab the package name and version name in case they are not
1152 # together with the highest version code
1153 if max_package is None and package is not None:
1154 max_package = package
1155 if max_version is None and version is not None:
1156 max_version = version
1158 if max_vercode is None or (vercode is not None and vercode > max_vercode):
1159 if not ignoresearch or not ignoresearch(version):
1160 if version is not None:
1161 max_version = version
1162 if vercode is not None:
1163 max_vercode = vercode
1164 if package is not None:
1165 max_package = package
1167 max_version = "Ignore"
1169 if max_version is None:
1170 max_version = "Unknown"
1172 if max_package and not is_valid_package_name(max_package):
1173 raise FDroidException("Invalid package name {0}".format(max_package))
1175 return (max_version, max_vercode, max_package)
1178 def is_valid_package_name(name):
1179 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1182 class FDroidException(Exception):
1184 def __init__(self, value, detail=None):
1186 self.detail = detail
1188 def shortened_detail(self):
1189 if len(self.detail) < 16000:
1191 return '[...]\n' + self.detail[-16000:]
1193 def get_wikitext(self):
1194 ret = repr(self.value) + "\n"
1197 ret += "<pre>\n" + self.shortened_detail() + "</pre>\n"
1203 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1207 class VCSException(FDroidException):
1211 class BuildException(FDroidException):
1215 # Get the specified source library.
1216 # Returns the path to it. Normally this is the path to be used when referencing
1217 # it, which may be a subdirectory of the actual project. If you want the base
1218 # directory of the project, pass 'basepath=True'.
1219 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1220 raw=False, prepare=True, preponly=False, refresh=True):
1228 name, ref = spec.split('@')
1230 number, name = name.split(':', 1)
1232 name, subdir = name.split('/', 1)
1234 if name not in fdroidserver.metadata.srclibs:
1235 raise VCSException('srclib ' + name + ' not found.')
1237 srclib = fdroidserver.metadata.srclibs[name]
1239 sdir = os.path.join(srclib_dir, name)
1242 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1243 vcs.srclib = (name, number, sdir)
1245 vcs.gotorevision(ref, refresh)
1252 libdir = os.path.join(sdir, subdir)
1253 elif srclib["Subdir"]:
1254 for subdir in srclib["Subdir"]:
1255 libdir_candidate = os.path.join(sdir, subdir)
1256 if os.path.exists(libdir_candidate):
1257 libdir = libdir_candidate
1263 remove_signing_keys(sdir)
1264 remove_debuggable_flags(sdir)
1268 if srclib["Prepare"]:
1269 cmd = replace_config_vars(srclib["Prepare"], None)
1271 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1272 if p.returncode != 0:
1273 raise BuildException("Error running prepare command for srclib %s"
1279 return (name, number, libdir)
1281 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1284 # Prepare the source code for a particular build
1285 # 'vcs' - the appropriate vcs object for the application
1286 # 'app' - the application details from the metadata
1287 # 'build' - the build details from the metadata
1288 # 'build_dir' - the path to the build directory, usually
1290 # 'srclib_dir' - the path to the source libraries directory, usually
1292 # 'extlib_dir' - the path to the external libraries directory, usually
1294 # Returns the (root, srclibpaths) where:
1295 # 'root' is the root directory, which may be the same as 'build_dir' or may
1296 # be a subdirectory of it.
1297 # 'srclibpaths' is information on the srclibs being used
1298 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1300 # Optionally, the actual app source can be in a subdirectory
1302 root_dir = os.path.join(build_dir, build.subdir)
1304 root_dir = build_dir
1306 # Get a working copy of the right revision
1307 logging.info("Getting source for revision " + build.commit)
1308 vcs.gotorevision(build.commit, refresh)
1310 # Initialise submodules if required
1311 if build.submodules:
1312 logging.info("Initialising submodules")
1313 vcs.initsubmodules()
1315 # Check that a subdir (if we're using one) exists. This has to happen
1316 # after the checkout, since it might not exist elsewhere
1317 if not os.path.exists(root_dir):
1318 raise BuildException('Missing subdir ' + root_dir)
1320 # Run an init command if one is required
1322 cmd = replace_config_vars(build.init, build)
1323 logging.info("Running 'init' commands in %s" % root_dir)
1325 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1326 if p.returncode != 0:
1327 raise BuildException("Error running init command for %s:%s" %
1328 (app.id, build.version), p.output)
1330 # Apply patches if any
1332 logging.info("Applying patches")
1333 for patch in build.patch:
1334 patch = patch.strip()
1335 logging.info("Applying " + patch)
1336 patch_path = os.path.join('metadata', app.id, patch)
1337 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1338 if p.returncode != 0:
1339 raise BuildException("Failed to apply patch %s" % patch_path)
1341 # Get required source libraries
1344 logging.info("Collecting source libraries")
1345 for lib in build.srclibs:
1346 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
1348 for name, number, libpath in srclibpaths:
1349 place_srclib(root_dir, int(number) if number else None, libpath)
1351 basesrclib = vcs.getsrclib()
1352 # If one was used for the main source, add that too.
1354 srclibpaths.append(basesrclib)
1356 # Update the local.properties file
1357 localprops = [os.path.join(build_dir, 'local.properties')]
1359 parts = build.subdir.split(os.sep)
1362 cur = os.path.join(cur, d)
1363 localprops += [os.path.join(cur, 'local.properties')]
1364 for path in localprops:
1366 if os.path.isfile(path):
1367 logging.info("Updating local.properties file at %s" % path)
1368 with open(path, 'r') as f:
1372 logging.info("Creating local.properties file at %s" % path)
1373 # Fix old-fashioned 'sdk-location' by copying
1374 # from sdk.dir, if necessary
1376 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1377 re.S | re.M).group(1)
1378 props += "sdk-location=%s\n" % sdkloc
1380 props += "sdk.dir=%s\n" % config['sdk_path']
1381 props += "sdk-location=%s\n" % config['sdk_path']
1382 ndk_path = build.ndk_path()
1385 props += "ndk.dir=%s\n" % ndk_path
1386 props += "ndk-location=%s\n" % ndk_path
1387 # Add java.encoding if necessary
1389 props += "java.encoding=%s\n" % build.encoding
1390 with open(path, 'w') as f:
1394 if build.build_method() == 'gradle':
1395 flavours = build.gradle
1398 n = build.target.split('-')[1]
1399 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1400 r'compileSdkVersion %s' % n,
1401 os.path.join(root_dir, 'build.gradle'))
1403 # Remove forced debuggable flags
1404 remove_debuggable_flags(root_dir)
1406 # Insert version code and number into the manifest if necessary
1407 if build.forceversion:
1408 logging.info("Changing the version name")
1409 for path in manifest_paths(root_dir, flavours):
1410 if not os.path.isfile(path):
1412 if has_extension(path, 'xml'):
1413 regsub_file(r'android:versionName="[^"]*"',
1414 r'android:versionName="%s"' % build.version,
1416 elif has_extension(path, 'gradle'):
1417 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1418 r"""\1versionName '%s'""" % build.version,
1421 if build.forcevercode:
1422 logging.info("Changing the version code")
1423 for path in manifest_paths(root_dir, flavours):
1424 if not os.path.isfile(path):
1426 if has_extension(path, 'xml'):
1427 regsub_file(r'android:versionCode="[^"]*"',
1428 r'android:versionCode="%s"' % build.vercode,
1430 elif has_extension(path, 'gradle'):
1431 regsub_file(r'versionCode[ =]+[0-9]+',
1432 r'versionCode %s' % build.vercode,
1435 # Delete unwanted files
1437 logging.info("Removing specified files")
1438 for part in getpaths(build_dir, build.rm):
1439 dest = os.path.join(build_dir, part)
1440 logging.info("Removing {0}".format(part))
1441 if os.path.lexists(dest):
1442 if os.path.islink(dest):
1443 FDroidPopen(['unlink', dest], output=False)
1445 FDroidPopen(['rm', '-rf', dest], output=False)
1447 logging.info("...but it didn't exist")
1449 remove_signing_keys(build_dir)
1451 # Add required external libraries
1453 logging.info("Collecting prebuilt libraries")
1454 libsdir = os.path.join(root_dir, 'libs')
1455 if not os.path.exists(libsdir):
1457 for lib in build.extlibs:
1459 logging.info("...installing extlib {0}".format(lib))
1460 libf = os.path.basename(lib)
1461 libsrc = os.path.join(extlib_dir, lib)
1462 if not os.path.exists(libsrc):
1463 raise BuildException("Missing extlib file {0}".format(libsrc))
1464 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1466 # Run a pre-build command if one is required
1468 logging.info("Running 'prebuild' commands in %s" % root_dir)
1470 cmd = replace_config_vars(build.prebuild, build)
1472 # Substitute source library paths into prebuild commands
1473 for name, number, libpath in srclibpaths:
1474 libpath = os.path.relpath(libpath, root_dir)
1475 cmd = cmd.replace('$$' + name + '$$', libpath)
1477 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1478 if p.returncode != 0:
1479 raise BuildException("Error running prebuild command for %s:%s" %
1480 (app.id, build.version), p.output)
1482 # Generate (or update) the ant build file, build.xml...
1483 if build.build_method() == 'ant' and build.update != ['no']:
1484 parms = ['android', 'update', 'lib-project']
1485 lparms = ['android', 'update', 'project']
1488 parms += ['-t', build.target]
1489 lparms += ['-t', build.target]
1491 update_dirs = build.update
1493 update_dirs = ant_subprojects(root_dir) + ['.']
1495 for d in update_dirs:
1496 subdir = os.path.join(root_dir, d)
1498 logging.debug("Updating main project")
1499 cmd = parms + ['-p', d]
1501 logging.debug("Updating subproject %s" % d)
1502 cmd = lparms + ['-p', d]
1503 p = SdkToolsPopen(cmd, cwd=root_dir)
1504 # Check to see whether an error was returned without a proper exit
1505 # code (this is the case for the 'no target set or target invalid'
1507 if p.returncode != 0 or p.output.startswith("Error: "):
1508 raise BuildException("Failed to update project at %s" % d, p.output)
1509 # Clean update dirs via ant
1511 logging.info("Cleaning subproject %s" % d)
1512 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1514 return (root_dir, srclibpaths)
1517 # Extend via globbing the paths from a field and return them as a map from
1518 # original path to resulting paths
1519 def getpaths_map(build_dir, globpaths):
1523 full_path = os.path.join(build_dir, p)
1524 full_path = os.path.normpath(full_path)
1525 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1527 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1531 # Extend via globbing the paths from a field and return them as a set
1532 def getpaths(build_dir, globpaths):
1533 paths_map = getpaths_map(build_dir, globpaths)
1535 for k, v in paths_map.items():
1542 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1548 self.path = os.path.join('stats', 'known_apks.txt')
1550 if os.path.isfile(self.path):
1551 with open(self.path, 'r') as f:
1553 t = line.rstrip().split(' ')
1555 self.apks[t[0]] = (t[1], None)
1557 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1558 self.changed = False
1560 def writeifchanged(self):
1561 if not self.changed:
1564 if not os.path.exists('stats'):
1568 for apk, app in self.apks.items():
1570 line = apk + ' ' + appid
1572 line += ' ' + time.strftime('%Y-%m-%d', added)
1575 with open(self.path, 'w') as f:
1576 for line in sorted(lst, key=natural_key):
1577 f.write(line + '\n')
1579 # Record an apk (if it's new, otherwise does nothing)
1580 # Returns the date it was added.
1581 def recordapk(self, apk, app):
1582 if apk not in self.apks:
1583 self.apks[apk] = (app, time.gmtime(time.time()))
1585 _, added = self.apks[apk]
1588 # Look up information - given the 'apkname', returns (app id, date added/None).
1589 # Or returns None for an unknown apk.
1590 def getapp(self, apkname):
1591 if apkname in self.apks:
1592 return self.apks[apkname]
1595 # Get the most recent 'num' apps added to the repo, as a list of package ids
1596 # with the most recent first.
1597 def getlatest(self, num):
1599 for apk, app in self.apks.items():
1603 if apps[appid] > added:
1607 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1608 lst = [app for app, _ in sortedapps]
1613 def isApkDebuggable(apkfile, config):
1614 """Returns True if the given apk file is debuggable
1616 :param apkfile: full path to the apk to check"""
1618 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1620 if p.returncode != 0:
1621 logging.critical("Failed to get apk manifest information")
1623 for line in p.output.splitlines():
1624 if 'android:debuggable' in line and not line.endswith('0x0'):
1631 self.returncode = None
1635 def SdkToolsPopen(commands, cwd=None, output=True):
1637 if cmd not in config:
1638 config[cmd] = find_sdk_tools_cmd(commands[0])
1639 abscmd = config[cmd]
1641 logging.critical("Could not find '%s' on your system" % cmd)
1643 return FDroidPopen([abscmd] + commands[1:],
1644 cwd=cwd, output=output)
1647 def FDroidPopenBytes(commands, cwd=None, output=True, stderr_to_stdout=True):
1649 Run a command and capture the possibly huge output as bytes.
1651 :param commands: command and argument list like in subprocess.Popen
1652 :param cwd: optionally specifies a working directory
1653 :returns: A PopenResult.
1658 set_FDroidPopen_env()
1661 cwd = os.path.normpath(cwd)
1662 logging.debug("Directory: %s" % cwd)
1663 logging.debug("> %s" % ' '.join(commands))
1665 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
1666 result = PopenResult()
1669 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
1670 stdout=subprocess.PIPE, stderr=stderr_param)
1671 except OSError as e:
1672 raise BuildException("OSError while trying to execute " +
1673 ' '.join(commands) + ': ' + str(e))
1675 if not stderr_to_stdout and options.verbose:
1676 stderr_queue = Queue()
1677 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
1679 while not stderr_reader.eof():
1680 while not stderr_queue.empty():
1681 line = stderr_queue.get()
1682 sys.stderr.buffer.write(line)
1687 stdout_queue = Queue()
1688 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1691 # Check the queue for output (until there is no more to get)
1692 while not stdout_reader.eof():
1693 while not stdout_queue.empty():
1694 line = stdout_queue.get()
1695 if output and options.verbose:
1696 # Output directly to console
1697 sys.stderr.buffer.write(line)
1703 result.returncode = p.wait()
1704 result.output = buf.getvalue()
1709 def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
1711 Run a command and capture the possibly huge output as a str.
1713 :param commands: command and argument list like in subprocess.Popen
1714 :param cwd: optionally specifies a working directory
1715 :returns: A PopenResult.
1717 result = FDroidPopenBytes(commands, cwd, output, stderr_to_stdout)
1718 result.output = result.output.decode('utf-8')
1722 gradle_comment = re.compile(r'[ ]*//')
1723 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1724 gradle_line_matches = [
1725 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1726 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1727 re.compile(r'.*\.readLine\(.*'),
1731 def remove_signing_keys(build_dir):
1732 for root, dirs, files in os.walk(build_dir):
1733 if 'build.gradle' in files:
1734 path = os.path.join(root, 'build.gradle')
1736 with open(path, "r") as o:
1737 lines = o.readlines()
1743 with open(path, "w") as o:
1744 while i < len(lines):
1747 while line.endswith('\\\n'):
1748 line = line.rstrip('\\\n') + lines[i]
1751 if gradle_comment.match(line):
1756 opened += line.count('{')
1757 opened -= line.count('}')
1760 if gradle_signing_configs.match(line):
1765 if any(s.match(line) for s in gradle_line_matches):
1773 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1776 'project.properties',
1778 'default.properties',
1779 'ant.properties', ]:
1780 if propfile in files:
1781 path = os.path.join(root, propfile)
1783 with open(path, "r") as o:
1784 lines = o.readlines()
1788 with open(path, "w") as o:
1790 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1797 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1800 def set_FDroidPopen_env(build=None):
1801 # There is only a weak standard, the variables used by gradle, so also set
1802 # up the most commonly used environment variables for SDK and NDK
1803 global env, orig_path
1806 orig_path = env['PATH']
1807 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
1808 env[n] = config['sdk_path']
1810 # Set up environment vars that depend on each build
1811 if build is not None:
1812 path = build.ndk_path()
1813 paths = orig_path.split(os.pathsep)
1817 env['PATH'] = os.pathsep.join(paths)
1819 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
1820 env[n] = build.ndk_path()
1823 def replace_config_vars(cmd, build):
1824 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1825 cmd = cmd.replace('$$NDK$$', get_ndk_path(build['ndk']))
1826 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1827 if build is not None:
1828 cmd = cmd.replace('$$COMMIT$$', build.commit)
1829 cmd = cmd.replace('$$VERSION$$', build.version)
1830 cmd = cmd.replace('$$VERCODE$$', build.vercode)
1834 def place_srclib(root_dir, number, libpath):
1837 relpath = os.path.relpath(libpath, root_dir)
1838 proppath = os.path.join(root_dir, 'project.properties')
1841 if os.path.isfile(proppath):
1842 with open(proppath, "r") as o:
1843 lines = o.readlines()
1845 with open(proppath, "w") as o:
1848 if line.startswith('android.library.reference.%d=' % number):
1849 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1854 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1856 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
1859 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
1860 """Verify that two apks are the same
1862 One of the inputs is signed, the other is unsigned. The signature metadata
1863 is transferred from the signed to the unsigned apk, and then jarsigner is
1864 used to verify that the signature from the signed apk is also varlid for
1866 :param signed_apk: Path to a signed apk file
1867 :param unsigned_apk: Path to an unsigned apk file expected to match it
1868 :param tmp_dir: Path to directory for temporary files
1869 :returns: None if the verification is successful, otherwise a string
1870 describing what went wrong.
1872 with ZipFile(signed_apk) as signed_apk_as_zip:
1873 meta_inf_files = ['META-INF/MANIFEST.MF']
1874 for f in signed_apk_as_zip.namelist():
1875 if apk_sigfile.match(f):
1876 meta_inf_files.append(f)
1877 if len(meta_inf_files) < 3:
1878 return "Signature files missing from {0}".format(signed_apk)
1879 signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
1880 with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
1881 for meta_inf_file in meta_inf_files:
1882 unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
1884 if subprocess.call([config['jarsigner'], '-verify', unsigned_apk]) != 0:
1885 logging.info("...NOT verified - {0}".format(signed_apk))
1886 return compare_apks(signed_apk, unsigned_apk, tmp_dir)
1887 logging.info("...successfully verified")
1890 apk_badchars = re.compile('''[/ :;'"]''')
1893 def compare_apks(apk1, apk2, tmp_dir):
1896 Returns None if the apk content is the same (apart from the signing key),
1897 otherwise a string describing what's different, or what went wrong when
1898 trying to do the comparison.
1901 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
1902 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
1903 for d in [apk1dir, apk2dir]:
1904 if os.path.exists(d):
1907 os.mkdir(os.path.join(d, 'jar-xf'))
1909 if subprocess.call(['jar', 'xf',
1910 os.path.abspath(apk1)],
1911 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
1912 return("Failed to unpack " + apk1)
1913 if subprocess.call(['jar', 'xf',
1914 os.path.abspath(apk2)],
1915 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
1916 return("Failed to unpack " + apk2)
1918 # try to find apktool in the path, if it hasn't been manually configed
1919 if 'apktool' not in config:
1920 tmp = find_command('apktool')
1922 config['apktool'] = tmp
1923 if 'apktool' in config:
1924 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
1926 return("Failed to unpack " + apk1)
1927 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
1929 return("Failed to unpack " + apk2)
1931 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
1932 lines = p.output.splitlines()
1933 if len(lines) != 1 or 'META-INF' not in lines[0]:
1934 meld = find_command('meld')
1935 if meld is not None:
1936 p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
1937 return("Unexpected diff output - " + p.output)
1939 # since everything verifies, delete the comparison to keep cruft down
1940 shutil.rmtree(apk1dir)
1941 shutil.rmtree(apk2dir)
1943 # If we get here, it seems like they're the same!
1947 def find_command(command):
1948 '''find the full path of a command, or None if it can't be found in the PATH'''
1951 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
1953 fpath, fname = os.path.split(command)
1958 for path in os.environ["PATH"].split(os.pathsep):
1959 path = path.strip('"')
1960 exe_file = os.path.join(path, command)
1961 if is_exe(exe_file):
1968 '''generate a random password for when generating keys'''
1969 h = hashlib.sha256()
1970 h.update(os.urandom(16)) # salt
1971 h.update(socket.getfqdn().encode('utf-8'))
1972 passwd = base64.b64encode(h.digest()).strip()
1973 return passwd.decode('utf-8')
1976 def genkeystore(localconfig):
1977 '''Generate a new key with random passwords and add it to new keystore'''
1978 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
1979 keystoredir = os.path.dirname(localconfig['keystore'])
1980 if keystoredir is None or keystoredir == '':
1981 keystoredir = os.path.join(os.getcwd(), keystoredir)
1982 if not os.path.exists(keystoredir):
1983 os.makedirs(keystoredir, mode=0o700)
1985 write_password_file("keystorepass", localconfig['keystorepass'])
1986 write_password_file("keypass", localconfig['keypass'])
1987 p = FDroidPopen([config['keytool'], '-genkey',
1988 '-keystore', localconfig['keystore'],
1989 '-alias', localconfig['repo_keyalias'],
1990 '-keyalg', 'RSA', '-keysize', '4096',
1991 '-sigalg', 'SHA256withRSA',
1992 '-validity', '10000',
1993 '-storepass:file', config['keystorepassfile'],
1994 '-keypass:file', config['keypassfile'],
1995 '-dname', localconfig['keydname']])
1996 # TODO keypass should be sent via stdin
1997 if p.returncode != 0:
1998 raise BuildException("Failed to generate key", p.output)
1999 os.chmod(localconfig['keystore'], 0o0600)
2000 # now show the lovely key that was just generated
2001 p = FDroidPopen([config['keytool'], '-list', '-v',
2002 '-keystore', localconfig['keystore'],
2003 '-alias', localconfig['repo_keyalias'],
2004 '-storepass:file', config['keystorepassfile']])
2005 logging.info(p.output.strip() + '\n\n')
2008 def write_to_config(thisconfig, key, value=None):
2009 '''write a key/value to the local config.py'''
2011 origkey = key + '_orig'
2012 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2013 with open('config.py', 'r') as f:
2015 pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
2016 repl = '\n' + key + ' = "' + value + '"'
2017 data = re.sub(pattern, repl, data)
2018 # if this key is not in the file, append it
2019 if not re.match('\s*' + key + '\s*=\s*"', data):
2021 # make sure the file ends with a carraige return
2022 if not re.match('\n$', data):
2024 with open('config.py', 'w') as f:
2028 def parse_xml(path):
2029 return XMLElementTree.parse(path).getroot()
2032 def string_is_integer(string):
2040 def get_per_app_repos():
2041 '''per-app repos are dirs named with the packageName of a single app'''
2043 # Android packageNames are Java packages, they may contain uppercase or
2044 # lowercase letters ('A' through 'Z'), numbers, and underscores
2045 # ('_'). However, individual package name parts may only start with
2046 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2047 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2050 for root, dirs, files in os.walk(os.getcwd()):
2052 print('checking', root, 'for', d)
2053 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2054 # standard parts of an fdroid repo, so never packageNames
2057 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):