3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
37 import xml.etree.ElementTree as XMLElementTree
39 from queue import Queue
41 from zipfile import ZipFile
43 import fdroidserver.metadata
44 from .asynchronousfilereader import AsynchronousFileReader
47 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
56 'sdk_path': "$ANDROID_HOME",
61 'r12b': "$ANDROID_NDK",
63 'build_tools': "24.0.2",
64 'force_build_tools': False,
69 'accepted_formats': ['txt', 'yml'],
70 'sync_from_local_copy_dir': False,
71 'per_app_repos': False,
72 'make_current_version_link': True,
73 'current_version_name_source': 'Name',
74 'update_stats': False,
78 'stats_to_carbon': False,
80 'build_server_always': False,
81 'keystore': 'keystore.jks',
82 'smartcardoptions': [],
88 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
89 'repo_name': "My First FDroid Repo Demo",
90 'repo_icon': "fdroid-icon.png",
91 'repo_description': '''
92 This is a repository of apps to be used with FDroid. Applications in this
93 repository are either official binaries built by the original application
94 developers, or are binaries built from source by the admin of f-droid.org
95 using the tools on https://gitlab.com/u/fdroid.
101 def setup_global_opts(parser):
102 parser.add_argument("-v", "--verbose", action="store_true", default=False,
103 help="Spew out even more information than normal")
104 parser.add_argument("-q", "--quiet", action="store_true", default=False,
105 help="Restrict output to warnings and errors")
108 def fill_config_defaults(thisconfig):
109 for k, v in default_config.items():
110 if k not in thisconfig:
113 # Expand paths (~users and $vars)
114 def expand_path(path):
118 path = os.path.expanduser(path)
119 path = os.path.expandvars(path)
124 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
129 thisconfig[k + '_orig'] = v
131 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
132 if thisconfig['java_paths'] is None:
133 thisconfig['java_paths'] = dict()
135 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
136 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
137 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
138 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
139 if os.getenv('JAVA_HOME') is not None:
140 pathlist.append(os.getenv('JAVA_HOME'))
141 if os.getenv('PROGRAMFILES') is not None:
142 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
143 for d in sorted(pathlist):
144 if os.path.islink(d):
146 j = os.path.basename(d)
147 # the last one found will be the canonical one, so order appropriately
149 r'^1\.([6-9])\.0\.jdk$', # OSX
150 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
151 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
152 r'^jdk([6-9])-openjdk$', # Arch
153 r'^java-([6-9])-openjdk$', # Arch
154 r'^java-([6-9])-jdk$', # Arch (oracle)
155 r'^java-1\.([6-9])\.0-.*$', # RedHat
156 r'^java-([6-9])-oracle$', # Debian WebUpd8
157 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
158 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
160 m = re.match(regex, j)
163 for p in [d, os.path.join(d, 'Contents', 'Home')]:
164 if os.path.exists(os.path.join(p, 'bin', 'javac')):
165 thisconfig['java_paths'][m.group(1)] = p
167 for java_version in ('7', '8', '9'):
168 if java_version not in thisconfig['java_paths']:
170 java_home = thisconfig['java_paths'][java_version]
171 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
172 if os.path.exists(jarsigner):
173 thisconfig['jarsigner'] = jarsigner
174 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
175 break # Java7 is preferred, so quit if found
177 for k in ['ndk_paths', 'java_paths']:
183 thisconfig[k][k2] = exp
184 thisconfig[k][k2 + '_orig'] = v
187 def regsub_file(pattern, repl, path):
188 with open(path, 'rb') as f:
190 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
191 with open(path, 'wb') as f:
195 def read_config(opts, config_file='config.py'):
196 """Read the repository config
198 The config is read from config_file, which is in the current
199 directory when any of the repo management commands are used. If
200 there is a local metadata file in the git repo, then config.py is
201 not required, just use defaults.
204 global config, options
206 if config is not None:
213 if os.path.isfile(config_file):
214 logging.debug("Reading %s" % config_file)
215 with io.open(config_file, "rb") as f:
216 code = compile(f.read(), config_file, 'exec')
217 exec(code, None, config)
218 elif len(get_local_metadata_files()) == 0:
219 logging.critical("Missing config file - is this a repo directory?")
222 # smartcardoptions must be a list since its command line args for Popen
223 if 'smartcardoptions' in config:
224 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
225 elif 'keystore' in config and config['keystore'] == 'NONE':
226 # keystore='NONE' means use smartcard, these are required defaults
227 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
228 'SunPKCS11-OpenSC', '-providerClass',
229 'sun.security.pkcs11.SunPKCS11',
230 '-providerArg', 'opensc-fdroid.cfg']
232 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
233 st = os.stat(config_file)
234 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
235 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
237 fill_config_defaults(config)
239 for k in ["keystorepass", "keypass"]:
241 write_password_file(k)
243 for k in ["repo_description", "archive_description"]:
245 config[k] = clean_description(config[k])
247 if 'serverwebroot' in config:
248 if isinstance(config['serverwebroot'], str):
249 roots = [config['serverwebroot']]
250 elif all(isinstance(item, str) for item in config['serverwebroot']):
251 roots = config['serverwebroot']
253 raise TypeError('only accepts strings, lists, and tuples')
255 for rootstr in roots:
256 # since this is used with rsync, where trailing slashes have
257 # meaning, ensure there is always a trailing slash
258 if rootstr[-1] != '/':
260 rootlist.append(rootstr.replace('//', '/'))
261 config['serverwebroot'] = rootlist
266 def find_sdk_tools_cmd(cmd):
267 '''find a working path to a tool from the Android SDK'''
270 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
271 # try to find a working path to this command, in all the recent possible paths
272 if 'build_tools' in config:
273 build_tools = os.path.join(config['sdk_path'], 'build-tools')
274 # if 'build_tools' was manually set and exists, check only that one
275 configed_build_tools = os.path.join(build_tools, config['build_tools'])
276 if os.path.exists(configed_build_tools):
277 tooldirs.append(configed_build_tools)
279 # no configed version, so hunt known paths for it
280 for f in sorted(os.listdir(build_tools), reverse=True):
281 if os.path.isdir(os.path.join(build_tools, f)):
282 tooldirs.append(os.path.join(build_tools, f))
283 tooldirs.append(build_tools)
284 sdk_tools = os.path.join(config['sdk_path'], 'tools')
285 if os.path.exists(sdk_tools):
286 tooldirs.append(sdk_tools)
287 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
288 if os.path.exists(sdk_platform_tools):
289 tooldirs.append(sdk_platform_tools)
290 tooldirs.append('/usr/bin')
292 if os.path.isfile(os.path.join(d, cmd)):
293 return os.path.join(d, cmd)
294 # did not find the command, exit with error message
295 ensure_build_tools_exists(config)
298 def test_sdk_exists(thisconfig):
299 if 'sdk_path' not in thisconfig:
300 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
303 logging.error("'sdk_path' not set in config.py!")
305 if thisconfig['sdk_path'] == default_config['sdk_path']:
306 logging.error('No Android SDK found!')
307 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
308 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
310 if not os.path.exists(thisconfig['sdk_path']):
311 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
313 if not os.path.isdir(thisconfig['sdk_path']):
314 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
316 for d in ['build-tools', 'platform-tools', 'tools']:
317 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
318 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
319 thisconfig['sdk_path'], d))
324 def ensure_build_tools_exists(thisconfig):
325 if not test_sdk_exists(thisconfig):
327 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
328 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
329 if not os.path.isdir(versioned_build_tools):
330 logging.critical('Android Build Tools path "'
331 + versioned_build_tools + '" does not exist!')
335 def write_password_file(pwtype, password=None):
337 writes out passwords to a protected file instead of passing passwords as
338 command line argments
340 filename = '.fdroid.' + pwtype + '.txt'
341 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
343 os.write(fd, config[pwtype].encode('utf-8'))
345 os.write(fd, password.encode('utf-8'))
347 config[pwtype + 'file'] = filename
350 def get_local_metadata_files():
351 '''get any metadata files local to an app's source repo
353 This tries to ignore anything that does not count as app metdata,
354 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
357 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
360 # Given the arguments in the form of multiple appid:[vc] strings, this returns
361 # a dictionary with the set of vercodes specified for each package.
362 def read_pkg_args(args, allow_vercodes=False):
369 if allow_vercodes and ':' in p:
370 package, vercode = p.split(':')
372 package, vercode = p, None
373 if package not in vercodes:
374 vercodes[package] = [vercode] if vercode else []
376 elif vercode and vercode not in vercodes[package]:
377 vercodes[package] += [vercode] if vercode else []
382 # On top of what read_pkg_args does, this returns the whole app metadata, but
383 # limiting the builds list to the builds matching the vercodes specified.
384 def read_app_args(args, allapps, allow_vercodes=False):
386 vercodes = read_pkg_args(args, allow_vercodes)
392 for appid, app in allapps.items():
393 if appid in vercodes:
396 if len(apps) != len(vercodes):
399 logging.critical("No such package: %s" % p)
400 raise FDroidException("Found invalid app ids in arguments")
402 raise FDroidException("No packages specified")
405 for appid, app in apps.items():
409 app.builds = [b for b in app.builds if b.vercode in vc]
410 if len(app.builds) != len(vercodes[appid]):
412 allvcs = [b.vercode for b in app.builds]
413 for v in vercodes[appid]:
415 logging.critical("No such vercode %s for app %s" % (v, appid))
418 raise FDroidException("Found invalid vercodes for some apps")
423 def get_extension(filename):
424 base, ext = os.path.splitext(filename)
427 return base, ext.lower()[1:]
430 def has_extension(filename, ext):
431 _, f_ext = get_extension(filename)
435 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
438 def clean_description(description):
439 'Remove unneeded newlines and spaces from a block of description text'
441 # this is split up by paragraph to make removing the newlines easier
442 for paragraph in re.split(r'\n\n', description):
443 paragraph = re.sub('\r', '', paragraph)
444 paragraph = re.sub('\n', ' ', paragraph)
445 paragraph = re.sub(' {2,}', ' ', paragraph)
446 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
447 returnstring += paragraph + '\n\n'
448 return returnstring.rstrip('\n')
451 def apknameinfo(filename):
452 filename = os.path.basename(filename)
453 m = apk_regex.match(filename)
455 result = (m.group(1), m.group(2))
456 except AttributeError:
457 raise FDroidException("Invalid apk name: %s" % filename)
461 def getapkname(app, build):
462 return "%s_%s.apk" % (app.id, build.vercode)
465 def getsrcname(app, build):
466 return "%s_%s_src.tar.gz" % (app.id, build.vercode)
478 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
481 def getvcs(vcstype, remote, local):
483 return vcs_git(remote, local)
484 if vcstype == 'git-svn':
485 return vcs_gitsvn(remote, local)
487 return vcs_hg(remote, local)
489 return vcs_bzr(remote, local)
490 if vcstype == 'srclib':
491 if local != os.path.join('build', 'srclib', remote):
492 raise VCSException("Error: srclib paths are hard-coded!")
493 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
495 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
496 raise VCSException("Invalid vcs type " + vcstype)
499 def getsrclibvcs(name):
500 if name not in fdroidserver.metadata.srclibs:
501 raise VCSException("Missing srclib " + name)
502 return fdroidserver.metadata.srclibs[name]['Repo Type']
507 def __init__(self, remote, local):
509 # svn, git-svn and bzr may require auth
511 if self.repotype() in ('git-svn', 'bzr'):
513 if self.repotype == 'git-svn':
514 raise VCSException("Authentication is not supported for git-svn")
515 self.username, remote = remote.split('@')
516 if ':' not in self.username:
517 raise VCSException("Password required with username")
518 self.username, self.password = self.username.split(':')
522 self.clone_failed = False
523 self.refreshed = False
529 # Take the local repository to a clean version of the given revision, which
530 # is specificed in the VCS's native format. Beforehand, the repository can
531 # be dirty, or even non-existent. If the repository does already exist
532 # locally, it will be updated from the origin, but only once in the
533 # lifetime of the vcs object.
534 # None is acceptable for 'rev' if you know you are cloning a clean copy of
535 # the repo - otherwise it must specify a valid revision.
536 def gotorevision(self, rev, refresh=True):
538 if self.clone_failed:
539 raise VCSException("Downloading the repository already failed once, not trying again.")
541 # The .fdroidvcs-id file for a repo tells us what VCS type
542 # and remote that directory was created from, allowing us to drop it
543 # automatically if either of those things changes.
544 fdpath = os.path.join(self.local, '..',
545 '.fdroidvcs-' + os.path.basename(self.local))
546 fdpath = os.path.normpath(fdpath)
547 cdata = self.repotype() + ' ' + self.remote
550 if os.path.exists(self.local):
551 if os.path.exists(fdpath):
552 with open(fdpath, 'r') as f:
553 fsdata = f.read().strip()
558 logging.info("Repository details for %s changed - deleting" % (
562 logging.info("Repository details for %s missing - deleting" % (
565 shutil.rmtree(self.local)
569 self.refreshed = True
572 self.gotorevisionx(rev)
573 except FDroidException as e:
576 # If necessary, write the .fdroidvcs file.
577 if writeback and not self.clone_failed:
578 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
579 with open(fdpath, 'w+') as f:
585 # Derived classes need to implement this. It's called once basic checking
586 # has been performend.
587 def gotorevisionx(self, rev):
588 raise VCSException("This VCS type doesn't define gotorevisionx")
590 # Initialise and update submodules
591 def initsubmodules(self):
592 raise VCSException('Submodules not supported for this vcs type')
594 # Get a list of all known tags
596 if not self._gettags:
597 raise VCSException('gettags not supported for this vcs type')
599 for tag in self._gettags():
600 if re.match('[-A-Za-z0-9_. /]+$', tag):
604 # Get a list of all the known tags, sorted from newest to oldest
605 def latesttags(self):
606 raise VCSException('latesttags not supported for this vcs type')
608 # Get current commit reference (hash, revision, etc)
610 raise VCSException('getref not supported for this vcs type')
612 # Returns the srclib (name, path) used in setting up the current
623 # If the local directory exists, but is somehow not a git repository, git
624 # will traverse up the directory tree until it finds one that is (i.e.
625 # fdroidserver) and then we'll proceed to destroy it! This is called as
628 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
629 result = p.output.rstrip()
630 if not result.endswith(self.local):
631 raise VCSException('Repository mismatch')
633 def gotorevisionx(self, rev):
634 if not os.path.exists(self.local):
636 p = FDroidPopen(['git', 'clone', self.remote, self.local])
637 if p.returncode != 0:
638 self.clone_failed = True
639 raise VCSException("Git clone failed", p.output)
643 # Discard any working tree changes
644 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
645 'git', 'reset', '--hard'], cwd=self.local, output=False)
646 if p.returncode != 0:
647 raise VCSException("Git reset failed", p.output)
648 # Remove untracked files now, in case they're tracked in the target
649 # revision (it happens!)
650 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
651 'git', 'clean', '-dffx'], cwd=self.local, output=False)
652 if p.returncode != 0:
653 raise VCSException("Git clean failed", p.output)
654 if not self.refreshed:
655 # Get latest commits and tags from remote
656 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
657 if p.returncode != 0:
658 raise VCSException("Git fetch failed", p.output)
659 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
660 if p.returncode != 0:
661 raise VCSException("Git fetch failed", p.output)
662 # Recreate origin/HEAD as git clone would do it, in case it disappeared
663 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
664 if p.returncode != 0:
665 lines = p.output.splitlines()
666 if 'Multiple remote HEAD branches' not in lines[0]:
667 raise VCSException("Git remote set-head failed", p.output)
668 branch = lines[1].split(' ')[-1]
669 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
670 if p2.returncode != 0:
671 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
672 self.refreshed = True
673 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
674 # a github repo. Most of the time this is the same as origin/master.
675 rev = rev or 'origin/HEAD'
676 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
677 if p.returncode != 0:
678 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
679 # Get rid of any uncontrolled files left behind
680 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
681 if p.returncode != 0:
682 raise VCSException("Git clean failed", p.output)
684 def initsubmodules(self):
686 submfile = os.path.join(self.local, '.gitmodules')
687 if not os.path.isfile(submfile):
688 raise VCSException("No git submodules available")
690 # fix submodules not accessible without an account and public key auth
691 with open(submfile, 'r') as f:
692 lines = f.readlines()
693 with open(submfile, 'w') as f:
695 if 'git@github.com' in line:
696 line = line.replace('git@github.com:', 'https://github.com/')
697 if 'git@gitlab.com' in line:
698 line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
701 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
702 if p.returncode != 0:
703 raise VCSException("Git submodule sync failed", p.output)
704 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
705 if p.returncode != 0:
706 raise VCSException("Git submodule update failed", p.output)
710 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
711 return p.output.splitlines()
713 tag_format = re.compile(r'tag: ([^),]*)')
715 def latesttags(self):
717 p = FDroidPopen(['git', 'log', '--tags',
718 '--simplify-by-decoration', '--pretty=format:%d'],
719 cwd=self.local, output=False)
721 for line in p.output.splitlines():
722 for tag in self.tag_format.findall(line):
727 class vcs_gitsvn(vcs):
732 # If the local directory exists, but is somehow not a git repository, git
733 # will traverse up the directory tree until it finds one that is (i.e.
734 # fdroidserver) and then we'll proceed to destory it! This is called as
737 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
738 result = p.output.rstrip()
739 if not result.endswith(self.local):
740 raise VCSException('Repository mismatch')
742 def gotorevisionx(self, rev):
743 if not os.path.exists(self.local):
745 gitsvn_args = ['git', 'svn', 'clone']
746 if ';' in self.remote:
747 remote_split = self.remote.split(';')
748 for i in remote_split[1:]:
749 if i.startswith('trunk='):
750 gitsvn_args.extend(['-T', i[6:]])
751 elif i.startswith('tags='):
752 gitsvn_args.extend(['-t', i[5:]])
753 elif i.startswith('branches='):
754 gitsvn_args.extend(['-b', i[9:]])
755 gitsvn_args.extend([remote_split[0], self.local])
756 p = FDroidPopen(gitsvn_args, output=False)
757 if p.returncode != 0:
758 self.clone_failed = True
759 raise VCSException("Git svn clone failed", p.output)
761 gitsvn_args.extend([self.remote, self.local])
762 p = FDroidPopen(gitsvn_args, output=False)
763 if p.returncode != 0:
764 self.clone_failed = True
765 raise VCSException("Git svn clone failed", p.output)
769 # Discard any working tree changes
770 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
771 if p.returncode != 0:
772 raise VCSException("Git reset failed", p.output)
773 # Remove untracked files now, in case they're tracked in the target
774 # revision (it happens!)
775 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
776 if p.returncode != 0:
777 raise VCSException("Git clean failed", p.output)
778 if not self.refreshed:
779 # Get new commits, branches and tags from repo
780 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
781 if p.returncode != 0:
782 raise VCSException("Git svn fetch failed")
783 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
784 if p.returncode != 0:
785 raise VCSException("Git svn rebase failed", p.output)
786 self.refreshed = True
788 rev = rev or 'master'
790 nospaces_rev = rev.replace(' ', '%20')
791 # Try finding a svn tag
792 for treeish in ['origin/', '']:
793 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
794 if p.returncode == 0:
796 if p.returncode != 0:
797 # No tag found, normal svn rev translation
798 # Translate svn rev into git format
799 rev_split = rev.split('/')
802 for treeish in ['origin/', '']:
803 if len(rev_split) > 1:
804 treeish += rev_split[0]
805 svn_rev = rev_split[1]
808 # if no branch is specified, then assume trunk (i.e. 'master' branch):
812 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
814 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
815 git_rev = p.output.rstrip()
817 if p.returncode == 0 and git_rev:
820 if p.returncode != 0 or not git_rev:
821 # Try a plain git checkout as a last resort
822 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
823 if p.returncode != 0:
824 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
826 # Check out the git rev equivalent to the svn rev
827 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
828 if p.returncode != 0:
829 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
831 # Get rid of any uncontrolled files left behind
832 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
833 if p.returncode != 0:
834 raise VCSException("Git clean failed", p.output)
838 for treeish in ['origin/', '']:
839 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
845 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
846 if p.returncode != 0:
848 return p.output.strip()
856 def gotorevisionx(self, rev):
857 if not os.path.exists(self.local):
858 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
859 if p.returncode != 0:
860 self.clone_failed = True
861 raise VCSException("Hg clone failed", p.output)
863 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
864 if p.returncode != 0:
865 raise VCSException("Hg status failed", p.output)
866 for line in p.output.splitlines():
867 if not line.startswith('? '):
868 raise VCSException("Unexpected output from hg status -uS: " + line)
869 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
870 if not self.refreshed:
871 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
872 if p.returncode != 0:
873 raise VCSException("Hg pull failed", p.output)
874 self.refreshed = True
876 rev = rev or 'default'
879 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
880 if p.returncode != 0:
881 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
882 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
883 # Also delete untracked files, we have to enable purge extension for that:
884 if "'purge' is provided by the following extension" in p.output:
885 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
886 myfile.write("\n[extensions]\nhgext.purge=\n")
887 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
888 if p.returncode != 0:
889 raise VCSException("HG purge failed", p.output)
890 elif p.returncode != 0:
891 raise VCSException("HG purge failed", p.output)
894 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
895 return p.output.splitlines()[1:]
903 def gotorevisionx(self, rev):
904 if not os.path.exists(self.local):
905 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
906 if p.returncode != 0:
907 self.clone_failed = True
908 raise VCSException("Bzr branch failed", p.output)
910 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
911 if p.returncode != 0:
912 raise VCSException("Bzr revert failed", p.output)
913 if not self.refreshed:
914 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
915 if p.returncode != 0:
916 raise VCSException("Bzr update failed", p.output)
917 self.refreshed = True
919 revargs = list(['-r', rev] if rev else [])
920 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
921 if p.returncode != 0:
922 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
925 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
926 return [tag.split(' ')[0].strip() for tag in
927 p.output.splitlines()]
930 def unescape_string(string):
933 if string[0] == '"' and string[-1] == '"':
936 return string.replace("\\'", "'")
939 def retrieve_string(app_dir, string, xmlfiles=None):
941 if not string.startswith('@string/'):
942 return unescape_string(string)
947 os.path.join(app_dir, 'res'),
948 os.path.join(app_dir, 'src', 'main', 'res'),
950 for r, d, f in os.walk(res_dir):
951 if os.path.basename(r) == 'values':
952 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
954 name = string[len('@string/'):]
956 def element_content(element):
957 if element.text is None:
959 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
960 return s.decode('utf-8').strip()
962 for path in xmlfiles:
963 if not os.path.isfile(path):
965 xml = parse_xml(path)
966 element = xml.find('string[@name="' + name + '"]')
967 if element is not None:
968 content = element_content(element)
969 return retrieve_string(app_dir, content, xmlfiles)
974 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
975 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
978 # Return list of existing files that will be used to find the highest vercode
979 def manifest_paths(app_dir, flavours):
981 possible_manifests = \
982 [os.path.join(app_dir, 'AndroidManifest.xml'),
983 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
984 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
985 os.path.join(app_dir, 'build.gradle')]
987 for flavour in flavours:
990 possible_manifests.append(
991 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
993 return [path for path in possible_manifests if os.path.isfile(path)]
996 # Retrieve the package name. Returns the name, or None if not found.
997 def fetch_real_name(app_dir, flavours):
998 for path in manifest_paths(app_dir, flavours):
999 if not has_extension(path, 'xml') or not os.path.isfile(path):
1001 logging.debug("fetch_real_name: Checking manifest at " + path)
1002 xml = parse_xml(path)
1003 app = xml.find('application')
1006 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1008 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1009 result = retrieve_string_singleline(app_dir, label)
1011 result = result.strip()
1016 def get_library_references(root_dir):
1018 proppath = os.path.join(root_dir, 'project.properties')
1019 if not os.path.isfile(proppath):
1021 with open(proppath, 'r', encoding='iso-8859-1') as f:
1023 if not line.startswith('android.library.reference.'):
1025 path = line.split('=')[1].strip()
1026 relpath = os.path.join(root_dir, path)
1027 if not os.path.isdir(relpath):
1029 logging.debug("Found subproject at %s" % path)
1030 libraries.append(path)
1034 def ant_subprojects(root_dir):
1035 subprojects = get_library_references(root_dir)
1036 for subpath in subprojects:
1037 subrelpath = os.path.join(root_dir, subpath)
1038 for p in get_library_references(subrelpath):
1039 relp = os.path.normpath(os.path.join(subpath, p))
1040 if relp not in subprojects:
1041 subprojects.insert(0, relp)
1045 def remove_debuggable_flags(root_dir):
1046 # Remove forced debuggable flags
1047 logging.debug("Removing debuggable flags from %s" % root_dir)
1048 for root, dirs, files in os.walk(root_dir):
1049 if 'AndroidManifest.xml' in files:
1050 regsub_file(r'android:debuggable="[^"]*"',
1052 os.path.join(root, 'AndroidManifest.xml'))
1055 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
1056 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1057 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1060 def app_matches_packagename(app, package):
1063 appid = app.UpdateCheckName or app.id
1064 if appid is None or appid == "Ignore":
1066 return appid == package
1069 # Extract some information from the AndroidManifest.xml at the given path.
1070 # Returns (version, vercode, package), any or all of which might be None.
1071 # All values returned are strings.
1072 def parse_androidmanifests(paths, app):
1074 ignoreversions = app.UpdateCheckIgnore
1075 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1078 return (None, None, None)
1086 if not os.path.isfile(path):
1089 logging.debug("Parsing manifest at {0}".format(path))
1090 gradle = has_extension(path, 'gradle')
1096 with open(path, 'r') as f:
1098 if gradle_comment.match(line):
1100 # Grab first occurence of each to avoid running into
1101 # alternative flavours and builds.
1103 matches = psearch_g(line)
1105 s = matches.group(2)
1106 if app_matches_packagename(app, s):
1109 matches = vnsearch_g(line)
1111 version = matches.group(2)
1113 matches = vcsearch_g(line)
1115 vercode = matches.group(1)
1118 xml = parse_xml(path)
1119 if "package" in xml.attrib:
1120 s = xml.attrib["package"]
1121 if app_matches_packagename(app, s):
1123 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1124 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1125 base_dir = os.path.dirname(path)
1126 version = retrieve_string_singleline(base_dir, version)
1127 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1128 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1129 if string_is_integer(a):
1132 logging.warning("Problem with xml at {0}".format(path))
1134 # Remember package name, may be defined separately from version+vercode
1136 package = max_package
1138 logging.debug("..got package={0}, version={1}, vercode={2}"
1139 .format(package, version, vercode))
1141 # Always grab the package name and version name in case they are not
1142 # together with the highest version code
1143 if max_package is None and package is not None:
1144 max_package = package
1145 if max_version is None and version is not None:
1146 max_version = version
1148 if max_vercode is None or (vercode is not None and vercode > max_vercode):
1149 if not ignoresearch or not ignoresearch(version):
1150 if version is not None:
1151 max_version = version
1152 if vercode is not None:
1153 max_vercode = vercode
1154 if package is not None:
1155 max_package = package
1157 max_version = "Ignore"
1159 if max_version is None:
1160 max_version = "Unknown"
1162 if max_package and not is_valid_package_name(max_package):
1163 raise FDroidException("Invalid package name {0}".format(max_package))
1165 return (max_version, max_vercode, max_package)
1168 def is_valid_package_name(name):
1169 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1172 class FDroidException(Exception):
1174 def __init__(self, value, detail=None):
1176 self.detail = detail
1178 def shortened_detail(self):
1179 if len(self.detail) < 16000:
1181 return '[...]\n' + self.detail[-16000:]
1183 def get_wikitext(self):
1184 ret = repr(self.value) + "\n"
1187 ret += "<pre>\n" + self.shortened_detail() + "</pre>\n"
1193 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1197 class VCSException(FDroidException):
1201 class BuildException(FDroidException):
1205 # Get the specified source library.
1206 # Returns the path to it. Normally this is the path to be used when referencing
1207 # it, which may be a subdirectory of the actual project. If you want the base
1208 # directory of the project, pass 'basepath=True'.
1209 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1210 raw=False, prepare=True, preponly=False, refresh=True,
1219 name, ref = spec.split('@')
1221 number, name = name.split(':', 1)
1223 name, subdir = name.split('/', 1)
1225 if name not in fdroidserver.metadata.srclibs:
1226 raise VCSException('srclib ' + name + ' not found.')
1228 srclib = fdroidserver.metadata.srclibs[name]
1230 sdir = os.path.join(srclib_dir, name)
1233 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1234 vcs.srclib = (name, number, sdir)
1236 vcs.gotorevision(ref, refresh)
1243 libdir = os.path.join(sdir, subdir)
1244 elif srclib["Subdir"]:
1245 for subdir in srclib["Subdir"]:
1246 libdir_candidate = os.path.join(sdir, subdir)
1247 if os.path.exists(libdir_candidate):
1248 libdir = libdir_candidate
1254 remove_signing_keys(sdir)
1255 remove_debuggable_flags(sdir)
1259 if srclib["Prepare"]:
1260 cmd = replace_config_vars(srclib["Prepare"], build)
1262 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1263 if p.returncode != 0:
1264 raise BuildException("Error running prepare command for srclib %s"
1270 return (name, number, libdir)
1272 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1275 # Prepare the source code for a particular build
1276 # 'vcs' - the appropriate vcs object for the application
1277 # 'app' - the application details from the metadata
1278 # 'build' - the build details from the metadata
1279 # 'build_dir' - the path to the build directory, usually
1281 # 'srclib_dir' - the path to the source libraries directory, usually
1283 # 'extlib_dir' - the path to the external libraries directory, usually
1285 # Returns the (root, srclibpaths) where:
1286 # 'root' is the root directory, which may be the same as 'build_dir' or may
1287 # be a subdirectory of it.
1288 # 'srclibpaths' is information on the srclibs being used
1289 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1291 # Optionally, the actual app source can be in a subdirectory
1293 root_dir = os.path.join(build_dir, build.subdir)
1295 root_dir = build_dir
1297 # Get a working copy of the right revision
1298 logging.info("Getting source for revision " + build.commit)
1299 vcs.gotorevision(build.commit, refresh)
1301 # Initialise submodules if required
1302 if build.submodules:
1303 logging.info("Initialising submodules")
1304 vcs.initsubmodules()
1306 # Check that a subdir (if we're using one) exists. This has to happen
1307 # after the checkout, since it might not exist elsewhere
1308 if not os.path.exists(root_dir):
1309 raise BuildException('Missing subdir ' + root_dir)
1311 # Run an init command if one is required
1313 cmd = replace_config_vars(build.init, build)
1314 logging.info("Running 'init' commands in %s" % root_dir)
1316 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1317 if p.returncode != 0:
1318 raise BuildException("Error running init command for %s:%s" %
1319 (app.id, build.version), p.output)
1321 # Apply patches if any
1323 logging.info("Applying patches")
1324 for patch in build.patch:
1325 patch = patch.strip()
1326 logging.info("Applying " + patch)
1327 patch_path = os.path.join('metadata', app.id, patch)
1328 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1329 if p.returncode != 0:
1330 raise BuildException("Failed to apply patch %s" % patch_path)
1332 # Get required source libraries
1335 logging.info("Collecting source libraries")
1336 for lib in build.srclibs:
1337 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1338 refresh=refresh, build=build))
1340 for name, number, libpath in srclibpaths:
1341 place_srclib(root_dir, int(number) if number else None, libpath)
1343 basesrclib = vcs.getsrclib()
1344 # If one was used for the main source, add that too.
1346 srclibpaths.append(basesrclib)
1348 # Update the local.properties file
1349 localprops = [os.path.join(build_dir, 'local.properties')]
1351 parts = build.subdir.split(os.sep)
1354 cur = os.path.join(cur, d)
1355 localprops += [os.path.join(cur, 'local.properties')]
1356 for path in localprops:
1358 if os.path.isfile(path):
1359 logging.info("Updating local.properties file at %s" % path)
1360 with open(path, 'r', encoding='iso-8859-1') as f:
1364 logging.info("Creating local.properties file at %s" % path)
1365 # Fix old-fashioned 'sdk-location' by copying
1366 # from sdk.dir, if necessary
1368 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1369 re.S | re.M).group(1)
1370 props += "sdk-location=%s\n" % sdkloc
1372 props += "sdk.dir=%s\n" % config['sdk_path']
1373 props += "sdk-location=%s\n" % config['sdk_path']
1374 ndk_path = build.ndk_path()
1375 # if for any reason the path isn't valid or the directory
1376 # doesn't exist, some versions of Gradle will error with a
1377 # cryptic message (even if the NDK is not even necessary).
1378 # https://gitlab.com/fdroid/fdroidserver/issues/171
1379 if ndk_path and os.path.exists(ndk_path):
1381 props += "ndk.dir=%s\n" % ndk_path
1382 props += "ndk-location=%s\n" % ndk_path
1383 # Add java.encoding if necessary
1385 props += "java.encoding=%s\n" % build.encoding
1386 with open(path, 'w', encoding='iso-8859-1') as f:
1390 if build.build_method() == 'gradle':
1391 flavours = build.gradle
1394 n = build.target.split('-')[1]
1395 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1396 r'compileSdkVersion %s' % n,
1397 os.path.join(root_dir, 'build.gradle'))
1399 # Remove forced debuggable flags
1400 remove_debuggable_flags(root_dir)
1402 # Insert version code and number into the manifest if necessary
1403 if build.forceversion:
1404 logging.info("Changing the version name")
1405 for path in manifest_paths(root_dir, flavours):
1406 if not os.path.isfile(path):
1408 if has_extension(path, 'xml'):
1409 regsub_file(r'android:versionName="[^"]*"',
1410 r'android:versionName="%s"' % build.version,
1412 elif has_extension(path, 'gradle'):
1413 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1414 r"""\1versionName '%s'""" % build.version,
1417 if build.forcevercode:
1418 logging.info("Changing the version code")
1419 for path in manifest_paths(root_dir, flavours):
1420 if not os.path.isfile(path):
1422 if has_extension(path, 'xml'):
1423 regsub_file(r'android:versionCode="[^"]*"',
1424 r'android:versionCode="%s"' % build.vercode,
1426 elif has_extension(path, 'gradle'):
1427 regsub_file(r'versionCode[ =]+[0-9]+',
1428 r'versionCode %s' % build.vercode,
1431 # Delete unwanted files
1433 logging.info("Removing specified files")
1434 for part in getpaths(build_dir, build.rm):
1435 dest = os.path.join(build_dir, part)
1436 logging.info("Removing {0}".format(part))
1437 if os.path.lexists(dest):
1438 if os.path.islink(dest):
1439 FDroidPopen(['unlink', dest], output=False)
1441 FDroidPopen(['rm', '-rf', dest], output=False)
1443 logging.info("...but it didn't exist")
1445 remove_signing_keys(build_dir)
1447 # Add required external libraries
1449 logging.info("Collecting prebuilt libraries")
1450 libsdir = os.path.join(root_dir, 'libs')
1451 if not os.path.exists(libsdir):
1453 for lib in build.extlibs:
1455 logging.info("...installing extlib {0}".format(lib))
1456 libf = os.path.basename(lib)
1457 libsrc = os.path.join(extlib_dir, lib)
1458 if not os.path.exists(libsrc):
1459 raise BuildException("Missing extlib file {0}".format(libsrc))
1460 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1462 # Run a pre-build command if one is required
1464 logging.info("Running 'prebuild' commands in %s" % root_dir)
1466 cmd = replace_config_vars(build.prebuild, build)
1468 # Substitute source library paths into prebuild commands
1469 for name, number, libpath in srclibpaths:
1470 libpath = os.path.relpath(libpath, root_dir)
1471 cmd = cmd.replace('$$' + name + '$$', libpath)
1473 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1474 if p.returncode != 0:
1475 raise BuildException("Error running prebuild command for %s:%s" %
1476 (app.id, build.version), p.output)
1478 # Generate (or update) the ant build file, build.xml...
1479 if build.build_method() == 'ant' and build.update != ['no']:
1480 parms = ['android', 'update', 'lib-project']
1481 lparms = ['android', 'update', 'project']
1484 parms += ['-t', build.target]
1485 lparms += ['-t', build.target]
1487 update_dirs = build.update
1489 update_dirs = ant_subprojects(root_dir) + ['.']
1491 for d in update_dirs:
1492 subdir = os.path.join(root_dir, d)
1494 logging.debug("Updating main project")
1495 cmd = parms + ['-p', d]
1497 logging.debug("Updating subproject %s" % d)
1498 cmd = lparms + ['-p', d]
1499 p = SdkToolsPopen(cmd, cwd=root_dir)
1500 # Check to see whether an error was returned without a proper exit
1501 # code (this is the case for the 'no target set or target invalid'
1503 if p.returncode != 0 or p.output.startswith("Error: "):
1504 raise BuildException("Failed to update project at %s" % d, p.output)
1505 # Clean update dirs via ant
1507 logging.info("Cleaning subproject %s" % d)
1508 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1510 return (root_dir, srclibpaths)
1513 # Extend via globbing the paths from a field and return them as a map from
1514 # original path to resulting paths
1515 def getpaths_map(build_dir, globpaths):
1519 full_path = os.path.join(build_dir, p)
1520 full_path = os.path.normpath(full_path)
1521 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1523 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1527 # Extend via globbing the paths from a field and return them as a set
1528 def getpaths(build_dir, globpaths):
1529 paths_map = getpaths_map(build_dir, globpaths)
1531 for k, v in paths_map.items():
1538 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1544 self.path = os.path.join('stats', 'known_apks.txt')
1546 if os.path.isfile(self.path):
1547 with open(self.path, 'r', encoding='utf8') as f:
1549 t = line.rstrip().split(' ')
1551 self.apks[t[0]] = (t[1], None)
1553 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1554 self.changed = False
1556 def writeifchanged(self):
1557 if not self.changed:
1560 if not os.path.exists('stats'):
1564 for apk, app in self.apks.items():
1566 line = apk + ' ' + appid
1568 line += ' ' + time.strftime('%Y-%m-%d', added)
1571 with open(self.path, 'w', encoding='utf8') as f:
1572 for line in sorted(lst, key=natural_key):
1573 f.write(line + '\n')
1575 # Record an apk (if it's new, otherwise does nothing)
1576 # Returns the date it was added.
1577 def recordapk(self, apk, app, default_date=None):
1578 if apk not in self.apks:
1579 if default_date is None:
1580 default_date = time.gmtime(time.time())
1581 self.apks[apk] = (app, default_date)
1583 _, added = self.apks[apk]
1586 # Look up information - given the 'apkname', returns (app id, date added/None).
1587 # Or returns None for an unknown apk.
1588 def getapp(self, apkname):
1589 if apkname in self.apks:
1590 return self.apks[apkname]
1593 # Get the most recent 'num' apps added to the repo, as a list of package ids
1594 # with the most recent first.
1595 def getlatest(self, num):
1597 for apk, app in self.apks.items():
1601 if apps[appid] > added:
1605 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1606 lst = [app for app, _ in sortedapps]
1611 def isApkDebuggable(apkfile, config):
1612 """Returns True if the given apk file is debuggable
1614 :param apkfile: full path to the apk to check"""
1616 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1618 if p.returncode != 0:
1619 logging.critical("Failed to get apk manifest information")
1621 for line in p.output.splitlines():
1622 if 'android:debuggable' in line and not line.endswith('0x0'):
1629 self.returncode = None
1633 def SdkToolsPopen(commands, cwd=None, output=True):
1635 if cmd not in config:
1636 config[cmd] = find_sdk_tools_cmd(commands[0])
1637 abscmd = config[cmd]
1639 logging.critical("Could not find '%s' on your system" % cmd)
1641 return FDroidPopen([abscmd] + commands[1:],
1642 cwd=cwd, output=output)
1645 def FDroidPopenBytes(commands, cwd=None, output=True, stderr_to_stdout=True):
1647 Run a command and capture the possibly huge output as bytes.
1649 :param commands: command and argument list like in subprocess.Popen
1650 :param cwd: optionally specifies a working directory
1651 :returns: A PopenResult.
1656 set_FDroidPopen_env()
1659 cwd = os.path.normpath(cwd)
1660 logging.debug("Directory: %s" % cwd)
1661 logging.debug("> %s" % ' '.join(commands))
1663 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
1664 result = PopenResult()
1667 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
1668 stdout=subprocess.PIPE, stderr=stderr_param)
1669 except OSError as e:
1670 raise BuildException("OSError while trying to execute " +
1671 ' '.join(commands) + ': ' + str(e))
1673 if not stderr_to_stdout and options.verbose:
1674 stderr_queue = Queue()
1675 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
1677 while not stderr_reader.eof():
1678 while not stderr_queue.empty():
1679 line = stderr_queue.get()
1680 sys.stderr.buffer.write(line)
1685 stdout_queue = Queue()
1686 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1689 # Check the queue for output (until there is no more to get)
1690 while not stdout_reader.eof():
1691 while not stdout_queue.empty():
1692 line = stdout_queue.get()
1693 if output and options.verbose:
1694 # Output directly to console
1695 sys.stderr.buffer.write(line)
1701 result.returncode = p.wait()
1702 result.output = buf.getvalue()
1707 def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
1709 Run a command and capture the possibly huge output as a str.
1711 :param commands: command and argument list like in subprocess.Popen
1712 :param cwd: optionally specifies a working directory
1713 :returns: A PopenResult.
1715 result = FDroidPopenBytes(commands, cwd, output, stderr_to_stdout)
1716 result.output = result.output.decode('utf-8')
1720 gradle_comment = re.compile(r'[ ]*//')
1721 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1722 gradle_line_matches = [
1723 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1724 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1725 re.compile(r'.*\.readLine\(.*'),
1729 def remove_signing_keys(build_dir):
1730 for root, dirs, files in os.walk(build_dir):
1731 if 'build.gradle' in files:
1732 path = os.path.join(root, 'build.gradle')
1734 with open(path, "r", encoding='utf8') as o:
1735 lines = o.readlines()
1741 with open(path, "w", encoding='utf8') as o:
1742 while i < len(lines):
1745 while line.endswith('\\\n'):
1746 line = line.rstrip('\\\n') + lines[i]
1749 if gradle_comment.match(line):
1754 opened += line.count('{')
1755 opened -= line.count('}')
1758 if gradle_signing_configs.match(line):
1763 if any(s.match(line) for s in gradle_line_matches):
1771 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1774 'project.properties',
1776 'default.properties',
1777 'ant.properties', ]:
1778 if propfile in files:
1779 path = os.path.join(root, propfile)
1781 with open(path, "r", encoding='iso-8859-1') as o:
1782 lines = o.readlines()
1786 with open(path, "w", encoding='iso-8859-1') as o:
1788 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1795 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1798 def set_FDroidPopen_env(build=None):
1800 set up the environment variables for the build environment
1802 There is only a weak standard, the variables used by gradle, so also set
1803 up the most commonly used environment variables for SDK and NDK. Also, if
1804 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
1806 global env, orig_path
1810 orig_path = env['PATH']
1811 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
1812 env[n] = config['sdk_path']
1813 for k, v in config['java_paths'].items():
1814 env['JAVA%s_HOME' % k] = v
1816 missinglocale = True
1817 for k, v in env.items():
1818 if k == 'LANG' and v != 'C':
1819 missinglocale = False
1821 missinglocale = False
1823 env['LANG'] = 'en_US.UTF-8'
1825 if build is not None:
1826 path = build.ndk_path()
1827 paths = orig_path.split(os.pathsep)
1828 if path not in paths:
1829 paths = [path] + paths
1830 env['PATH'] = os.pathsep.join(paths)
1831 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
1832 env[n] = build.ndk_path()
1835 def replace_config_vars(cmd, build):
1836 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1837 cmd = cmd.replace('$$NDK$$', build.ndk_path())
1838 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1839 if build is not None:
1840 cmd = cmd.replace('$$COMMIT$$', build.commit)
1841 cmd = cmd.replace('$$VERSION$$', build.version)
1842 cmd = cmd.replace('$$VERCODE$$', build.vercode)
1846 def place_srclib(root_dir, number, libpath):
1849 relpath = os.path.relpath(libpath, root_dir)
1850 proppath = os.path.join(root_dir, 'project.properties')
1853 if os.path.isfile(proppath):
1854 with open(proppath, "r", encoding='iso-8859-1') as o:
1855 lines = o.readlines()
1857 with open(proppath, "w", encoding='iso-8859-1') as o:
1860 if line.startswith('android.library.reference.%d=' % number):
1861 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1866 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1868 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
1871 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
1872 """Verify that two apks are the same
1874 One of the inputs is signed, the other is unsigned. The signature metadata
1875 is transferred from the signed to the unsigned apk, and then jarsigner is
1876 used to verify that the signature from the signed apk is also varlid for
1878 :param signed_apk: Path to a signed apk file
1879 :param unsigned_apk: Path to an unsigned apk file expected to match it
1880 :param tmp_dir: Path to directory for temporary files
1881 :returns: None if the verification is successful, otherwise a string
1882 describing what went wrong.
1884 with ZipFile(signed_apk) as signed_apk_as_zip:
1885 meta_inf_files = ['META-INF/MANIFEST.MF']
1886 for f in signed_apk_as_zip.namelist():
1887 if apk_sigfile.match(f):
1888 meta_inf_files.append(f)
1889 if len(meta_inf_files) < 3:
1890 return "Signature files missing from {0}".format(signed_apk)
1891 signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
1892 with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
1893 for meta_inf_file in meta_inf_files:
1894 unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
1896 if subprocess.call([config['jarsigner'], '-verify', unsigned_apk]) != 0:
1897 logging.info("...NOT verified - {0}".format(signed_apk))
1898 return compare_apks(signed_apk, unsigned_apk, tmp_dir)
1899 logging.info("...successfully verified")
1902 apk_badchars = re.compile('''[/ :;'"]''')
1905 def compare_apks(apk1, apk2, tmp_dir):
1908 Returns None if the apk content is the same (apart from the signing key),
1909 otherwise a string describing what's different, or what went wrong when
1910 trying to do the comparison.
1913 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
1914 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
1915 for d in [apk1dir, apk2dir]:
1916 if os.path.exists(d):
1919 os.mkdir(os.path.join(d, 'jar-xf'))
1921 if subprocess.call(['jar', 'xf',
1922 os.path.abspath(apk1)],
1923 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
1924 return("Failed to unpack " + apk1)
1925 if subprocess.call(['jar', 'xf',
1926 os.path.abspath(apk2)],
1927 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
1928 return("Failed to unpack " + apk2)
1930 # try to find apktool in the path, if it hasn't been manually configed
1931 if 'apktool' not in config:
1932 tmp = find_command('apktool')
1934 config['apktool'] = tmp
1935 if 'apktool' in config:
1936 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
1938 return("Failed to unpack " + apk1)
1939 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
1941 return("Failed to unpack " + apk2)
1943 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
1944 lines = p.output.splitlines()
1945 if len(lines) != 1 or 'META-INF' not in lines[0]:
1946 meld = find_command('meld')
1947 if meld is not None:
1948 p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
1949 return("Unexpected diff output - " + p.output)
1951 # since everything verifies, delete the comparison to keep cruft down
1952 shutil.rmtree(apk1dir)
1953 shutil.rmtree(apk2dir)
1955 # If we get here, it seems like they're the same!
1959 def find_command(command):
1960 '''find the full path of a command, or None if it can't be found in the PATH'''
1963 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
1965 fpath, fname = os.path.split(command)
1970 for path in os.environ["PATH"].split(os.pathsep):
1971 path = path.strip('"')
1972 exe_file = os.path.join(path, command)
1973 if is_exe(exe_file):
1980 '''generate a random password for when generating keys'''
1981 h = hashlib.sha256()
1982 h.update(os.urandom(16)) # salt
1983 h.update(socket.getfqdn().encode('utf-8'))
1984 passwd = base64.b64encode(h.digest()).strip()
1985 return passwd.decode('utf-8')
1988 def genkeystore(localconfig):
1989 '''Generate a new key with random passwords and add it to new keystore'''
1990 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
1991 keystoredir = os.path.dirname(localconfig['keystore'])
1992 if keystoredir is None or keystoredir == '':
1993 keystoredir = os.path.join(os.getcwd(), keystoredir)
1994 if not os.path.exists(keystoredir):
1995 os.makedirs(keystoredir, mode=0o700)
1997 write_password_file("keystorepass", localconfig['keystorepass'])
1998 write_password_file("keypass", localconfig['keypass'])
1999 p = FDroidPopen([config['keytool'], '-genkey',
2000 '-keystore', localconfig['keystore'],
2001 '-alias', localconfig['repo_keyalias'],
2002 '-keyalg', 'RSA', '-keysize', '4096',
2003 '-sigalg', 'SHA256withRSA',
2004 '-validity', '10000',
2005 '-storepass:file', config['keystorepassfile'],
2006 '-keypass:file', config['keypassfile'],
2007 '-dname', localconfig['keydname']])
2008 # TODO keypass should be sent via stdin
2009 if p.returncode != 0:
2010 raise BuildException("Failed to generate key", p.output)
2011 os.chmod(localconfig['keystore'], 0o0600)
2012 # now show the lovely key that was just generated
2013 p = FDroidPopen([config['keytool'], '-list', '-v',
2014 '-keystore', localconfig['keystore'],
2015 '-alias', localconfig['repo_keyalias'],
2016 '-storepass:file', config['keystorepassfile']])
2017 logging.info(p.output.strip() + '\n\n')
2020 def write_to_config(thisconfig, key, value=None):
2021 '''write a key/value to the local config.py'''
2023 origkey = key + '_orig'
2024 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2025 with open('config.py', 'r', encoding='utf8') as f:
2027 pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
2028 repl = '\n' + key + ' = "' + value + '"'
2029 data = re.sub(pattern, repl, data)
2030 # if this key is not in the file, append it
2031 if not re.match('\s*' + key + '\s*=\s*"', data):
2033 # make sure the file ends with a carraige return
2034 if not re.match('\n$', data):
2036 with open('config.py', 'w', encoding='utf8') as f:
2040 def parse_xml(path):
2041 return XMLElementTree.parse(path).getroot()
2044 def string_is_integer(string):
2052 def get_per_app_repos():
2053 '''per-app repos are dirs named with the packageName of a single app'''
2055 # Android packageNames are Java packages, they may contain uppercase or
2056 # lowercase letters ('A' through 'Z'), numbers, and underscores
2057 # ('_'). However, individual package name parts may only start with
2058 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2059 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2062 for root, dirs, files in os.walk(os.getcwd()):
2064 print('checking', root, 'for', d)
2065 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2066 # standard parts of an fdroid repo, so never packageNames
2069 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):