3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
37 import xml.etree.ElementTree as XMLElementTree
39 from queue import Queue
41 from zipfile import ZipFile
43 import fdroidserver.metadata
44 from .asynchronousfilereader import AsynchronousFileReader
47 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
56 'sdk_path': "$ANDROID_HOME",
59 'r10e': "$ANDROID_NDK",
61 'build_tools': "24.0.0",
62 'force_build_tools': False,
67 'accepted_formats': ['txt', 'yml'],
68 'sync_from_local_copy_dir': False,
69 'per_app_repos': False,
70 'make_current_version_link': True,
71 'current_version_name_source': 'Name',
72 'update_stats': False,
76 'stats_to_carbon': False,
78 'build_server_always': False,
79 'keystore': 'keystore.jks',
80 'smartcardoptions': [],
86 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
87 'repo_name': "My First FDroid Repo Demo",
88 'repo_icon': "fdroid-icon.png",
89 'repo_description': '''
90 This is a repository of apps to be used with FDroid. Applications in this
91 repository are either official binaries built by the original application
92 developers, or are binaries built from source by the admin of f-droid.org
93 using the tools on https://gitlab.com/u/fdroid.
99 def setup_global_opts(parser):
100 parser.add_argument("-v", "--verbose", action="store_true", default=False,
101 help="Spew out even more information than normal")
102 parser.add_argument("-q", "--quiet", action="store_true", default=False,
103 help="Restrict output to warnings and errors")
106 def fill_config_defaults(thisconfig):
107 for k, v in default_config.items():
108 if k not in thisconfig:
111 # Expand paths (~users and $vars)
112 def expand_path(path):
116 path = os.path.expanduser(path)
117 path = os.path.expandvars(path)
122 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
127 thisconfig[k + '_orig'] = v
129 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
130 if thisconfig['java_paths'] is None:
131 thisconfig['java_paths'] = dict()
133 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
134 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
135 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
136 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
137 if os.getenv('JAVA_HOME') is not None:
138 pathlist.append(os.getenv('JAVA_HOME'))
139 if os.getenv('PROGRAMFILES') is not None:
140 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
141 for d in sorted(pathlist):
142 if os.path.islink(d):
144 j = os.path.basename(d)
145 # the last one found will be the canonical one, so order appropriately
147 r'^1\.([6-9])\.0\.jdk$', # OSX
148 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
149 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
150 r'^jdk([6-9])-openjdk$', # Arch
151 r'^java-([6-9])-openjdk$', # Arch
152 r'^java-([6-9])-jdk$', # Arch (oracle)
153 r'^java-1\.([6-9])\.0-.*$', # RedHat
154 r'^java-([6-9])-oracle$', # Debian WebUpd8
155 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
156 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
158 m = re.match(regex, j)
161 for p in [d, os.path.join(d, 'Contents', 'Home')]:
162 if os.path.exists(os.path.join(p, 'bin', 'javac')):
163 thisconfig['java_paths'][m.group(1)] = p
165 for java_version in ('7', '8', '9'):
166 if java_version not in thisconfig['java_paths']:
168 java_home = thisconfig['java_paths'][java_version]
169 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
170 if os.path.exists(jarsigner):
171 thisconfig['jarsigner'] = jarsigner
172 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
173 break # Java7 is preferred, so quit if found
175 for k in ['ndk_paths', 'java_paths']:
181 thisconfig[k][k2] = exp
182 thisconfig[k][k2 + '_orig'] = v
185 def regsub_file(pattern, repl, path):
186 with open(path, 'rb') as f:
188 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
189 with open(path, 'wb') as f:
193 def read_config(opts, config_file='config.py'):
194 """Read the repository config
196 The config is read from config_file, which is in the current
197 directory when any of the repo management commands are used. If
198 there is a local metadata file in the git repo, then config.py is
199 not required, just use defaults.
202 global config, options
204 if config is not None:
211 if os.path.isfile(config_file):
212 logging.debug("Reading %s" % config_file)
213 with io.open(config_file, "rb") as f:
214 code = compile(f.read(), config_file, 'exec')
215 exec(code, None, config)
216 elif len(get_local_metadata_files()) == 0:
217 logging.critical("Missing config file - is this a repo directory?")
220 # smartcardoptions must be a list since its command line args for Popen
221 if 'smartcardoptions' in config:
222 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
223 elif 'keystore' in config and config['keystore'] == 'NONE':
224 # keystore='NONE' means use smartcard, these are required defaults
225 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
226 'SunPKCS11-OpenSC', '-providerClass',
227 'sun.security.pkcs11.SunPKCS11',
228 '-providerArg', 'opensc-fdroid.cfg']
230 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
231 st = os.stat(config_file)
232 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
233 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
235 fill_config_defaults(config)
237 for k in ["keystorepass", "keypass"]:
239 write_password_file(k)
241 for k in ["repo_description", "archive_description"]:
243 config[k] = clean_description(config[k])
245 if 'serverwebroot' in config:
246 if isinstance(config['serverwebroot'], str):
247 roots = [config['serverwebroot']]
248 elif all(isinstance(item, str) for item in config['serverwebroot']):
249 roots = config['serverwebroot']
251 raise TypeError('only accepts strings, lists, and tuples')
253 for rootstr in roots:
254 # since this is used with rsync, where trailing slashes have
255 # meaning, ensure there is always a trailing slash
256 if rootstr[-1] != '/':
258 rootlist.append(rootstr.replace('//', '/'))
259 config['serverwebroot'] = rootlist
264 def find_sdk_tools_cmd(cmd):
265 '''find a working path to a tool from the Android SDK'''
268 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
269 # try to find a working path to this command, in all the recent possible paths
270 if 'build_tools' in config:
271 build_tools = os.path.join(config['sdk_path'], 'build-tools')
272 # if 'build_tools' was manually set and exists, check only that one
273 configed_build_tools = os.path.join(build_tools, config['build_tools'])
274 if os.path.exists(configed_build_tools):
275 tooldirs.append(configed_build_tools)
277 # no configed version, so hunt known paths for it
278 for f in sorted(os.listdir(build_tools), reverse=True):
279 if os.path.isdir(os.path.join(build_tools, f)):
280 tooldirs.append(os.path.join(build_tools, f))
281 tooldirs.append(build_tools)
282 sdk_tools = os.path.join(config['sdk_path'], 'tools')
283 if os.path.exists(sdk_tools):
284 tooldirs.append(sdk_tools)
285 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
286 if os.path.exists(sdk_platform_tools):
287 tooldirs.append(sdk_platform_tools)
288 tooldirs.append('/usr/bin')
290 if os.path.isfile(os.path.join(d, cmd)):
291 return os.path.join(d, cmd)
292 # did not find the command, exit with error message
293 ensure_build_tools_exists(config)
296 def test_sdk_exists(thisconfig):
297 if 'sdk_path' not in thisconfig:
298 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
301 logging.error("'sdk_path' not set in config.py!")
303 if thisconfig['sdk_path'] == default_config['sdk_path']:
304 logging.error('No Android SDK found!')
305 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
306 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
308 if not os.path.exists(thisconfig['sdk_path']):
309 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
311 if not os.path.isdir(thisconfig['sdk_path']):
312 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
314 for d in ['build-tools', 'platform-tools', 'tools']:
315 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
316 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
317 thisconfig['sdk_path'], d))
322 def ensure_build_tools_exists(thisconfig):
323 if not test_sdk_exists(thisconfig):
325 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
326 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
327 if not os.path.isdir(versioned_build_tools):
328 logging.critical('Android Build Tools path "'
329 + versioned_build_tools + '" does not exist!')
333 def write_password_file(pwtype, password=None):
335 writes out passwords to a protected file instead of passing passwords as
336 command line argments
338 filename = '.fdroid.' + pwtype + '.txt'
339 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
341 os.write(fd, config[pwtype].encode('utf-8'))
343 os.write(fd, password.encode('utf-8'))
345 config[pwtype + 'file'] = filename
348 def get_local_metadata_files():
349 '''get any metadata files local to an app's source repo
351 This tries to ignore anything that does not count as app metdata,
352 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
355 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
358 # Given the arguments in the form of multiple appid:[vc] strings, this returns
359 # a dictionary with the set of vercodes specified for each package.
360 def read_pkg_args(args, allow_vercodes=False):
367 if allow_vercodes and ':' in p:
368 package, vercode = p.split(':')
370 package, vercode = p, None
371 if package not in vercodes:
372 vercodes[package] = [vercode] if vercode else []
374 elif vercode and vercode not in vercodes[package]:
375 vercodes[package] += [vercode] if vercode else []
380 # On top of what read_pkg_args does, this returns the whole app metadata, but
381 # limiting the builds list to the builds matching the vercodes specified.
382 def read_app_args(args, allapps, allow_vercodes=False):
384 vercodes = read_pkg_args(args, allow_vercodes)
390 for appid, app in allapps.items():
391 if appid in vercodes:
394 if len(apps) != len(vercodes):
397 logging.critical("No such package: %s" % p)
398 raise FDroidException("Found invalid app ids in arguments")
400 raise FDroidException("No packages specified")
403 for appid, app in apps.items():
407 app.builds = [b for b in app.builds if b.vercode in vc]
408 if len(app.builds) != len(vercodes[appid]):
410 allvcs = [b.vercode for b in app.builds]
411 for v in vercodes[appid]:
413 logging.critical("No such vercode %s for app %s" % (v, appid))
416 raise FDroidException("Found invalid vercodes for some apps")
421 def get_extension(filename):
422 base, ext = os.path.splitext(filename)
425 return base, ext.lower()[1:]
428 def has_extension(filename, ext):
429 _, f_ext = get_extension(filename)
433 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
436 def clean_description(description):
437 'Remove unneeded newlines and spaces from a block of description text'
439 # this is split up by paragraph to make removing the newlines easier
440 for paragraph in re.split(r'\n\n', description):
441 paragraph = re.sub('\r', '', paragraph)
442 paragraph = re.sub('\n', ' ', paragraph)
443 paragraph = re.sub(' {2,}', ' ', paragraph)
444 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
445 returnstring += paragraph + '\n\n'
446 return returnstring.rstrip('\n')
449 def apknameinfo(filename):
450 filename = os.path.basename(filename)
451 m = apk_regex.match(filename)
453 result = (m.group(1), m.group(2))
454 except AttributeError:
455 raise FDroidException("Invalid apk name: %s" % filename)
459 def getapkname(app, build):
460 return "%s_%s.apk" % (app.id, build.vercode)
463 def getsrcname(app, build):
464 return "%s_%s_src.tar.gz" % (app.id, build.vercode)
476 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
479 def getvcs(vcstype, remote, local):
481 return vcs_git(remote, local)
482 if vcstype == 'git-svn':
483 return vcs_gitsvn(remote, local)
485 return vcs_hg(remote, local)
487 return vcs_bzr(remote, local)
488 if vcstype == 'srclib':
489 if local != os.path.join('build', 'srclib', remote):
490 raise VCSException("Error: srclib paths are hard-coded!")
491 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
493 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
494 raise VCSException("Invalid vcs type " + vcstype)
497 def getsrclibvcs(name):
498 if name not in fdroidserver.metadata.srclibs:
499 raise VCSException("Missing srclib " + name)
500 return fdroidserver.metadata.srclibs[name]['Repo Type']
505 def __init__(self, remote, local):
507 # svn, git-svn and bzr may require auth
509 if self.repotype() in ('git-svn', 'bzr'):
511 if self.repotype == 'git-svn':
512 raise VCSException("Authentication is not supported for git-svn")
513 self.username, remote = remote.split('@')
514 if ':' not in self.username:
515 raise VCSException("Password required with username")
516 self.username, self.password = self.username.split(':')
520 self.clone_failed = False
521 self.refreshed = False
527 # Take the local repository to a clean version of the given revision, which
528 # is specificed in the VCS's native format. Beforehand, the repository can
529 # be dirty, or even non-existent. If the repository does already exist
530 # locally, it will be updated from the origin, but only once in the
531 # lifetime of the vcs object.
532 # None is acceptable for 'rev' if you know you are cloning a clean copy of
533 # the repo - otherwise it must specify a valid revision.
534 def gotorevision(self, rev, refresh=True):
536 if self.clone_failed:
537 raise VCSException("Downloading the repository already failed once, not trying again.")
539 # The .fdroidvcs-id file for a repo tells us what VCS type
540 # and remote that directory was created from, allowing us to drop it
541 # automatically if either of those things changes.
542 fdpath = os.path.join(self.local, '..',
543 '.fdroidvcs-' + os.path.basename(self.local))
544 fdpath = os.path.normpath(fdpath)
545 cdata = self.repotype() + ' ' + self.remote
548 if os.path.exists(self.local):
549 if os.path.exists(fdpath):
550 with open(fdpath, 'r') as f:
551 fsdata = f.read().strip()
556 logging.info("Repository details for %s changed - deleting" % (
560 logging.info("Repository details for %s missing - deleting" % (
563 shutil.rmtree(self.local)
567 self.refreshed = True
570 self.gotorevisionx(rev)
571 except FDroidException as e:
574 # If necessary, write the .fdroidvcs file.
575 if writeback and not self.clone_failed:
576 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
577 with open(fdpath, 'w+') as f:
583 # Derived classes need to implement this. It's called once basic checking
584 # has been performend.
585 def gotorevisionx(self, rev):
586 raise VCSException("This VCS type doesn't define gotorevisionx")
588 # Initialise and update submodules
589 def initsubmodules(self):
590 raise VCSException('Submodules not supported for this vcs type')
592 # Get a list of all known tags
594 if not self._gettags:
595 raise VCSException('gettags not supported for this vcs type')
597 for tag in self._gettags():
598 if re.match('[-A-Za-z0-9_. /]+$', tag):
602 # Get a list of all the known tags, sorted from newest to oldest
603 def latesttags(self):
604 raise VCSException('latesttags not supported for this vcs type')
606 # Get current commit reference (hash, revision, etc)
608 raise VCSException('getref not supported for this vcs type')
610 # Returns the srclib (name, path) used in setting up the current
621 # If the local directory exists, but is somehow not a git repository, git
622 # will traverse up the directory tree until it finds one that is (i.e.
623 # fdroidserver) and then we'll proceed to destroy it! This is called as
626 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
627 result = p.output.rstrip()
628 if not result.endswith(self.local):
629 raise VCSException('Repository mismatch')
631 def gotorevisionx(self, rev):
632 if not os.path.exists(self.local):
634 p = FDroidPopen(['git', 'clone', self.remote, self.local])
635 if p.returncode != 0:
636 self.clone_failed = True
637 raise VCSException("Git clone failed", p.output)
641 # Discard any working tree changes
642 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
643 'git', 'reset', '--hard'], cwd=self.local, output=False)
644 if p.returncode != 0:
645 raise VCSException("Git reset failed", p.output)
646 # Remove untracked files now, in case they're tracked in the target
647 # revision (it happens!)
648 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
649 'git', 'clean', '-dffx'], cwd=self.local, output=False)
650 if p.returncode != 0:
651 raise VCSException("Git clean failed", p.output)
652 if not self.refreshed:
653 # Get latest commits and tags from remote
654 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
655 if p.returncode != 0:
656 raise VCSException("Git fetch failed", p.output)
657 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
658 if p.returncode != 0:
659 raise VCSException("Git fetch failed", p.output)
660 # Recreate origin/HEAD as git clone would do it, in case it disappeared
661 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
662 if p.returncode != 0:
663 lines = p.output.splitlines()
664 if 'Multiple remote HEAD branches' not in lines[0]:
665 raise VCSException("Git remote set-head failed", p.output)
666 branch = lines[1].split(' ')[-1]
667 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
668 if p2.returncode != 0:
669 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
670 self.refreshed = True
671 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
672 # a github repo. Most of the time this is the same as origin/master.
673 rev = rev or 'origin/HEAD'
674 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
675 if p.returncode != 0:
676 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
677 # Get rid of any uncontrolled files left behind
678 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
679 if p.returncode != 0:
680 raise VCSException("Git clean failed", p.output)
682 def initsubmodules(self):
684 submfile = os.path.join(self.local, '.gitmodules')
685 if not os.path.isfile(submfile):
686 raise VCSException("No git submodules available")
688 # fix submodules not accessible without an account and public key auth
689 with open(submfile, 'r') as f:
690 lines = f.readlines()
691 with open(submfile, 'w') as f:
693 if 'git@github.com' in line:
694 line = line.replace('git@github.com:', 'https://github.com/')
695 if 'git@gitlab.com' in line:
696 line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
699 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
700 if p.returncode != 0:
701 raise VCSException("Git submodule sync failed", p.output)
702 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
703 if p.returncode != 0:
704 raise VCSException("Git submodule update failed", p.output)
708 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
709 return p.output.splitlines()
711 tag_format = re.compile(r'tag: ([^),]*)')
713 def latesttags(self):
715 p = FDroidPopen(['git', 'log', '--tags',
716 '--simplify-by-decoration', '--pretty=format:%d'],
717 cwd=self.local, output=False)
719 for line in p.output.splitlines():
720 for tag in self.tag_format.findall(line):
725 class vcs_gitsvn(vcs):
730 # If the local directory exists, but is somehow not a git repository, git
731 # will traverse up the directory tree until it finds one that is (i.e.
732 # fdroidserver) and then we'll proceed to destory it! This is called as
735 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
736 result = p.output.rstrip()
737 if not result.endswith(self.local):
738 raise VCSException('Repository mismatch')
740 def gotorevisionx(self, rev):
741 if not os.path.exists(self.local):
743 gitsvn_args = ['git', 'svn', 'clone']
744 if ';' in self.remote:
745 remote_split = self.remote.split(';')
746 for i in remote_split[1:]:
747 if i.startswith('trunk='):
748 gitsvn_args.extend(['-T', i[6:]])
749 elif i.startswith('tags='):
750 gitsvn_args.extend(['-t', i[5:]])
751 elif i.startswith('branches='):
752 gitsvn_args.extend(['-b', i[9:]])
753 gitsvn_args.extend([remote_split[0], self.local])
754 p = FDroidPopen(gitsvn_args, output=False)
755 if p.returncode != 0:
756 self.clone_failed = True
757 raise VCSException("Git svn clone failed", p.output)
759 gitsvn_args.extend([self.remote, self.local])
760 p = FDroidPopen(gitsvn_args, output=False)
761 if p.returncode != 0:
762 self.clone_failed = True
763 raise VCSException("Git svn clone failed", p.output)
767 # Discard any working tree changes
768 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
769 if p.returncode != 0:
770 raise VCSException("Git reset failed", p.output)
771 # Remove untracked files now, in case they're tracked in the target
772 # revision (it happens!)
773 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
774 if p.returncode != 0:
775 raise VCSException("Git clean failed", p.output)
776 if not self.refreshed:
777 # Get new commits, branches and tags from repo
778 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
779 if p.returncode != 0:
780 raise VCSException("Git svn fetch failed")
781 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
782 if p.returncode != 0:
783 raise VCSException("Git svn rebase failed", p.output)
784 self.refreshed = True
786 rev = rev or 'master'
788 nospaces_rev = rev.replace(' ', '%20')
789 # Try finding a svn tag
790 for treeish in ['origin/', '']:
791 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
792 if p.returncode == 0:
794 if p.returncode != 0:
795 # No tag found, normal svn rev translation
796 # Translate svn rev into git format
797 rev_split = rev.split('/')
800 for treeish in ['origin/', '']:
801 if len(rev_split) > 1:
802 treeish += rev_split[0]
803 svn_rev = rev_split[1]
806 # if no branch is specified, then assume trunk (i.e. 'master' branch):
810 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
812 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
813 git_rev = p.output.rstrip()
815 if p.returncode == 0 and git_rev:
818 if p.returncode != 0 or not git_rev:
819 # Try a plain git checkout as a last resort
820 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
821 if p.returncode != 0:
822 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
824 # Check out the git rev equivalent to the svn rev
825 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
826 if p.returncode != 0:
827 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
829 # Get rid of any uncontrolled files left behind
830 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
831 if p.returncode != 0:
832 raise VCSException("Git clean failed", p.output)
836 for treeish in ['origin/', '']:
837 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
843 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
844 if p.returncode != 0:
846 return p.output.strip()
854 def gotorevisionx(self, rev):
855 if not os.path.exists(self.local):
856 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
857 if p.returncode != 0:
858 self.clone_failed = True
859 raise VCSException("Hg clone failed", p.output)
861 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
862 if p.returncode != 0:
863 raise VCSException("Hg status failed", p.output)
864 for line in p.output.splitlines():
865 if not line.startswith('? '):
866 raise VCSException("Unexpected output from hg status -uS: " + line)
867 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
868 if not self.refreshed:
869 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
870 if p.returncode != 0:
871 raise VCSException("Hg pull failed", p.output)
872 self.refreshed = True
874 rev = rev or 'default'
877 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
878 if p.returncode != 0:
879 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
880 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
881 # Also delete untracked files, we have to enable purge extension for that:
882 if "'purge' is provided by the following extension" in p.output:
883 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
884 myfile.write("\n[extensions]\nhgext.purge=\n")
885 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
886 if p.returncode != 0:
887 raise VCSException("HG purge failed", p.output)
888 elif p.returncode != 0:
889 raise VCSException("HG purge failed", p.output)
892 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
893 return p.output.splitlines()[1:]
901 def gotorevisionx(self, rev):
902 if not os.path.exists(self.local):
903 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
904 if p.returncode != 0:
905 self.clone_failed = True
906 raise VCSException("Bzr branch failed", p.output)
908 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
909 if p.returncode != 0:
910 raise VCSException("Bzr revert failed", p.output)
911 if not self.refreshed:
912 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
913 if p.returncode != 0:
914 raise VCSException("Bzr update failed", p.output)
915 self.refreshed = True
917 revargs = list(['-r', rev] if rev else [])
918 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
919 if p.returncode != 0:
920 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
923 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
924 return [tag.split(' ')[0].strip() for tag in
925 p.output.splitlines()]
928 def unescape_string(string):
931 if string[0] == '"' and string[-1] == '"':
934 return string.replace("\\'", "'")
937 def retrieve_string(app_dir, string, xmlfiles=None):
939 if not string.startswith('@string/'):
940 return unescape_string(string)
945 os.path.join(app_dir, 'res'),
946 os.path.join(app_dir, 'src', 'main', 'res'),
948 for r, d, f in os.walk(res_dir):
949 if os.path.basename(r) == 'values':
950 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
952 name = string[len('@string/'):]
954 def element_content(element):
955 if element.text is None:
957 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
958 return s.decode('utf-8').strip()
960 for path in xmlfiles:
961 if not os.path.isfile(path):
963 xml = parse_xml(path)
964 element = xml.find('string[@name="' + name + '"]')
965 if element is not None:
966 content = element_content(element)
967 return retrieve_string(app_dir, content, xmlfiles)
972 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
973 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
976 # Return list of existing files that will be used to find the highest vercode
977 def manifest_paths(app_dir, flavours):
979 possible_manifests = \
980 [os.path.join(app_dir, 'AndroidManifest.xml'),
981 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
982 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
983 os.path.join(app_dir, 'build.gradle')]
985 for flavour in flavours:
988 possible_manifests.append(
989 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
991 return [path for path in possible_manifests if os.path.isfile(path)]
994 # Retrieve the package name. Returns the name, or None if not found.
995 def fetch_real_name(app_dir, flavours):
996 for path in manifest_paths(app_dir, flavours):
997 if not has_extension(path, 'xml') or not os.path.isfile(path):
999 logging.debug("fetch_real_name: Checking manifest at " + path)
1000 xml = parse_xml(path)
1001 app = xml.find('application')
1004 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1006 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1007 result = retrieve_string_singleline(app_dir, label)
1009 result = result.strip()
1014 def get_library_references(root_dir):
1016 proppath = os.path.join(root_dir, 'project.properties')
1017 if not os.path.isfile(proppath):
1019 with open(proppath, 'r', encoding='iso-8859-1') as f:
1021 if not line.startswith('android.library.reference.'):
1023 path = line.split('=')[1].strip()
1024 relpath = os.path.join(root_dir, path)
1025 if not os.path.isdir(relpath):
1027 logging.debug("Found subproject at %s" % path)
1028 libraries.append(path)
1032 def ant_subprojects(root_dir):
1033 subprojects = get_library_references(root_dir)
1034 for subpath in subprojects:
1035 subrelpath = os.path.join(root_dir, subpath)
1036 for p in get_library_references(subrelpath):
1037 relp = os.path.normpath(os.path.join(subpath, p))
1038 if relp not in subprojects:
1039 subprojects.insert(0, relp)
1043 def remove_debuggable_flags(root_dir):
1044 # Remove forced debuggable flags
1045 logging.debug("Removing debuggable flags from %s" % root_dir)
1046 for root, dirs, files in os.walk(root_dir):
1047 if 'AndroidManifest.xml' in files:
1048 regsub_file(r'android:debuggable="[^"]*"',
1050 os.path.join(root, 'AndroidManifest.xml'))
1053 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
1054 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1055 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1058 def app_matches_packagename(app, package):
1061 appid = app.UpdateCheckName or app.id
1062 if appid is None or appid == "Ignore":
1064 return appid == package
1067 # Extract some information from the AndroidManifest.xml at the given path.
1068 # Returns (version, vercode, package), any or all of which might be None.
1069 # All values returned are strings.
1070 def parse_androidmanifests(paths, app):
1072 ignoreversions = app.UpdateCheckIgnore
1073 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1076 return (None, None, None)
1084 if not os.path.isfile(path):
1087 logging.debug("Parsing manifest at {0}".format(path))
1088 gradle = has_extension(path, 'gradle')
1094 with open(path, 'r') as f:
1096 if gradle_comment.match(line):
1098 # Grab first occurence of each to avoid running into
1099 # alternative flavours and builds.
1101 matches = psearch_g(line)
1103 s = matches.group(2)
1104 if app_matches_packagename(app, s):
1107 matches = vnsearch_g(line)
1109 version = matches.group(2)
1111 matches = vcsearch_g(line)
1113 vercode = matches.group(1)
1116 xml = parse_xml(path)
1117 if "package" in xml.attrib:
1118 s = xml.attrib["package"]
1119 if app_matches_packagename(app, s):
1121 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1122 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1123 base_dir = os.path.dirname(path)
1124 version = retrieve_string_singleline(base_dir, version)
1125 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1126 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1127 if string_is_integer(a):
1130 logging.warning("Problem with xml at {0}".format(path))
1132 # Remember package name, may be defined separately from version+vercode
1134 package = max_package
1136 logging.debug("..got package={0}, version={1}, vercode={2}"
1137 .format(package, version, vercode))
1139 # Always grab the package name and version name in case they are not
1140 # together with the highest version code
1141 if max_package is None and package is not None:
1142 max_package = package
1143 if max_version is None and version is not None:
1144 max_version = version
1146 if max_vercode is None or (vercode is not None and vercode > max_vercode):
1147 if not ignoresearch or not ignoresearch(version):
1148 if version is not None:
1149 max_version = version
1150 if vercode is not None:
1151 max_vercode = vercode
1152 if package is not None:
1153 max_package = package
1155 max_version = "Ignore"
1157 if max_version is None:
1158 max_version = "Unknown"
1160 if max_package and not is_valid_package_name(max_package):
1161 raise FDroidException("Invalid package name {0}".format(max_package))
1163 return (max_version, max_vercode, max_package)
1166 def is_valid_package_name(name):
1167 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1170 class FDroidException(Exception):
1172 def __init__(self, value, detail=None):
1174 self.detail = detail
1176 def shortened_detail(self):
1177 if len(self.detail) < 16000:
1179 return '[...]\n' + self.detail[-16000:]
1181 def get_wikitext(self):
1182 ret = repr(self.value) + "\n"
1185 ret += "<pre>\n" + self.shortened_detail() + "</pre>\n"
1191 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1195 class VCSException(FDroidException):
1199 class BuildException(FDroidException):
1203 # Get the specified source library.
1204 # Returns the path to it. Normally this is the path to be used when referencing
1205 # it, which may be a subdirectory of the actual project. If you want the base
1206 # directory of the project, pass 'basepath=True'.
1207 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1208 raw=False, prepare=True, preponly=False, refresh=True,
1217 name, ref = spec.split('@')
1219 number, name = name.split(':', 1)
1221 name, subdir = name.split('/', 1)
1223 if name not in fdroidserver.metadata.srclibs:
1224 raise VCSException('srclib ' + name + ' not found.')
1226 srclib = fdroidserver.metadata.srclibs[name]
1228 sdir = os.path.join(srclib_dir, name)
1231 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1232 vcs.srclib = (name, number, sdir)
1234 vcs.gotorevision(ref, refresh)
1241 libdir = os.path.join(sdir, subdir)
1242 elif srclib["Subdir"]:
1243 for subdir in srclib["Subdir"]:
1244 libdir_candidate = os.path.join(sdir, subdir)
1245 if os.path.exists(libdir_candidate):
1246 libdir = libdir_candidate
1252 remove_signing_keys(sdir)
1253 remove_debuggable_flags(sdir)
1257 if srclib["Prepare"]:
1258 cmd = replace_config_vars(srclib["Prepare"], build)
1260 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1261 if p.returncode != 0:
1262 raise BuildException("Error running prepare command for srclib %s"
1268 return (name, number, libdir)
1270 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1273 # Prepare the source code for a particular build
1274 # 'vcs' - the appropriate vcs object for the application
1275 # 'app' - the application details from the metadata
1276 # 'build' - the build details from the metadata
1277 # 'build_dir' - the path to the build directory, usually
1279 # 'srclib_dir' - the path to the source libraries directory, usually
1281 # 'extlib_dir' - the path to the external libraries directory, usually
1283 # Returns the (root, srclibpaths) where:
1284 # 'root' is the root directory, which may be the same as 'build_dir' or may
1285 # be a subdirectory of it.
1286 # 'srclibpaths' is information on the srclibs being used
1287 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1289 # Optionally, the actual app source can be in a subdirectory
1291 root_dir = os.path.join(build_dir, build.subdir)
1293 root_dir = build_dir
1295 # Get a working copy of the right revision
1296 logging.info("Getting source for revision " + build.commit)
1297 vcs.gotorevision(build.commit, refresh)
1299 # Initialise submodules if required
1300 if build.submodules:
1301 logging.info("Initialising submodules")
1302 vcs.initsubmodules()
1304 # Check that a subdir (if we're using one) exists. This has to happen
1305 # after the checkout, since it might not exist elsewhere
1306 if not os.path.exists(root_dir):
1307 raise BuildException('Missing subdir ' + root_dir)
1309 # Run an init command if one is required
1311 cmd = replace_config_vars(build.init, build)
1312 logging.info("Running 'init' commands in %s" % root_dir)
1314 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1315 if p.returncode != 0:
1316 raise BuildException("Error running init command for %s:%s" %
1317 (app.id, build.version), p.output)
1319 # Apply patches if any
1321 logging.info("Applying patches")
1322 for patch in build.patch:
1323 patch = patch.strip()
1324 logging.info("Applying " + patch)
1325 patch_path = os.path.join('metadata', app.id, patch)
1326 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1327 if p.returncode != 0:
1328 raise BuildException("Failed to apply patch %s" % patch_path)
1330 # Get required source libraries
1333 logging.info("Collecting source libraries")
1334 for lib in build.srclibs:
1335 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1336 refresh=refresh, build=build))
1338 for name, number, libpath in srclibpaths:
1339 place_srclib(root_dir, int(number) if number else None, libpath)
1341 basesrclib = vcs.getsrclib()
1342 # If one was used for the main source, add that too.
1344 srclibpaths.append(basesrclib)
1346 # Update the local.properties file
1347 localprops = [os.path.join(build_dir, 'local.properties')]
1349 parts = build.subdir.split(os.sep)
1352 cur = os.path.join(cur, d)
1353 localprops += [os.path.join(cur, 'local.properties')]
1354 for path in localprops:
1356 if os.path.isfile(path):
1357 logging.info("Updating local.properties file at %s" % path)
1358 with open(path, 'r', encoding='iso-8859-1') as f:
1362 logging.info("Creating local.properties file at %s" % path)
1363 # Fix old-fashioned 'sdk-location' by copying
1364 # from sdk.dir, if necessary
1366 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1367 re.S | re.M).group(1)
1368 props += "sdk-location=%s\n" % sdkloc
1370 props += "sdk.dir=%s\n" % config['sdk_path']
1371 props += "sdk-location=%s\n" % config['sdk_path']
1372 ndk_path = build.ndk_path()
1373 # if for any reason the path isn't valid or the directory
1374 # doesn't exist, some versions of Gradle will error with a
1375 # cryptic message (even if the NDK is not even necessary).
1376 # https://gitlab.com/fdroid/fdroidserver/issues/171
1377 if ndk_path and os.path.exists(ndk_path):
1379 props += "ndk.dir=%s\n" % ndk_path
1380 props += "ndk-location=%s\n" % ndk_path
1381 # Add java.encoding if necessary
1383 props += "java.encoding=%s\n" % build.encoding
1384 with open(path, 'w', encoding='iso-8859-1') as f:
1388 if build.build_method() == 'gradle':
1389 flavours = build.gradle
1392 n = build.target.split('-')[1]
1393 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1394 r'compileSdkVersion %s' % n,
1395 os.path.join(root_dir, 'build.gradle'))
1397 # Remove forced debuggable flags
1398 remove_debuggable_flags(root_dir)
1400 # Insert version code and number into the manifest if necessary
1401 if build.forceversion:
1402 logging.info("Changing the version name")
1403 for path in manifest_paths(root_dir, flavours):
1404 if not os.path.isfile(path):
1406 if has_extension(path, 'xml'):
1407 regsub_file(r'android:versionName="[^"]*"',
1408 r'android:versionName="%s"' % build.version,
1410 elif has_extension(path, 'gradle'):
1411 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1412 r"""\1versionName '%s'""" % build.version,
1415 if build.forcevercode:
1416 logging.info("Changing the version code")
1417 for path in manifest_paths(root_dir, flavours):
1418 if not os.path.isfile(path):
1420 if has_extension(path, 'xml'):
1421 regsub_file(r'android:versionCode="[^"]*"',
1422 r'android:versionCode="%s"' % build.vercode,
1424 elif has_extension(path, 'gradle'):
1425 regsub_file(r'versionCode[ =]+[0-9]+',
1426 r'versionCode %s' % build.vercode,
1429 # Delete unwanted files
1431 logging.info("Removing specified files")
1432 for part in getpaths(build_dir, build.rm):
1433 dest = os.path.join(build_dir, part)
1434 logging.info("Removing {0}".format(part))
1435 if os.path.lexists(dest):
1436 if os.path.islink(dest):
1437 FDroidPopen(['unlink', dest], output=False)
1439 FDroidPopen(['rm', '-rf', dest], output=False)
1441 logging.info("...but it didn't exist")
1443 remove_signing_keys(build_dir)
1445 # Add required external libraries
1447 logging.info("Collecting prebuilt libraries")
1448 libsdir = os.path.join(root_dir, 'libs')
1449 if not os.path.exists(libsdir):
1451 for lib in build.extlibs:
1453 logging.info("...installing extlib {0}".format(lib))
1454 libf = os.path.basename(lib)
1455 libsrc = os.path.join(extlib_dir, lib)
1456 if not os.path.exists(libsrc):
1457 raise BuildException("Missing extlib file {0}".format(libsrc))
1458 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1460 # Run a pre-build command if one is required
1462 logging.info("Running 'prebuild' commands in %s" % root_dir)
1464 cmd = replace_config_vars(build.prebuild, build)
1466 # Substitute source library paths into prebuild commands
1467 for name, number, libpath in srclibpaths:
1468 libpath = os.path.relpath(libpath, root_dir)
1469 cmd = cmd.replace('$$' + name + '$$', libpath)
1471 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1472 if p.returncode != 0:
1473 raise BuildException("Error running prebuild command for %s:%s" %
1474 (app.id, build.version), p.output)
1476 # Generate (or update) the ant build file, build.xml...
1477 if build.build_method() == 'ant' and build.update != ['no']:
1478 parms = ['android', 'update', 'lib-project']
1479 lparms = ['android', 'update', 'project']
1482 parms += ['-t', build.target]
1483 lparms += ['-t', build.target]
1485 update_dirs = build.update
1487 update_dirs = ant_subprojects(root_dir) + ['.']
1489 for d in update_dirs:
1490 subdir = os.path.join(root_dir, d)
1492 logging.debug("Updating main project")
1493 cmd = parms + ['-p', d]
1495 logging.debug("Updating subproject %s" % d)
1496 cmd = lparms + ['-p', d]
1497 p = SdkToolsPopen(cmd, cwd=root_dir)
1498 # Check to see whether an error was returned without a proper exit
1499 # code (this is the case for the 'no target set or target invalid'
1501 if p.returncode != 0 or p.output.startswith("Error: "):
1502 raise BuildException("Failed to update project at %s" % d, p.output)
1503 # Clean update dirs via ant
1505 logging.info("Cleaning subproject %s" % d)
1506 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1508 return (root_dir, srclibpaths)
1511 # Extend via globbing the paths from a field and return them as a map from
1512 # original path to resulting paths
1513 def getpaths_map(build_dir, globpaths):
1517 full_path = os.path.join(build_dir, p)
1518 full_path = os.path.normpath(full_path)
1519 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1521 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1525 # Extend via globbing the paths from a field and return them as a set
1526 def getpaths(build_dir, globpaths):
1527 paths_map = getpaths_map(build_dir, globpaths)
1529 for k, v in paths_map.items():
1536 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1542 self.path = os.path.join('stats', 'known_apks.txt')
1544 if os.path.isfile(self.path):
1545 with open(self.path, 'r', encoding='utf8') as f:
1547 t = line.rstrip().split(' ')
1549 self.apks[t[0]] = (t[1], None)
1551 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1552 self.changed = False
1554 def writeifchanged(self):
1555 if not self.changed:
1558 if not os.path.exists('stats'):
1562 for apk, app in self.apks.items():
1564 line = apk + ' ' + appid
1566 line += ' ' + time.strftime('%Y-%m-%d', added)
1569 with open(self.path, 'w', encoding='utf8') as f:
1570 for line in sorted(lst, key=natural_key):
1571 f.write(line + '\n')
1573 # Record an apk (if it's new, otherwise does nothing)
1574 # Returns the date it was added.
1575 def recordapk(self, apk, app, default_date=None):
1576 if apk not in self.apks:
1577 if default_date is None:
1578 default_date = time.gmtime(time.time())
1579 self.apks[apk] = (app, default_date)
1581 _, added = self.apks[apk]
1584 # Look up information - given the 'apkname', returns (app id, date added/None).
1585 # Or returns None for an unknown apk.
1586 def getapp(self, apkname):
1587 if apkname in self.apks:
1588 return self.apks[apkname]
1591 # Get the most recent 'num' apps added to the repo, as a list of package ids
1592 # with the most recent first.
1593 def getlatest(self, num):
1595 for apk, app in self.apks.items():
1599 if apps[appid] > added:
1603 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1604 lst = [app for app, _ in sortedapps]
1609 def isApkDebuggable(apkfile, config):
1610 """Returns True if the given apk file is debuggable
1612 :param apkfile: full path to the apk to check"""
1614 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1616 if p.returncode != 0:
1617 logging.critical("Failed to get apk manifest information")
1619 for line in p.output.splitlines():
1620 if 'android:debuggable' in line and not line.endswith('0x0'):
1627 self.returncode = None
1631 def SdkToolsPopen(commands, cwd=None, output=True):
1633 if cmd not in config:
1634 config[cmd] = find_sdk_tools_cmd(commands[0])
1635 abscmd = config[cmd]
1637 logging.critical("Could not find '%s' on your system" % cmd)
1639 return FDroidPopen([abscmd] + commands[1:],
1640 cwd=cwd, output=output)
1643 def FDroidPopenBytes(commands, cwd=None, output=True, stderr_to_stdout=True):
1645 Run a command and capture the possibly huge output as bytes.
1647 :param commands: command and argument list like in subprocess.Popen
1648 :param cwd: optionally specifies a working directory
1649 :returns: A PopenResult.
1654 set_FDroidPopen_env()
1657 cwd = os.path.normpath(cwd)
1658 logging.debug("Directory: %s" % cwd)
1659 logging.debug("> %s" % ' '.join(commands))
1661 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
1662 result = PopenResult()
1665 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
1666 stdout=subprocess.PIPE, stderr=stderr_param)
1667 except OSError as e:
1668 raise BuildException("OSError while trying to execute " +
1669 ' '.join(commands) + ': ' + str(e))
1671 if not stderr_to_stdout and options.verbose:
1672 stderr_queue = Queue()
1673 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
1675 while not stderr_reader.eof():
1676 while not stderr_queue.empty():
1677 line = stderr_queue.get()
1678 sys.stderr.buffer.write(line)
1683 stdout_queue = Queue()
1684 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1687 # Check the queue for output (until there is no more to get)
1688 while not stdout_reader.eof():
1689 while not stdout_queue.empty():
1690 line = stdout_queue.get()
1691 if output and options.verbose:
1692 # Output directly to console
1693 sys.stderr.buffer.write(line)
1699 result.returncode = p.wait()
1700 result.output = buf.getvalue()
1705 def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
1707 Run a command and capture the possibly huge output as a str.
1709 :param commands: command and argument list like in subprocess.Popen
1710 :param cwd: optionally specifies a working directory
1711 :returns: A PopenResult.
1713 result = FDroidPopenBytes(commands, cwd, output, stderr_to_stdout)
1714 result.output = result.output.decode('utf-8')
1718 gradle_comment = re.compile(r'[ ]*//')
1719 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1720 gradle_line_matches = [
1721 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1722 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1723 re.compile(r'.*\.readLine\(.*'),
1727 def remove_signing_keys(build_dir):
1728 for root, dirs, files in os.walk(build_dir):
1729 if 'build.gradle' in files:
1730 path = os.path.join(root, 'build.gradle')
1732 with open(path, "r", encoding='utf8') as o:
1733 lines = o.readlines()
1739 with open(path, "w", encoding='utf8') as o:
1740 while i < len(lines):
1743 while line.endswith('\\\n'):
1744 line = line.rstrip('\\\n') + lines[i]
1747 if gradle_comment.match(line):
1752 opened += line.count('{')
1753 opened -= line.count('}')
1756 if gradle_signing_configs.match(line):
1761 if any(s.match(line) for s in gradle_line_matches):
1769 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1772 'project.properties',
1774 'default.properties',
1775 'ant.properties', ]:
1776 if propfile in files:
1777 path = os.path.join(root, propfile)
1779 with open(path, "r", encoding='iso-8859-1') as o:
1780 lines = o.readlines()
1784 with open(path, "w", encoding='iso-8859-1') as o:
1786 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1793 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1796 def set_FDroidPopen_env(build=None):
1798 set up the environment variables for the build environment
1800 There is only a weak standard, the variables used by gradle, so also set
1801 up the most commonly used environment variables for SDK and NDK. Also, if
1802 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
1804 global env, orig_path
1808 orig_path = env['PATH']
1809 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
1810 env[n] = config['sdk_path']
1811 for k, v in config['java_paths'].items():
1812 env['JAVA%s_HOME' % k] = v
1814 missinglocale = True
1815 for k, v in env.items():
1816 if k == 'LANG' and v != 'C':
1817 missinglocale = False
1819 missinglocale = False
1821 env['LANG'] = 'en_US.UTF-8'
1823 if build is not None:
1824 path = build.ndk_path()
1825 paths = orig_path.split(os.pathsep)
1826 if path not in paths:
1827 paths = [path] + paths
1828 env['PATH'] = os.pathsep.join(paths)
1829 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
1830 env[n] = build.ndk_path()
1833 def replace_config_vars(cmd, build):
1834 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1835 cmd = cmd.replace('$$NDK$$', build.ndk_path())
1836 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1837 if build is not None:
1838 cmd = cmd.replace('$$COMMIT$$', build.commit)
1839 cmd = cmd.replace('$$VERSION$$', build.version)
1840 cmd = cmd.replace('$$VERCODE$$', build.vercode)
1844 def place_srclib(root_dir, number, libpath):
1847 relpath = os.path.relpath(libpath, root_dir)
1848 proppath = os.path.join(root_dir, 'project.properties')
1851 if os.path.isfile(proppath):
1852 with open(proppath, "r", encoding='iso-8859-1') as o:
1853 lines = o.readlines()
1855 with open(proppath, "w", encoding='iso-8859-1') as o:
1858 if line.startswith('android.library.reference.%d=' % number):
1859 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1864 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1866 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
1869 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
1870 """Verify that two apks are the same
1872 One of the inputs is signed, the other is unsigned. The signature metadata
1873 is transferred from the signed to the unsigned apk, and then jarsigner is
1874 used to verify that the signature from the signed apk is also varlid for
1876 :param signed_apk: Path to a signed apk file
1877 :param unsigned_apk: Path to an unsigned apk file expected to match it
1878 :param tmp_dir: Path to directory for temporary files
1879 :returns: None if the verification is successful, otherwise a string
1880 describing what went wrong.
1882 with ZipFile(signed_apk) as signed_apk_as_zip:
1883 meta_inf_files = ['META-INF/MANIFEST.MF']
1884 for f in signed_apk_as_zip.namelist():
1885 if apk_sigfile.match(f):
1886 meta_inf_files.append(f)
1887 if len(meta_inf_files) < 3:
1888 return "Signature files missing from {0}".format(signed_apk)
1889 signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
1890 with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
1891 for meta_inf_file in meta_inf_files:
1892 unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
1894 if subprocess.call([config['jarsigner'], '-verify', unsigned_apk]) != 0:
1895 logging.info("...NOT verified - {0}".format(signed_apk))
1896 return compare_apks(signed_apk, unsigned_apk, tmp_dir)
1897 logging.info("...successfully verified")
1900 apk_badchars = re.compile('''[/ :;'"]''')
1903 def compare_apks(apk1, apk2, tmp_dir):
1906 Returns None if the apk content is the same (apart from the signing key),
1907 otherwise a string describing what's different, or what went wrong when
1908 trying to do the comparison.
1911 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
1912 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
1913 for d in [apk1dir, apk2dir]:
1914 if os.path.exists(d):
1917 os.mkdir(os.path.join(d, 'jar-xf'))
1919 if subprocess.call(['jar', 'xf',
1920 os.path.abspath(apk1)],
1921 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
1922 return("Failed to unpack " + apk1)
1923 if subprocess.call(['jar', 'xf',
1924 os.path.abspath(apk2)],
1925 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
1926 return("Failed to unpack " + apk2)
1928 # try to find apktool in the path, if it hasn't been manually configed
1929 if 'apktool' not in config:
1930 tmp = find_command('apktool')
1932 config['apktool'] = tmp
1933 if 'apktool' in config:
1934 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
1936 return("Failed to unpack " + apk1)
1937 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
1939 return("Failed to unpack " + apk2)
1941 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
1942 lines = p.output.splitlines()
1943 if len(lines) != 1 or 'META-INF' not in lines[0]:
1944 meld = find_command('meld')
1945 if meld is not None:
1946 p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
1947 return("Unexpected diff output - " + p.output)
1949 # since everything verifies, delete the comparison to keep cruft down
1950 shutil.rmtree(apk1dir)
1951 shutil.rmtree(apk2dir)
1953 # If we get here, it seems like they're the same!
1957 def find_command(command):
1958 '''find the full path of a command, or None if it can't be found in the PATH'''
1961 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
1963 fpath, fname = os.path.split(command)
1968 for path in os.environ["PATH"].split(os.pathsep):
1969 path = path.strip('"')
1970 exe_file = os.path.join(path, command)
1971 if is_exe(exe_file):
1978 '''generate a random password for when generating keys'''
1979 h = hashlib.sha256()
1980 h.update(os.urandom(16)) # salt
1981 h.update(socket.getfqdn().encode('utf-8'))
1982 passwd = base64.b64encode(h.digest()).strip()
1983 return passwd.decode('utf-8')
1986 def genkeystore(localconfig):
1987 '''Generate a new key with random passwords and add it to new keystore'''
1988 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
1989 keystoredir = os.path.dirname(localconfig['keystore'])
1990 if keystoredir is None or keystoredir == '':
1991 keystoredir = os.path.join(os.getcwd(), keystoredir)
1992 if not os.path.exists(keystoredir):
1993 os.makedirs(keystoredir, mode=0o700)
1995 write_password_file("keystorepass", localconfig['keystorepass'])
1996 write_password_file("keypass", localconfig['keypass'])
1997 p = FDroidPopen([config['keytool'], '-genkey',
1998 '-keystore', localconfig['keystore'],
1999 '-alias', localconfig['repo_keyalias'],
2000 '-keyalg', 'RSA', '-keysize', '4096',
2001 '-sigalg', 'SHA256withRSA',
2002 '-validity', '10000',
2003 '-storepass:file', config['keystorepassfile'],
2004 '-keypass:file', config['keypassfile'],
2005 '-dname', localconfig['keydname']])
2006 # TODO keypass should be sent via stdin
2007 if p.returncode != 0:
2008 raise BuildException("Failed to generate key", p.output)
2009 os.chmod(localconfig['keystore'], 0o0600)
2010 # now show the lovely key that was just generated
2011 p = FDroidPopen([config['keytool'], '-list', '-v',
2012 '-keystore', localconfig['keystore'],
2013 '-alias', localconfig['repo_keyalias'],
2014 '-storepass:file', config['keystorepassfile']])
2015 logging.info(p.output.strip() + '\n\n')
2018 def write_to_config(thisconfig, key, value=None):
2019 '''write a key/value to the local config.py'''
2021 origkey = key + '_orig'
2022 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2023 with open('config.py', 'r', encoding='utf8') as f:
2025 pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
2026 repl = '\n' + key + ' = "' + value + '"'
2027 data = re.sub(pattern, repl, data)
2028 # if this key is not in the file, append it
2029 if not re.match('\s*' + key + '\s*=\s*"', data):
2031 # make sure the file ends with a carraige return
2032 if not re.match('\n$', data):
2034 with open('config.py', 'w', encoding='utf8') as f:
2038 def parse_xml(path):
2039 return XMLElementTree.parse(path).getroot()
2042 def string_is_integer(string):
2050 def get_per_app_repos():
2051 '''per-app repos are dirs named with the packageName of a single app'''
2053 # Android packageNames are Java packages, they may contain uppercase or
2054 # lowercase letters ('A' through 'Z'), numbers, and underscores
2055 # ('_'). However, individual package name parts may only start with
2056 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2057 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2060 for root, dirs, files in os.walk(os.getcwd()):
2062 print('checking', root, 'for', d)
2063 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2064 # standard parts of an fdroid repo, so never packageNames
2067 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):