3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
37 import xml.etree.ElementTree as XMLElementTree
39 from distutils.version import LooseVersion
40 from queue import Queue
41 from zipfile import ZipFile
43 import fdroidserver.metadata
44 from .asynchronousfilereader import AsynchronousFileReader
47 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
56 'sdk_path': "$ANDROID_HOME",
61 'r12b': "$ANDROID_NDK",
65 'build_tools': "25.0.2",
66 'force_build_tools': False,
71 'accepted_formats': ['txt', 'yml'],
72 'sync_from_local_copy_dir': False,
73 'per_app_repos': False,
74 'make_current_version_link': True,
75 'current_version_name_source': 'Name',
76 'update_stats': False,
80 'stats_to_carbon': False,
82 'build_server_always': False,
83 'keystore': 'keystore.jks',
84 'smartcardoptions': [],
90 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
91 'repo_name': "My First FDroid Repo Demo",
92 'repo_icon': "fdroid-icon.png",
93 'repo_description': '''
94 This is a repository of apps to be used with FDroid. Applications in this
95 repository are either official binaries built by the original application
96 developers, or are binaries built from source by the admin of f-droid.org
97 using the tools on https://gitlab.com/u/fdroid.
103 def setup_global_opts(parser):
104 parser.add_argument("-v", "--verbose", action="store_true", default=False,
105 help="Spew out even more information than normal")
106 parser.add_argument("-q", "--quiet", action="store_true", default=False,
107 help="Restrict output to warnings and errors")
110 def fill_config_defaults(thisconfig):
111 for k, v in default_config.items():
112 if k not in thisconfig:
115 # Expand paths (~users and $vars)
116 def expand_path(path):
120 path = os.path.expanduser(path)
121 path = os.path.expandvars(path)
126 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
131 thisconfig[k + '_orig'] = v
133 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
134 if thisconfig['java_paths'] is None:
135 thisconfig['java_paths'] = dict()
137 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
138 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
139 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
140 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
141 if os.getenv('JAVA_HOME') is not None:
142 pathlist.append(os.getenv('JAVA_HOME'))
143 if os.getenv('PROGRAMFILES') is not None:
144 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
145 for d in sorted(pathlist):
146 if os.path.islink(d):
148 j = os.path.basename(d)
149 # the last one found will be the canonical one, so order appropriately
151 r'^1\.([6-9])\.0\.jdk$', # OSX
152 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
153 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
154 r'^jdk([6-9])-openjdk$', # Arch
155 r'^java-([6-9])-openjdk$', # Arch
156 r'^java-([6-9])-jdk$', # Arch (oracle)
157 r'^java-1\.([6-9])\.0-.*$', # RedHat
158 r'^java-([6-9])-oracle$', # Debian WebUpd8
159 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
160 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
162 m = re.match(regex, j)
165 for p in [d, os.path.join(d, 'Contents', 'Home')]:
166 if os.path.exists(os.path.join(p, 'bin', 'javac')):
167 thisconfig['java_paths'][m.group(1)] = p
169 for java_version in ('7', '8', '9'):
170 if java_version not in thisconfig['java_paths']:
172 java_home = thisconfig['java_paths'][java_version]
173 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
174 if os.path.exists(jarsigner):
175 thisconfig['jarsigner'] = jarsigner
176 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
177 break # Java7 is preferred, so quit if found
179 for k in ['ndk_paths', 'java_paths']:
185 thisconfig[k][k2] = exp
186 thisconfig[k][k2 + '_orig'] = v
189 def regsub_file(pattern, repl, path):
190 with open(path, 'rb') as f:
192 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
193 with open(path, 'wb') as f:
197 def read_config(opts, config_file='config.py'):
198 """Read the repository config
200 The config is read from config_file, which is in the current
201 directory when any of the repo management commands are used. If
202 there is a local metadata file in the git repo, then config.py is
203 not required, just use defaults.
206 global config, options
208 if config is not None:
215 if os.path.isfile(config_file):
216 logging.debug("Reading %s" % config_file)
217 with io.open(config_file, "rb") as f:
218 code = compile(f.read(), config_file, 'exec')
219 exec(code, None, config)
220 elif len(get_local_metadata_files()) == 0:
221 logging.critical("Missing config file - is this a repo directory?")
224 # smartcardoptions must be a list since its command line args for Popen
225 if 'smartcardoptions' in config:
226 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
227 elif 'keystore' in config and config['keystore'] == 'NONE':
228 # keystore='NONE' means use smartcard, these are required defaults
229 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
230 'SunPKCS11-OpenSC', '-providerClass',
231 'sun.security.pkcs11.SunPKCS11',
232 '-providerArg', 'opensc-fdroid.cfg']
234 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
235 st = os.stat(config_file)
236 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
237 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
239 fill_config_defaults(config)
241 for k in ["keystorepass", "keypass"]:
243 write_password_file(k)
245 for k in ["repo_description", "archive_description"]:
247 config[k] = clean_description(config[k])
249 if 'serverwebroot' in config:
250 if isinstance(config['serverwebroot'], str):
251 roots = [config['serverwebroot']]
252 elif all(isinstance(item, str) for item in config['serverwebroot']):
253 roots = config['serverwebroot']
255 raise TypeError('only accepts strings, lists, and tuples')
257 for rootstr in roots:
258 # since this is used with rsync, where trailing slashes have
259 # meaning, ensure there is always a trailing slash
260 if rootstr[-1] != '/':
262 rootlist.append(rootstr.replace('//', '/'))
263 config['serverwebroot'] = rootlist
268 def find_sdk_tools_cmd(cmd):
269 '''find a working path to a tool from the Android SDK'''
272 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
273 # try to find a working path to this command, in all the recent possible paths
274 if 'build_tools' in config:
275 build_tools = os.path.join(config['sdk_path'], 'build-tools')
276 # if 'build_tools' was manually set and exists, check only that one
277 configed_build_tools = os.path.join(build_tools, config['build_tools'])
278 if os.path.exists(configed_build_tools):
279 tooldirs.append(configed_build_tools)
281 # no configed version, so hunt known paths for it
282 for f in sorted(os.listdir(build_tools), reverse=True):
283 if os.path.isdir(os.path.join(build_tools, f)):
284 tooldirs.append(os.path.join(build_tools, f))
285 tooldirs.append(build_tools)
286 sdk_tools = os.path.join(config['sdk_path'], 'tools')
287 if os.path.exists(sdk_tools):
288 tooldirs.append(sdk_tools)
289 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
290 if os.path.exists(sdk_platform_tools):
291 tooldirs.append(sdk_platform_tools)
292 tooldirs.append('/usr/bin')
294 path = os.path.join(d, cmd)
295 if os.path.isfile(path):
297 test_aapt_version(path)
299 # did not find the command, exit with error message
300 ensure_build_tools_exists(config)
303 def test_aapt_version(aapt):
304 '''Check whether the version of aapt is new enough'''
305 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
306 if output is None or output == '':
307 logging.error(aapt + ' failed to execute!')
309 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
314 # the Debian package has the version string like "v0.2-23.0.2"
315 if '.' not in bugfix and LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.2166767'):
316 logging.warning(aapt + ' is too old, fdroid requires build-tools-23.0.0 or newer!')
318 logging.warning('Unknown version of aapt, might cause problems: ' + output)
321 def test_sdk_exists(thisconfig):
322 if 'sdk_path' not in thisconfig:
323 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
324 test_aapt_version(thisconfig['aapt'])
327 logging.error("'sdk_path' not set in config.py!")
329 if thisconfig['sdk_path'] == default_config['sdk_path']:
330 logging.error('No Android SDK found!')
331 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
332 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
334 if not os.path.exists(thisconfig['sdk_path']):
335 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
337 if not os.path.isdir(thisconfig['sdk_path']):
338 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
340 for d in ['build-tools', 'platform-tools', 'tools']:
341 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
342 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
343 thisconfig['sdk_path'], d))
348 def ensure_build_tools_exists(thisconfig):
349 if not test_sdk_exists(thisconfig):
351 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
352 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
353 if not os.path.isdir(versioned_build_tools):
354 logging.critical('Android Build Tools path "'
355 + versioned_build_tools + '" does not exist!')
359 def write_password_file(pwtype, password=None):
361 writes out passwords to a protected file instead of passing passwords as
362 command line argments
364 filename = '.fdroid.' + pwtype + '.txt'
365 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
367 os.write(fd, config[pwtype].encode('utf-8'))
369 os.write(fd, password.encode('utf-8'))
371 config[pwtype + 'file'] = filename
374 def get_local_metadata_files():
375 '''get any metadata files local to an app's source repo
377 This tries to ignore anything that does not count as app metdata,
378 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
381 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
384 def read_pkg_args(args, allow_vercodes=False):
386 Given the arguments in the form of multiple appid:[vc] strings, this returns
387 a dictionary with the set of vercodes specified for each package.
395 if allow_vercodes and ':' in p:
396 package, vercode = p.split(':')
398 package, vercode = p, None
399 if package not in vercodes:
400 vercodes[package] = [vercode] if vercode else []
402 elif vercode and vercode not in vercodes[package]:
403 vercodes[package] += [vercode] if vercode else []
408 def read_app_args(args, allapps, allow_vercodes=False):
410 On top of what read_pkg_args does, this returns the whole app metadata, but
411 limiting the builds list to the builds matching the vercodes specified.
414 vercodes = read_pkg_args(args, allow_vercodes)
420 for appid, app in allapps.items():
421 if appid in vercodes:
424 if len(apps) != len(vercodes):
427 logging.critical("No such package: %s" % p)
428 raise FDroidException("Found invalid app ids in arguments")
430 raise FDroidException("No packages specified")
433 for appid, app in apps.items():
437 app.builds = [b for b in app.builds if b.vercode in vc]
438 if len(app.builds) != len(vercodes[appid]):
440 allvcs = [b.vercode for b in app.builds]
441 for v in vercodes[appid]:
443 logging.critical("No such vercode %s for app %s" % (v, appid))
446 raise FDroidException("Found invalid vercodes for some apps")
451 def get_extension(filename):
452 base, ext = os.path.splitext(filename)
455 return base, ext.lower()[1:]
458 def has_extension(filename, ext):
459 _, f_ext = get_extension(filename)
463 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
466 def clean_description(description):
467 'Remove unneeded newlines and spaces from a block of description text'
469 # this is split up by paragraph to make removing the newlines easier
470 for paragraph in re.split(r'\n\n', description):
471 paragraph = re.sub('\r', '', paragraph)
472 paragraph = re.sub('\n', ' ', paragraph)
473 paragraph = re.sub(' {2,}', ' ', paragraph)
474 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
475 returnstring += paragraph + '\n\n'
476 return returnstring.rstrip('\n')
479 def apknameinfo(filename):
480 filename = os.path.basename(filename)
481 m = apk_regex.match(filename)
483 result = (m.group(1), m.group(2))
484 except AttributeError:
485 raise FDroidException("Invalid apk name: %s" % filename)
489 def get_release_filename(app, build):
491 return "%s_%s.%s" % (app.id, build.vercode, get_file_extension(build.output))
493 return "%s_%s.apk" % (app.id, build.vercode)
496 def getsrcname(app, build):
497 return "%s_%s_src.tar.gz" % (app.id, build.vercode)
509 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
512 def get_build_dir(app):
513 '''get the dir that this app will be built in'''
515 if app.RepoType == 'srclib':
516 return os.path.join('build', 'srclib', app.Repo)
518 return os.path.join('build', app.id)
522 '''checkout code from VCS and return instance of vcs and the build dir'''
523 build_dir = get_build_dir(app)
525 # Set up vcs interface and make sure we have the latest code...
526 logging.debug("Getting {0} vcs interface for {1}"
527 .format(app.RepoType, app.Repo))
528 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
532 vcs = getvcs(app.RepoType, remote, build_dir)
534 return vcs, build_dir
537 def getvcs(vcstype, remote, local):
539 return vcs_git(remote, local)
540 if vcstype == 'git-svn':
541 return vcs_gitsvn(remote, local)
543 return vcs_hg(remote, local)
545 return vcs_bzr(remote, local)
546 if vcstype == 'srclib':
547 if local != os.path.join('build', 'srclib', remote):
548 raise VCSException("Error: srclib paths are hard-coded!")
549 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
551 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
552 raise VCSException("Invalid vcs type " + vcstype)
555 def getsrclibvcs(name):
556 if name not in fdroidserver.metadata.srclibs:
557 raise VCSException("Missing srclib " + name)
558 return fdroidserver.metadata.srclibs[name]['Repo Type']
563 def __init__(self, remote, local):
565 # svn, git-svn and bzr may require auth
567 if self.repotype() in ('git-svn', 'bzr'):
569 if self.repotype == 'git-svn':
570 raise VCSException("Authentication is not supported for git-svn")
571 self.username, remote = remote.split('@')
572 if ':' not in self.username:
573 raise VCSException("Password required with username")
574 self.username, self.password = self.username.split(':')
578 self.clone_failed = False
579 self.refreshed = False
585 # Take the local repository to a clean version of the given revision, which
586 # is specificed in the VCS's native format. Beforehand, the repository can
587 # be dirty, or even non-existent. If the repository does already exist
588 # locally, it will be updated from the origin, but only once in the
589 # lifetime of the vcs object.
590 # None is acceptable for 'rev' if you know you are cloning a clean copy of
591 # the repo - otherwise it must specify a valid revision.
592 def gotorevision(self, rev, refresh=True):
594 if self.clone_failed:
595 raise VCSException("Downloading the repository already failed once, not trying again.")
597 # The .fdroidvcs-id file for a repo tells us what VCS type
598 # and remote that directory was created from, allowing us to drop it
599 # automatically if either of those things changes.
600 fdpath = os.path.join(self.local, '..',
601 '.fdroidvcs-' + os.path.basename(self.local))
602 fdpath = os.path.normpath(fdpath)
603 cdata = self.repotype() + ' ' + self.remote
606 if os.path.exists(self.local):
607 if os.path.exists(fdpath):
608 with open(fdpath, 'r') as f:
609 fsdata = f.read().strip()
614 logging.info("Repository details for %s changed - deleting" % (
618 logging.info("Repository details for %s missing - deleting" % (
621 shutil.rmtree(self.local)
625 self.refreshed = True
628 self.gotorevisionx(rev)
629 except FDroidException as e:
632 # If necessary, write the .fdroidvcs file.
633 if writeback and not self.clone_failed:
634 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
635 with open(fdpath, 'w+') as f:
641 # Derived classes need to implement this. It's called once basic checking
642 # has been performend.
643 def gotorevisionx(self, rev):
644 raise VCSException("This VCS type doesn't define gotorevisionx")
646 # Initialise and update submodules
647 def initsubmodules(self):
648 raise VCSException('Submodules not supported for this vcs type')
650 # Get a list of all known tags
652 if not self._gettags:
653 raise VCSException('gettags not supported for this vcs type')
655 for tag in self._gettags():
656 if re.match('[-A-Za-z0-9_. /]+$', tag):
660 # Get a list of all the known tags, sorted from newest to oldest
661 def latesttags(self):
662 raise VCSException('latesttags not supported for this vcs type')
664 # Get current commit reference (hash, revision, etc)
666 raise VCSException('getref not supported for this vcs type')
668 # Returns the srclib (name, path) used in setting up the current
679 # If the local directory exists, but is somehow not a git repository, git
680 # will traverse up the directory tree until it finds one that is (i.e.
681 # fdroidserver) and then we'll proceed to destroy it! This is called as
684 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
685 result = p.output.rstrip()
686 if not result.endswith(self.local):
687 raise VCSException('Repository mismatch')
689 def gotorevisionx(self, rev):
690 if not os.path.exists(self.local):
692 p = FDroidPopen(['git', 'clone', self.remote, self.local])
693 if p.returncode != 0:
694 self.clone_failed = True
695 raise VCSException("Git clone failed", p.output)
699 # Discard any working tree changes
700 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
701 'git', 'reset', '--hard'], cwd=self.local, output=False)
702 if p.returncode != 0:
703 raise VCSException("Git reset failed", p.output)
704 # Remove untracked files now, in case they're tracked in the target
705 # revision (it happens!)
706 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
707 'git', 'clean', '-dffx'], cwd=self.local, output=False)
708 if p.returncode != 0:
709 raise VCSException("Git clean failed", p.output)
710 if not self.refreshed:
711 # Get latest commits and tags from remote
712 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
713 if p.returncode != 0:
714 raise VCSException("Git fetch failed", p.output)
715 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
716 if p.returncode != 0:
717 raise VCSException("Git fetch failed", p.output)
718 # Recreate origin/HEAD as git clone would do it, in case it disappeared
719 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
720 if p.returncode != 0:
721 lines = p.output.splitlines()
722 if 'Multiple remote HEAD branches' not in lines[0]:
723 raise VCSException("Git remote set-head failed", p.output)
724 branch = lines[1].split(' ')[-1]
725 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
726 if p2.returncode != 0:
727 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
728 self.refreshed = True
729 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
730 # a github repo. Most of the time this is the same as origin/master.
731 rev = rev or 'origin/HEAD'
732 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
733 if p.returncode != 0:
734 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
735 # Get rid of any uncontrolled files left behind
736 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
737 if p.returncode != 0:
738 raise VCSException("Git clean failed", p.output)
740 def initsubmodules(self):
742 submfile = os.path.join(self.local, '.gitmodules')
743 if not os.path.isfile(submfile):
744 raise VCSException("No git submodules available")
746 # fix submodules not accessible without an account and public key auth
747 with open(submfile, 'r') as f:
748 lines = f.readlines()
749 with open(submfile, 'w') as f:
751 if 'git@github.com' in line:
752 line = line.replace('git@github.com:', 'https://github.com/')
753 if 'git@gitlab.com' in line:
754 line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
757 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
758 if p.returncode != 0:
759 raise VCSException("Git submodule sync failed", p.output)
760 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
761 if p.returncode != 0:
762 raise VCSException("Git submodule update failed", p.output)
766 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
767 return p.output.splitlines()
769 tag_format = re.compile(r'tag: ([^),]*)')
771 def latesttags(self):
773 p = FDroidPopen(['git', 'log', '--tags',
774 '--simplify-by-decoration', '--pretty=format:%d'],
775 cwd=self.local, output=False)
777 for line in p.output.splitlines():
778 for tag in self.tag_format.findall(line):
783 class vcs_gitsvn(vcs):
788 # If the local directory exists, but is somehow not a git repository, git
789 # will traverse up the directory tree until it finds one that is (i.e.
790 # fdroidserver) and then we'll proceed to destory it! This is called as
793 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
794 result = p.output.rstrip()
795 if not result.endswith(self.local):
796 raise VCSException('Repository mismatch')
798 def gotorevisionx(self, rev):
799 if not os.path.exists(self.local):
801 gitsvn_args = ['git', 'svn', 'clone']
802 if ';' in self.remote:
803 remote_split = self.remote.split(';')
804 for i in remote_split[1:]:
805 if i.startswith('trunk='):
806 gitsvn_args.extend(['-T', i[6:]])
807 elif i.startswith('tags='):
808 gitsvn_args.extend(['-t', i[5:]])
809 elif i.startswith('branches='):
810 gitsvn_args.extend(['-b', i[9:]])
811 gitsvn_args.extend([remote_split[0], self.local])
812 p = FDroidPopen(gitsvn_args, output=False)
813 if p.returncode != 0:
814 self.clone_failed = True
815 raise VCSException("Git svn clone failed", p.output)
817 gitsvn_args.extend([self.remote, self.local])
818 p = FDroidPopen(gitsvn_args, output=False)
819 if p.returncode != 0:
820 self.clone_failed = True
821 raise VCSException("Git svn clone failed", p.output)
825 # Discard any working tree changes
826 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
827 if p.returncode != 0:
828 raise VCSException("Git reset failed", p.output)
829 # Remove untracked files now, in case they're tracked in the target
830 # revision (it happens!)
831 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
832 if p.returncode != 0:
833 raise VCSException("Git clean failed", p.output)
834 if not self.refreshed:
835 # Get new commits, branches and tags from repo
836 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
837 if p.returncode != 0:
838 raise VCSException("Git svn fetch failed")
839 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
840 if p.returncode != 0:
841 raise VCSException("Git svn rebase failed", p.output)
842 self.refreshed = True
844 rev = rev or 'master'
846 nospaces_rev = rev.replace(' ', '%20')
847 # Try finding a svn tag
848 for treeish in ['origin/', '']:
849 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
850 if p.returncode == 0:
852 if p.returncode != 0:
853 # No tag found, normal svn rev translation
854 # Translate svn rev into git format
855 rev_split = rev.split('/')
858 for treeish in ['origin/', '']:
859 if len(rev_split) > 1:
860 treeish += rev_split[0]
861 svn_rev = rev_split[1]
864 # if no branch is specified, then assume trunk (i.e. 'master' branch):
868 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
870 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
871 git_rev = p.output.rstrip()
873 if p.returncode == 0 and git_rev:
876 if p.returncode != 0 or not git_rev:
877 # Try a plain git checkout as a last resort
878 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
879 if p.returncode != 0:
880 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
882 # Check out the git rev equivalent to the svn rev
883 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
884 if p.returncode != 0:
885 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
887 # Get rid of any uncontrolled files left behind
888 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
889 if p.returncode != 0:
890 raise VCSException("Git clean failed", p.output)
894 for treeish in ['origin/', '']:
895 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
901 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
902 if p.returncode != 0:
904 return p.output.strip()
912 def gotorevisionx(self, rev):
913 if not os.path.exists(self.local):
914 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
915 if p.returncode != 0:
916 self.clone_failed = True
917 raise VCSException("Hg clone failed", p.output)
919 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
920 if p.returncode != 0:
921 raise VCSException("Hg status failed", p.output)
922 for line in p.output.splitlines():
923 if not line.startswith('? '):
924 raise VCSException("Unexpected output from hg status -uS: " + line)
925 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
926 if not self.refreshed:
927 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
928 if p.returncode != 0:
929 raise VCSException("Hg pull failed", p.output)
930 self.refreshed = True
932 rev = rev or 'default'
935 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
936 if p.returncode != 0:
937 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
938 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
939 # Also delete untracked files, we have to enable purge extension for that:
940 if "'purge' is provided by the following extension" in p.output:
941 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
942 myfile.write("\n[extensions]\nhgext.purge=\n")
943 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
944 if p.returncode != 0:
945 raise VCSException("HG purge failed", p.output)
946 elif p.returncode != 0:
947 raise VCSException("HG purge failed", p.output)
950 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
951 return p.output.splitlines()[1:]
959 def gotorevisionx(self, rev):
960 if not os.path.exists(self.local):
961 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
962 if p.returncode != 0:
963 self.clone_failed = True
964 raise VCSException("Bzr branch failed", p.output)
966 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
967 if p.returncode != 0:
968 raise VCSException("Bzr revert failed", p.output)
969 if not self.refreshed:
970 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
971 if p.returncode != 0:
972 raise VCSException("Bzr update failed", p.output)
973 self.refreshed = True
975 revargs = list(['-r', rev] if rev else [])
976 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
977 if p.returncode != 0:
978 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
981 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
982 return [tag.split(' ')[0].strip() for tag in
983 p.output.splitlines()]
986 def unescape_string(string):
989 if string[0] == '"' and string[-1] == '"':
992 return string.replace("\\'", "'")
995 def retrieve_string(app_dir, string, xmlfiles=None):
997 if not string.startswith('@string/'):
998 return unescape_string(string)
1000 if xmlfiles is None:
1003 os.path.join(app_dir, 'res'),
1004 os.path.join(app_dir, 'src', 'main', 'res'),
1006 for r, d, f in os.walk(res_dir):
1007 if os.path.basename(r) == 'values':
1008 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
1010 name = string[len('@string/'):]
1012 def element_content(element):
1013 if element.text is None:
1015 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1016 return s.decode('utf-8').strip()
1018 for path in xmlfiles:
1019 if not os.path.isfile(path):
1021 xml = parse_xml(path)
1022 element = xml.find('string[@name="' + name + '"]')
1023 if element is not None:
1024 content = element_content(element)
1025 return retrieve_string(app_dir, content, xmlfiles)
1030 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1031 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1034 def manifest_paths(app_dir, flavours):
1035 '''Return list of existing files that will be used to find the highest vercode'''
1037 possible_manifests = \
1038 [os.path.join(app_dir, 'AndroidManifest.xml'),
1039 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1040 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1041 os.path.join(app_dir, 'build.gradle')]
1043 for flavour in flavours:
1044 if flavour == 'yes':
1046 possible_manifests.append(
1047 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1049 return [path for path in possible_manifests if os.path.isfile(path)]
1052 def fetch_real_name(app_dir, flavours):
1053 '''Retrieve the package name. Returns the name, or None if not found.'''
1054 for path in manifest_paths(app_dir, flavours):
1055 if not has_extension(path, 'xml') or not os.path.isfile(path):
1057 logging.debug("fetch_real_name: Checking manifest at " + path)
1058 xml = parse_xml(path)
1059 app = xml.find('application')
1062 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1064 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1065 result = retrieve_string_singleline(app_dir, label)
1067 result = result.strip()
1072 def get_library_references(root_dir):
1074 proppath = os.path.join(root_dir, 'project.properties')
1075 if not os.path.isfile(proppath):
1077 with open(proppath, 'r', encoding='iso-8859-1') as f:
1079 if not line.startswith('android.library.reference.'):
1081 path = line.split('=')[1].strip()
1082 relpath = os.path.join(root_dir, path)
1083 if not os.path.isdir(relpath):
1085 logging.debug("Found subproject at %s" % path)
1086 libraries.append(path)
1090 def ant_subprojects(root_dir):
1091 subprojects = get_library_references(root_dir)
1092 for subpath in subprojects:
1093 subrelpath = os.path.join(root_dir, subpath)
1094 for p in get_library_references(subrelpath):
1095 relp = os.path.normpath(os.path.join(subpath, p))
1096 if relp not in subprojects:
1097 subprojects.insert(0, relp)
1101 def remove_debuggable_flags(root_dir):
1102 # Remove forced debuggable flags
1103 logging.debug("Removing debuggable flags from %s" % root_dir)
1104 for root, dirs, files in os.walk(root_dir):
1105 if 'AndroidManifest.xml' in files:
1106 regsub_file(r'android:debuggable="[^"]*"',
1108 os.path.join(root, 'AndroidManifest.xml'))
1111 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
1112 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1113 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1116 def app_matches_packagename(app, package):
1119 appid = app.UpdateCheckName or app.id
1120 if appid is None or appid == "Ignore":
1122 return appid == package
1125 def parse_androidmanifests(paths, app):
1127 Extract some information from the AndroidManifest.xml at the given path.
1128 Returns (version, vercode, package), any or all of which might be None.
1129 All values returned are strings.
1132 ignoreversions = app.UpdateCheckIgnore
1133 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1136 return (None, None, None)
1144 if not os.path.isfile(path):
1147 logging.debug("Parsing manifest at {0}".format(path))
1148 gradle = has_extension(path, 'gradle')
1154 with open(path, 'r') as f:
1156 if gradle_comment.match(line):
1158 # Grab first occurence of each to avoid running into
1159 # alternative flavours and builds.
1161 matches = psearch_g(line)
1163 s = matches.group(2)
1164 if app_matches_packagename(app, s):
1167 matches = vnsearch_g(line)
1169 version = matches.group(2)
1171 matches = vcsearch_g(line)
1173 vercode = matches.group(1)
1176 xml = parse_xml(path)
1177 if "package" in xml.attrib:
1178 s = xml.attrib["package"]
1179 if app_matches_packagename(app, s):
1181 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1182 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1183 base_dir = os.path.dirname(path)
1184 version = retrieve_string_singleline(base_dir, version)
1185 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1186 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1187 if string_is_integer(a):
1190 logging.warning("Problem with xml at {0}".format(path))
1192 # Remember package name, may be defined separately from version+vercode
1194 package = max_package
1196 logging.debug("..got package={0}, version={1}, vercode={2}"
1197 .format(package, version, vercode))
1199 # Always grab the package name and version name in case they are not
1200 # together with the highest version code
1201 if max_package is None and package is not None:
1202 max_package = package
1203 if max_version is None and version is not None:
1204 max_version = version
1206 if max_vercode is None or (vercode is not None and vercode > max_vercode):
1207 if not ignoresearch or not ignoresearch(version):
1208 if version is not None:
1209 max_version = version
1210 if vercode is not None:
1211 max_vercode = vercode
1212 if package is not None:
1213 max_package = package
1215 max_version = "Ignore"
1217 if max_version is None:
1218 max_version = "Unknown"
1220 if max_package and not is_valid_package_name(max_package):
1221 raise FDroidException("Invalid package name {0}".format(max_package))
1223 return (max_version, max_vercode, max_package)
1226 def is_valid_package_name(name):
1227 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1230 class FDroidException(Exception):
1232 def __init__(self, value, detail=None):
1234 self.detail = detail
1236 def shortened_detail(self):
1237 if len(self.detail) < 16000:
1239 return '[...]\n' + self.detail[-16000:]
1241 def get_wikitext(self):
1242 ret = repr(self.value) + "\n"
1245 ret += "<pre>\n" + self.shortened_detail() + "</pre>\n"
1251 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1255 class VCSException(FDroidException):
1259 class BuildException(FDroidException):
1263 # Get the specified source library.
1264 # Returns the path to it. Normally this is the path to be used when referencing
1265 # it, which may be a subdirectory of the actual project. If you want the base
1266 # directory of the project, pass 'basepath=True'.
1267 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1268 raw=False, prepare=True, preponly=False, refresh=True,
1277 name, ref = spec.split('@')
1279 number, name = name.split(':', 1)
1281 name, subdir = name.split('/', 1)
1283 if name not in fdroidserver.metadata.srclibs:
1284 raise VCSException('srclib ' + name + ' not found.')
1286 srclib = fdroidserver.metadata.srclibs[name]
1288 sdir = os.path.join(srclib_dir, name)
1291 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1292 vcs.srclib = (name, number, sdir)
1294 vcs.gotorevision(ref, refresh)
1301 libdir = os.path.join(sdir, subdir)
1302 elif srclib["Subdir"]:
1303 for subdir in srclib["Subdir"]:
1304 libdir_candidate = os.path.join(sdir, subdir)
1305 if os.path.exists(libdir_candidate):
1306 libdir = libdir_candidate
1312 remove_signing_keys(sdir)
1313 remove_debuggable_flags(sdir)
1317 if srclib["Prepare"]:
1318 cmd = replace_config_vars(srclib["Prepare"], build)
1320 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1321 if p.returncode != 0:
1322 raise BuildException("Error running prepare command for srclib %s"
1328 return (name, number, libdir)
1331 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1334 # Prepare the source code for a particular build
1335 # 'vcs' - the appropriate vcs object for the application
1336 # 'app' - the application details from the metadata
1337 # 'build' - the build details from the metadata
1338 # 'build_dir' - the path to the build directory, usually
1340 # 'srclib_dir' - the path to the source libraries directory, usually
1342 # 'extlib_dir' - the path to the external libraries directory, usually
1344 # Returns the (root, srclibpaths) where:
1345 # 'root' is the root directory, which may be the same as 'build_dir' or may
1346 # be a subdirectory of it.
1347 # 'srclibpaths' is information on the srclibs being used
1348 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1350 # Optionally, the actual app source can be in a subdirectory
1352 root_dir = os.path.join(build_dir, build.subdir)
1354 root_dir = build_dir
1356 # Get a working copy of the right revision
1357 logging.info("Getting source for revision " + build.commit)
1358 vcs.gotorevision(build.commit, refresh)
1360 # Initialise submodules if required
1361 if build.submodules:
1362 logging.info("Initialising submodules")
1363 vcs.initsubmodules()
1365 # Check that a subdir (if we're using one) exists. This has to happen
1366 # after the checkout, since it might not exist elsewhere
1367 if not os.path.exists(root_dir):
1368 raise BuildException('Missing subdir ' + root_dir)
1370 # Run an init command if one is required
1372 cmd = replace_config_vars(build.init, build)
1373 logging.info("Running 'init' commands in %s" % root_dir)
1375 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1376 if p.returncode != 0:
1377 raise BuildException("Error running init command for %s:%s" %
1378 (app.id, build.version), p.output)
1380 # Apply patches if any
1382 logging.info("Applying patches")
1383 for patch in build.patch:
1384 patch = patch.strip()
1385 logging.info("Applying " + patch)
1386 patch_path = os.path.join('metadata', app.id, patch)
1387 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1388 if p.returncode != 0:
1389 raise BuildException("Failed to apply patch %s" % patch_path)
1391 # Get required source libraries
1394 logging.info("Collecting source libraries")
1395 for lib in build.srclibs:
1396 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1397 refresh=refresh, build=build))
1399 for name, number, libpath in srclibpaths:
1400 place_srclib(root_dir, int(number) if number else None, libpath)
1402 basesrclib = vcs.getsrclib()
1403 # If one was used for the main source, add that too.
1405 srclibpaths.append(basesrclib)
1407 # Update the local.properties file
1408 localprops = [os.path.join(build_dir, 'local.properties')]
1410 parts = build.subdir.split(os.sep)
1413 cur = os.path.join(cur, d)
1414 localprops += [os.path.join(cur, 'local.properties')]
1415 for path in localprops:
1417 if os.path.isfile(path):
1418 logging.info("Updating local.properties file at %s" % path)
1419 with open(path, 'r', encoding='iso-8859-1') as f:
1423 logging.info("Creating local.properties file at %s" % path)
1424 # Fix old-fashioned 'sdk-location' by copying
1425 # from sdk.dir, if necessary
1427 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1428 re.S | re.M).group(1)
1429 props += "sdk-location=%s\n" % sdkloc
1431 props += "sdk.dir=%s\n" % config['sdk_path']
1432 props += "sdk-location=%s\n" % config['sdk_path']
1433 ndk_path = build.ndk_path()
1434 # if for any reason the path isn't valid or the directory
1435 # doesn't exist, some versions of Gradle will error with a
1436 # cryptic message (even if the NDK is not even necessary).
1437 # https://gitlab.com/fdroid/fdroidserver/issues/171
1438 if ndk_path and os.path.exists(ndk_path):
1440 props += "ndk.dir=%s\n" % ndk_path
1441 props += "ndk-location=%s\n" % ndk_path
1442 # Add java.encoding if necessary
1444 props += "java.encoding=%s\n" % build.encoding
1445 with open(path, 'w', encoding='iso-8859-1') as f:
1449 if build.build_method() == 'gradle':
1450 flavours = build.gradle
1453 n = build.target.split('-')[1]
1454 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1455 r'compileSdkVersion %s' % n,
1456 os.path.join(root_dir, 'build.gradle'))
1458 # Remove forced debuggable flags
1459 remove_debuggable_flags(root_dir)
1461 # Insert version code and number into the manifest if necessary
1462 if build.forceversion:
1463 logging.info("Changing the version name")
1464 for path in manifest_paths(root_dir, flavours):
1465 if not os.path.isfile(path):
1467 if has_extension(path, 'xml'):
1468 regsub_file(r'android:versionName="[^"]*"',
1469 r'android:versionName="%s"' % build.version,
1471 elif has_extension(path, 'gradle'):
1472 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1473 r"""\1versionName '%s'""" % build.version,
1476 if build.forcevercode:
1477 logging.info("Changing the version code")
1478 for path in manifest_paths(root_dir, flavours):
1479 if not os.path.isfile(path):
1481 if has_extension(path, 'xml'):
1482 regsub_file(r'android:versionCode="[^"]*"',
1483 r'android:versionCode="%s"' % build.vercode,
1485 elif has_extension(path, 'gradle'):
1486 regsub_file(r'versionCode[ =]+[0-9]+',
1487 r'versionCode %s' % build.vercode,
1490 # Delete unwanted files
1492 logging.info("Removing specified files")
1493 for part in getpaths(build_dir, build.rm):
1494 dest = os.path.join(build_dir, part)
1495 logging.info("Removing {0}".format(part))
1496 if os.path.lexists(dest):
1497 if os.path.islink(dest):
1498 FDroidPopen(['unlink', dest], output=False)
1500 FDroidPopen(['rm', '-rf', dest], output=False)
1502 logging.info("...but it didn't exist")
1504 remove_signing_keys(build_dir)
1506 # Add required external libraries
1508 logging.info("Collecting prebuilt libraries")
1509 libsdir = os.path.join(root_dir, 'libs')
1510 if not os.path.exists(libsdir):
1512 for lib in build.extlibs:
1514 logging.info("...installing extlib {0}".format(lib))
1515 libf = os.path.basename(lib)
1516 libsrc = os.path.join(extlib_dir, lib)
1517 if not os.path.exists(libsrc):
1518 raise BuildException("Missing extlib file {0}".format(libsrc))
1519 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1521 # Run a pre-build command if one is required
1523 logging.info("Running 'prebuild' commands in %s" % root_dir)
1525 cmd = replace_config_vars(build.prebuild, build)
1527 # Substitute source library paths into prebuild commands
1528 for name, number, libpath in srclibpaths:
1529 libpath = os.path.relpath(libpath, root_dir)
1530 cmd = cmd.replace('$$' + name + '$$', libpath)
1532 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1533 if p.returncode != 0:
1534 raise BuildException("Error running prebuild command for %s:%s" %
1535 (app.id, build.version), p.output)
1537 # Generate (or update) the ant build file, build.xml...
1538 if build.build_method() == 'ant' and build.update != ['no']:
1539 parms = ['android', 'update', 'lib-project']
1540 lparms = ['android', 'update', 'project']
1543 parms += ['-t', build.target]
1544 lparms += ['-t', build.target]
1546 update_dirs = build.update
1548 update_dirs = ant_subprojects(root_dir) + ['.']
1550 for d in update_dirs:
1551 subdir = os.path.join(root_dir, d)
1553 logging.debug("Updating main project")
1554 cmd = parms + ['-p', d]
1556 logging.debug("Updating subproject %s" % d)
1557 cmd = lparms + ['-p', d]
1558 p = SdkToolsPopen(cmd, cwd=root_dir)
1559 # Check to see whether an error was returned without a proper exit
1560 # code (this is the case for the 'no target set or target invalid'
1562 if p.returncode != 0 or p.output.startswith("Error: "):
1563 raise BuildException("Failed to update project at %s" % d, p.output)
1564 # Clean update dirs via ant
1566 logging.info("Cleaning subproject %s" % d)
1567 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1569 return (root_dir, srclibpaths)
1572 # Extend via globbing the paths from a field and return them as a map from
1573 # original path to resulting paths
1574 def getpaths_map(build_dir, globpaths):
1578 full_path = os.path.join(build_dir, p)
1579 full_path = os.path.normpath(full_path)
1580 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1582 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1586 # Extend via globbing the paths from a field and return them as a set
1587 def getpaths(build_dir, globpaths):
1588 paths_map = getpaths_map(build_dir, globpaths)
1590 for k, v in paths_map.items():
1597 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1603 self.path = os.path.join('stats', 'known_apks.txt')
1605 if os.path.isfile(self.path):
1606 with open(self.path, 'r', encoding='utf8') as f:
1608 t = line.rstrip().split(' ')
1610 self.apks[t[0]] = (t[1], None)
1612 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1613 self.changed = False
1615 def writeifchanged(self):
1616 if not self.changed:
1619 if not os.path.exists('stats'):
1623 for apk, app in self.apks.items():
1625 line = apk + ' ' + appid
1627 line += ' ' + time.strftime('%Y-%m-%d', added)
1630 with open(self.path, 'w', encoding='utf8') as f:
1631 for line in sorted(lst, key=natural_key):
1632 f.write(line + '\n')
1634 # Record an apk (if it's new, otherwise does nothing)
1635 # Returns the date it was added.
1636 def recordapk(self, apk, app, default_date=None):
1637 if apk not in self.apks:
1638 if default_date is None:
1639 default_date = time.gmtime(time.time())
1640 self.apks[apk] = (app, default_date)
1642 _, added = self.apks[apk]
1645 # Look up information - given the 'apkname', returns (app id, date added/None).
1646 # Or returns None for an unknown apk.
1647 def getapp(self, apkname):
1648 if apkname in self.apks:
1649 return self.apks[apkname]
1652 # Get the most recent 'num' apps added to the repo, as a list of package ids
1653 # with the most recent first.
1654 def getlatest(self, num):
1656 for apk, app in self.apks.items():
1660 if apps[appid] > added:
1664 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1665 lst = [app for app, _ in sortedapps]
1670 def get_file_extension(filename):
1671 """get the normalized file extension, can be blank string but never None"""
1673 return os.path.splitext(filename)[1].lower()[1:]
1676 def isApkAndDebuggable(apkfile, config):
1677 """Returns True if the given file is an APK and is debuggable
1679 :param apkfile: full path to the apk to check"""
1681 if get_file_extension(apkfile) != 'apk':
1684 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1686 if p.returncode != 0:
1687 logging.critical("Failed to get apk manifest information")
1689 for line in p.output.splitlines():
1690 if 'android:debuggable' in line and not line.endswith('0x0'):
1697 self.returncode = None
1701 def SdkToolsPopen(commands, cwd=None, output=True):
1703 if cmd not in config:
1704 config[cmd] = find_sdk_tools_cmd(commands[0])
1705 abscmd = config[cmd]
1707 logging.critical("Could not find '%s' on your system" % cmd)
1710 test_aapt_version(config['aapt'])
1711 return FDroidPopen([abscmd] + commands[1:],
1712 cwd=cwd, output=output)
1715 def FDroidPopenBytes(commands, cwd=None, output=True, stderr_to_stdout=True):
1717 Run a command and capture the possibly huge output as bytes.
1719 :param commands: command and argument list like in subprocess.Popen
1720 :param cwd: optionally specifies a working directory
1721 :returns: A PopenResult.
1726 set_FDroidPopen_env()
1729 cwd = os.path.normpath(cwd)
1730 logging.debug("Directory: %s" % cwd)
1731 logging.debug("> %s" % ' '.join(commands))
1733 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
1734 result = PopenResult()
1737 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
1738 stdout=subprocess.PIPE, stderr=stderr_param)
1739 except OSError as e:
1740 raise BuildException("OSError while trying to execute " +
1741 ' '.join(commands) + ': ' + str(e))
1743 if not stderr_to_stdout and options.verbose:
1744 stderr_queue = Queue()
1745 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
1747 while not stderr_reader.eof():
1748 while not stderr_queue.empty():
1749 line = stderr_queue.get()
1750 sys.stderr.buffer.write(line)
1755 stdout_queue = Queue()
1756 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1759 # Check the queue for output (until there is no more to get)
1760 while not stdout_reader.eof():
1761 while not stdout_queue.empty():
1762 line = stdout_queue.get()
1763 if output and options.verbose:
1764 # Output directly to console
1765 sys.stderr.buffer.write(line)
1771 result.returncode = p.wait()
1772 result.output = buf.getvalue()
1777 def FDroidPopen(commands, cwd=None, output=True, stderr_to_stdout=True):
1779 Run a command and capture the possibly huge output as a str.
1781 :param commands: command and argument list like in subprocess.Popen
1782 :param cwd: optionally specifies a working directory
1783 :returns: A PopenResult.
1785 result = FDroidPopenBytes(commands, cwd, output, stderr_to_stdout)
1786 result.output = result.output.decode('utf-8')
1790 gradle_comment = re.compile(r'[ ]*//')
1791 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1792 gradle_line_matches = [
1793 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1794 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1795 re.compile(r'.*\.readLine\(.*'),
1799 def remove_signing_keys(build_dir):
1800 for root, dirs, files in os.walk(build_dir):
1801 if 'build.gradle' in files:
1802 path = os.path.join(root, 'build.gradle')
1804 with open(path, "r", encoding='utf8') as o:
1805 lines = o.readlines()
1811 with open(path, "w", encoding='utf8') as o:
1812 while i < len(lines):
1815 while line.endswith('\\\n'):
1816 line = line.rstrip('\\\n') + lines[i]
1819 if gradle_comment.match(line):
1824 opened += line.count('{')
1825 opened -= line.count('}')
1828 if gradle_signing_configs.match(line):
1833 if any(s.match(line) for s in gradle_line_matches):
1841 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1844 'project.properties',
1846 'default.properties',
1847 'ant.properties', ]:
1848 if propfile in files:
1849 path = os.path.join(root, propfile)
1851 with open(path, "r", encoding='iso-8859-1') as o:
1852 lines = o.readlines()
1856 with open(path, "w", encoding='iso-8859-1') as o:
1858 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1865 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1868 def set_FDroidPopen_env(build=None):
1870 set up the environment variables for the build environment
1872 There is only a weak standard, the variables used by gradle, so also set
1873 up the most commonly used environment variables for SDK and NDK. Also, if
1874 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
1876 global env, orig_path
1880 orig_path = env['PATH']
1881 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
1882 env[n] = config['sdk_path']
1883 for k, v in config['java_paths'].items():
1884 env['JAVA%s_HOME' % k] = v
1886 missinglocale = True
1887 for k, v in env.items():
1888 if k == 'LANG' and v != 'C':
1889 missinglocale = False
1891 missinglocale = False
1893 env['LANG'] = 'en_US.UTF-8'
1895 if build is not None:
1896 path = build.ndk_path()
1897 paths = orig_path.split(os.pathsep)
1898 if path not in paths:
1899 paths = [path] + paths
1900 env['PATH'] = os.pathsep.join(paths)
1901 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
1902 env[n] = build.ndk_path()
1905 def replace_config_vars(cmd, build):
1906 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1907 cmd = cmd.replace('$$NDK$$', build.ndk_path())
1908 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1909 cmd = cmd.replace('$$QT$$', config['qt_sdk_path'] or '')
1910 if build is not None:
1911 cmd = cmd.replace('$$COMMIT$$', build.commit)
1912 cmd = cmd.replace('$$VERSION$$', build.version)
1913 cmd = cmd.replace('$$VERCODE$$', build.vercode)
1917 def place_srclib(root_dir, number, libpath):
1920 relpath = os.path.relpath(libpath, root_dir)
1921 proppath = os.path.join(root_dir, 'project.properties')
1924 if os.path.isfile(proppath):
1925 with open(proppath, "r", encoding='iso-8859-1') as o:
1926 lines = o.readlines()
1928 with open(proppath, "w", encoding='iso-8859-1') as o:
1931 if line.startswith('android.library.reference.%d=' % number):
1932 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1937 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1940 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
1943 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
1944 """Verify that two apks are the same
1946 One of the inputs is signed, the other is unsigned. The signature metadata
1947 is transferred from the signed to the unsigned apk, and then jarsigner is
1948 used to verify that the signature from the signed apk is also varlid for
1950 :param signed_apk: Path to a signed apk file
1951 :param unsigned_apk: Path to an unsigned apk file expected to match it
1952 :param tmp_dir: Path to directory for temporary files
1953 :returns: None if the verification is successful, otherwise a string
1954 describing what went wrong.
1956 with ZipFile(signed_apk) as signed_apk_as_zip:
1957 meta_inf_files = ['META-INF/MANIFEST.MF']
1958 for f in signed_apk_as_zip.namelist():
1959 if apk_sigfile.match(f):
1960 meta_inf_files.append(f)
1961 if len(meta_inf_files) < 3:
1962 return "Signature files missing from {0}".format(signed_apk)
1963 signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
1964 with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
1965 for meta_inf_file in meta_inf_files:
1966 unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
1968 if subprocess.call([config['jarsigner'], '-verify', unsigned_apk]) != 0:
1969 logging.info("...NOT verified - {0}".format(signed_apk))
1970 return compare_apks(signed_apk, unsigned_apk, tmp_dir)
1971 logging.info("...successfully verified")
1975 apk_badchars = re.compile('''[/ :;'"]''')
1978 def compare_apks(apk1, apk2, tmp_dir):
1981 Returns None if the apk content is the same (apart from the signing key),
1982 otherwise a string describing what's different, or what went wrong when
1983 trying to do the comparison.
1986 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
1987 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
1988 for d in [apk1dir, apk2dir]:
1989 if os.path.exists(d):
1992 os.mkdir(os.path.join(d, 'jar-xf'))
1994 if subprocess.call(['jar', 'xf',
1995 os.path.abspath(apk1)],
1996 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
1997 return("Failed to unpack " + apk1)
1998 if subprocess.call(['jar', 'xf',
1999 os.path.abspath(apk2)],
2000 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2001 return("Failed to unpack " + apk2)
2003 # try to find apktool in the path, if it hasn't been manually configed
2004 if 'apktool' not in config:
2005 tmp = find_command('apktool')
2007 config['apktool'] = tmp
2008 if 'apktool' in config:
2009 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2011 return("Failed to unpack " + apk1)
2012 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2014 return("Failed to unpack " + apk2)
2016 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2017 lines = p.output.splitlines()
2018 if len(lines) != 1 or 'META-INF' not in lines[0]:
2019 meld = find_command('meld')
2020 if meld is not None:
2021 p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
2022 return("Unexpected diff output - " + p.output)
2024 # since everything verifies, delete the comparison to keep cruft down
2025 shutil.rmtree(apk1dir)
2026 shutil.rmtree(apk2dir)
2028 # If we get here, it seems like they're the same!
2032 def find_command(command):
2033 '''find the full path of a command, or None if it can't be found in the PATH'''
2036 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2038 fpath, fname = os.path.split(command)
2043 for path in os.environ["PATH"].split(os.pathsep):
2044 path = path.strip('"')
2045 exe_file = os.path.join(path, command)
2046 if is_exe(exe_file):
2053 '''generate a random password for when generating keys'''
2054 h = hashlib.sha256()
2055 h.update(os.urandom(16)) # salt
2056 h.update(socket.getfqdn().encode('utf-8'))
2057 passwd = base64.b64encode(h.digest()).strip()
2058 return passwd.decode('utf-8')
2061 def genkeystore(localconfig):
2062 '''Generate a new key with random passwords and add it to new keystore'''
2063 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2064 keystoredir = os.path.dirname(localconfig['keystore'])
2065 if keystoredir is None or keystoredir == '':
2066 keystoredir = os.path.join(os.getcwd(), keystoredir)
2067 if not os.path.exists(keystoredir):
2068 os.makedirs(keystoredir, mode=0o700)
2070 write_password_file("keystorepass", localconfig['keystorepass'])
2071 write_password_file("keypass", localconfig['keypass'])
2072 p = FDroidPopen([config['keytool'], '-genkey',
2073 '-keystore', localconfig['keystore'],
2074 '-alias', localconfig['repo_keyalias'],
2075 '-keyalg', 'RSA', '-keysize', '4096',
2076 '-sigalg', 'SHA256withRSA',
2077 '-validity', '10000',
2078 '-storepass:file', config['keystorepassfile'],
2079 '-keypass:file', config['keypassfile'],
2080 '-dname', localconfig['keydname']])
2081 # TODO keypass should be sent via stdin
2082 if p.returncode != 0:
2083 raise BuildException("Failed to generate key", p.output)
2084 os.chmod(localconfig['keystore'], 0o0600)
2085 # now show the lovely key that was just generated
2086 p = FDroidPopen([config['keytool'], '-list', '-v',
2087 '-keystore', localconfig['keystore'],
2088 '-alias', localconfig['repo_keyalias'],
2089 '-storepass:file', config['keystorepassfile']])
2090 logging.info(p.output.strip() + '\n\n')
2093 def write_to_config(thisconfig, key, value=None):
2094 '''write a key/value to the local config.py'''
2096 origkey = key + '_orig'
2097 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2098 with open('config.py', 'r', encoding='utf8') as f:
2100 pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
2101 repl = '\n' + key + ' = "' + value + '"'
2102 data = re.sub(pattern, repl, data)
2103 # if this key is not in the file, append it
2104 if not re.match('\s*' + key + '\s*=\s*"', data):
2106 # make sure the file ends with a carraige return
2107 if not re.match('\n$', data):
2109 with open('config.py', 'w', encoding='utf8') as f:
2113 def parse_xml(path):
2114 return XMLElementTree.parse(path).getroot()
2117 def string_is_integer(string):
2125 def get_per_app_repos():
2126 '''per-app repos are dirs named with the packageName of a single app'''
2128 # Android packageNames are Java packages, they may contain uppercase or
2129 # lowercase letters ('A' through 'Z'), numbers, and underscores
2130 # ('_'). However, individual package name parts may only start with
2131 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2132 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2135 for root, dirs, files in os.walk(os.getcwd()):
2137 print('checking', root, 'for', d)
2138 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2139 # standard parts of an fdroid repo, so never packageNames
2142 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
2148 def is_repo_file(filename):
2149 '''Whether the file in a repo is a build product to be delivered to users'''
2150 return os.path.isfile(filename) \
2151 and os.path.basename(filename) not in [