From: Daniel Martí Date: Mon, 27 Jul 2015 17:41:23 +0000 (-0700) Subject: Merge branch 'some-compatibility-fixes' of https://gitlab.com/eighthave/fdroidserver X-Git-Tag: 0.4.0~11 X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=commitdiff_plain;h=b4d768b225aad3e7855763a5b8ba25b825b5a937;hp=-c;p=fdroidserver.git Merge branch 'some-compatibility-fixes' of https://gitlab.com/eighthave/fdroidserver --- b4d768b225aad3e7855763a5b8ba25b825b5a937 diff --combined fdroidserver/build.py index d29901bb,e6ac3399..84af7f41 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@@ -452,7 -452,7 +452,7 @@@ def capitalize_intact(string) return string[0].upper() + string[1:] -def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver): +def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh): """Do a build locally.""" if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']: @@@ -479,7 -479,7 +479,7 @@@ # Prepare the source code... root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild, build_dir, srclib_dir, - extlib_dir, onserver) + extlib_dir, onserver, refresh) # We need to clean via the build tool in case the binary dirs are # different from the default ones @@@ -876,7 -876,7 +876,7 @@@ def trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir, - tmp_dir, repo_dir, vcs, test, server, force, onserver): + tmp_dir, repo_dir, vcs, test, server, force, onserver, refresh): """ Build a particular version of an application, if it needs building. @@@ -921,7 -921,7 +921,7 @@@ build_server(app, thisbuild, vcs, build_dir, output_dir, force) else: - build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver) + build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh) return True @@@ -949,8 -949,6 +949,8 @@@ def parse_commandline() help="Skip scanning the source code for binaries and other problems") parser.add_option("--no-tarball", dest="notarball", action="store_true", default=False, help="Don't create a source tarball, useful when testing a build") + parser.add_option("--no-refresh", dest="refresh", action="store_false", default=True, + help="Don't refresh the repository, useful when testing a build with no internet connection") parser.add_option("-f", "--force", action="store_true", default=False, help="Force build of disabled apps, and carries on regardless of scan problems. Only allowed in test mode.") parser.add_option("-a", "--all", action="store_true", default=False, @@@ -1076,7 -1074,7 +1076,7 @@@ def main() also_check_dir, srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, options.test, options.server, options.force, - options.onserver): + options.onserver, options.refresh): if app.get('Binaries', None): # This is an app where we build from source, and @@@ -1090,9 -1088,7 +1090,7 @@@ logging.info("...retrieving " + url) of = "{0}_{1}.apk.binary".format(app['id'], thisbuild['vercode']) of = os.path.join(output_dir, of) - p = FDroidPopen(['wget', '-nv', '-O', of, url]) - if p.returncode != 0 or not os.path.exists(of): - raise BuildException("...failed to retrieve " + url) + common.download_file(url, local_filename=of) build_succeeded.append(app) wikilog = "Build succeeded" diff --combined fdroidserver/common.py index d427c0d0,9daddb9c..3c672728 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@@ -22,6 -22,7 +22,7 @@@ import sy import re import shutil import glob + import requests import stat import subprocess import time @@@ -70,8 -71,8 +71,8 @@@ default_config = 'keystore': 'keystore.jks', 'smartcardoptions': [], 'char_limits': { - 'Summary': 50, - 'Description': 1500 + 'Summary': 80, + 'Description': 4000 }, 'keyaliases': {}, 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo", @@@ -457,7 -458,7 +458,7 @@@ class vcs # lifetime of the vcs object. # None is acceptable for 'rev' if you know you are cloning a clean copy of # the repo - otherwise it must specify a valid revision. - def gotorevision(self, rev): + def gotorevision(self, rev, refresh=True): if self.clone_failed: raise VCSException("Downloading the repository already failed once, not trying again.") @@@ -488,8 -489,6 +489,8 @@@ shutil.rmtree(self.local) exc = None + if not refresh: + self.refreshed = True try: self.gotorevisionx(rev) @@@ -1097,7 -1096,7 +1098,7 @@@ class BuildException(FDroidException) # it, which may be a subdirectory of the actual project. If you want the base # directory of the project, pass 'basepath=True'. def getsrclib(spec, srclib_dir, subdir=None, basepath=False, - raw=False, prepare=True, preponly=False): + raw=False, prepare=True, preponly=False, refresh=True): number = None subdir = None @@@ -1122,7 -1121,7 +1123,7 @@@ vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir) vcs.srclib = (name, number, sdir) if ref: - vcs.gotorevision(ref) + vcs.gotorevision(ref, refresh) if raw: return vcs @@@ -1173,7 -1172,7 +1174,7 @@@ # 'root' is the root directory, which may be the same as 'build_dir' or may # be a subdirectory of it. # 'srclibpaths' is information on the srclibs being used -def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False): +def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True): # Optionally, the actual app source can be in a subdirectory if build['subdir']: @@@ -1183,7 -1182,7 +1184,7 @@@ # Get a working copy of the right revision logging.info("Getting source for revision " + build['commit']) - vcs.gotorevision(build['commit']) + vcs.gotorevision(build['commit'], refresh) # Initialise submodules if required if build['submodules']: @@@ -1221,7 -1220,7 +1222,7 @@@ if build['srclibs']: logging.info("Collecting source libraries") for lib in build['srclibs']: - srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver)) + srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh)) for name, number, libpath in srclibpaths: place_srclib(root_dir, int(number) if number else None, libpath) @@@ -2072,3 -2071,17 +2073,17 @@@ def string_is_integer(string) return True except ValueError: return False + + + def download_file(url, local_filename=None, dldir='tmp'): + filename = url.split('/')[-1] + if local_filename is None: + local_filename = os.path.join(dldir, filename) + # the stream=True parameter keeps memory usage low + r = requests.get(url, stream=True) + with open(local_filename, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024): + if chunk: # filter out keep-alive new chunks + f.write(chunk) + f.flush() + return local_filename