Merge branch 'some-compatibility-fixes' of https://gitlab.com/eighthave/fdroidserver

diff --combined fdroidserver/build.py
index d29901bb006830cbc46bedbb77d8d1ce0a0f6b05,e6ac33994d86b1b8dc2acda39abb74855b62bca2..84af7f41c5b0497e422a36bf6aa7833771a59227
--- 1/fdroidserver/build.py
--- 2/fdroidserver/build.py
+++ b/fdroidserver/build.py
@@@ -452,7 -452,7 +452,7 @@@ def capitalize_intact(string)
      return string[0].upper() + string[1:]
  
  
 -def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver):
 +def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh):
      """Do a build locally."""
  
      if thisbuild['buildjni'] and thisbuild['buildjni'] != ['no']:
@@@ -479,7 -479,7 +479,7 @@@
      # Prepare the source code...
      root_dir, srclibpaths = common.prepare_source(vcs, app, thisbuild,
                                                    build_dir, srclib_dir,
 -                                                  extlib_dir, onserver)
 +                                                  extlib_dir, onserver, refresh)
  
      # We need to clean via the build tool in case the binary dirs are
      # different from the default ones
@@@ -876,7 -876,7 +876,7 @@@
  
  
  def trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir,
 -             tmp_dir, repo_dir, vcs, test, server, force, onserver):
 +             tmp_dir, repo_dir, vcs, test, server, force, onserver, refresh):
      """
      Build a particular version of an application, if it needs building.
  
@@@ -921,7 -921,7 +921,7 @@@
  
          build_server(app, thisbuild, vcs, build_dir, output_dir, force)
      else:
 -        build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver)
 +        build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_dir, tmp_dir, force, onserver, refresh)
      return True
  
  
@@@ -949,8 -949,6 +949,8 @@@ def parse_commandline()
                        help="Skip scanning the source code for binaries and other problems")
      parser.add_option("--no-tarball", dest="notarball", action="store_true", default=False,
                        help="Don't create a source tarball, useful when testing a build")
 +    parser.add_option("--no-refresh", dest="refresh", action="store_false", default=True,
 +                      help="Don't refresh the repository, useful when testing a build with no internet connection")
      parser.add_option("-f", "--force", action="store_true", default=False,
                        help="Force build of disabled apps, and carries on regardless of scan problems. Only allowed in test mode.")
      parser.add_option("-a", "--all", action="store_true", default=False,
@@@ -1076,7 -1074,7 +1076,7 @@@ def main()
                              also_check_dir, srclib_dir, extlib_dir,
                              tmp_dir, repo_dir, vcs, options.test,
                              options.server, options.force,
 -                            options.onserver):
 +                            options.onserver, options.refresh):
  
                      if app.get('Binaries', None):
                          # This is an app where we build from source, and
@@@ -1090,9 -1088,7 +1090,7 @@@
                          logging.info("...retrieving " + url)
                          of = "{0}_{1}.apk.binary".format(app['id'], thisbuild['vercode'])
                          of = os.path.join(output_dir, of)
-                         p = FDroidPopen(['wget', '-nv', '-O', of, url])
-                         if p.returncode != 0 or not os.path.exists(of):
-                             raise BuildException("...failed to retrieve " + url)
+                         common.download_file(url, local_filename=of)
  
                      build_succeeded.append(app)
                      wikilog = "Build succeeded"

diff --combined fdroidserver/common.py
index d427c0d0b596f8dfdc6c94a04841d6795e099575,9daddb9c7b7e852e6448234926ed2de25f49531d..3c672728a860544a7d74a72783503927e93b8cc9
--- 1/fdroidserver/common.py
--- 2/fdroidserver/common.py
+++ b/fdroidserver/common.py
@@@ -22,6 -22,7 +22,7 @@@ import sy
  import re
  import shutil
  import glob
+ import requests
  import stat
  import subprocess
  import time
@@@ -70,8 -71,8 +71,8 @@@ default_config = 
      'keystore': 'keystore.jks',
      'smartcardoptions': [],
      'char_limits': {
-         'Summary': 50,
-         'Description': 1500
+         'Summary': 80,
+         'Description': 4000
      },
      'keyaliases': {},
      'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
@@@ -457,7 -458,7 +458,7 @@@ class vcs
      # lifetime of the vcs object.
      # None is acceptable for 'rev' if you know you are cloning a clean copy of
      # the repo - otherwise it must specify a valid revision.
 -    def gotorevision(self, rev):
 +    def gotorevision(self, rev, refresh=True):
  
          if self.clone_failed:
              raise VCSException("Downloading the repository already failed once, not trying again.")
@@@ -488,8 -489,6 +489,8 @@@
              shutil.rmtree(self.local)
  
          exc = None
 +        if not refresh:
 +            self.refreshed = True
  
          try:
              self.gotorevisionx(rev)
@@@ -1097,7 -1096,7 +1098,7 @@@ class BuildException(FDroidException)
  # it, which may be a subdirectory of the actual project. If you want the base
  # directory of the project, pass 'basepath=True'.
  def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
 -              raw=False, prepare=True, preponly=False):
 +              raw=False, prepare=True, preponly=False, refresh=True):
  
      number = None
      subdir = None
@@@ -1122,7 -1121,7 +1123,7 @@@
          vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
          vcs.srclib = (name, number, sdir)
          if ref:
 -            vcs.gotorevision(ref)
 +            vcs.gotorevision(ref, refresh)
  
          if raw:
              return vcs
@@@ -1173,7 -1172,7 +1174,7 @@@
  #   'root' is the root directory, which may be the same as 'build_dir' or may
  #          be a subdirectory of it.
  #   'srclibpaths' is information on the srclibs being used
 -def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False):
 +def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
  
      # Optionally, the actual app source can be in a subdirectory
      if build['subdir']:
@@@ -1183,7 -1182,7 +1184,7 @@@
  
      # Get a working copy of the right revision
      logging.info("Getting source for revision " + build['commit'])
 -    vcs.gotorevision(build['commit'])
 +    vcs.gotorevision(build['commit'], refresh)
  
      # Initialise submodules if required
      if build['submodules']:
@@@ -1221,7 -1220,7 +1222,7 @@@
      if build['srclibs']:
          logging.info("Collecting source libraries")
          for lib in build['srclibs']:
 -            srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver))
 +            srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
  
      for name, number, libpath in srclibpaths:
          place_srclib(root_dir, int(number) if number else None, libpath)
@@@ -2072,3 -2071,17 +2073,17 @@@ def string_is_integer(string)
          return True
      except ValueError:
          return False
+ 
+ 
+ def download_file(url, local_filename=None, dldir='tmp'):
+     filename = url.split('/')[-1]
+     if local_filename is None:
+         local_filename = os.path.join(dldir, filename)
+     # the stream=True parameter keeps memory usage low
+     r = requests.get(url, stream=True)
+     with open(local_filename, 'wb') as f:
+         for chunk in r.iter_content(chunk_size=1024):
+             if chunk:  # filter out keep-alive new chunks
+                 f.write(chunk)
+                 f.flush()
+     return local_filename