fdroidserver/mirror.py

   1 #!/usr/bin/env python3
   2
   3 import ipaddress
   4 import logging
   5 import os
   6 import posixpath
   7 import socket
   8 import subprocess
   9 import sys
  10 from argparse import ArgumentParser
  11 import urllib.parse
  12
  13 from . import _
  14 from . import common
  15 from . import index
  16 from . import update
  17
  18 options = None
  19
  20
  21 def _run_wget(path, urls):
  22     if options.verbose:
  23         verbose = '--verbose'
  24     else:
  25         verbose = '--no-verbose'
  26
  27     if not urls:
  28         return
  29     logging.debug(_('Running wget in {path}').format(path=path))
  30     os.makedirs(path, exist_ok=True)
  31     os.chdir(path)
  32     urls_file = '.fdroid-mirror-wget-input-file'
  33     with open(urls_file, 'w') as fp:
  34         for url in urls:
  35             fp.write(url.split('?')[0] + '\n')  # wget puts query string in the filename
  36     subprocess.call(['wget', verbose, '--continue', '--user-agent="fdroid mirror"',
  37                      '--input-file=' + urls_file])
  38     os.remove(urls_file)
  39
  40
  41 def main():
  42     global options
  43
  44     parser = ArgumentParser(usage=_("%(prog)s [options] url"))
  45     common.setup_global_opts(parser)
  46     parser.add_argument("url", nargs='?',
  47                         help=_('Base URL to mirror, can include the index signing key '
  48                                + 'using the query string: ?fingerprint='))
  49     parser.add_argument("--archive", action='store_true', default=False,
  50                         help=_("Also mirror the full archive section"))
  51     parser.add_argument("--output-dir", default=None,
  52                         help=_("The directory to write the mirror to"))
  53     options = parser.parse_args()
  54
  55     if options.url is None:
  56         logging.error(_('A URL is required as an argument!') + '\n')
  57         parser.print_help()
  58         sys.exit(1)
  59
  60     scheme, hostname, path, params, query, fragment = urllib.parse.urlparse(options.url)
  61     fingerprint = urllib.parse.parse_qs(query).get('fingerprint')
  62
  63     def _append_to_url_path(*args):
  64         '''Append the list of path components to URL, keeping the rest the same'''
  65         newpath = posixpath.join(path, *args)
  66         return urllib.parse.urlunparse((scheme, hostname, newpath, params, query, fragment))
  67
  68     if fingerprint:
  69         config = common.read_config(options)
  70         if not ('jarsigner' in config or 'apksigner' in config):
  71             logging.error(_('Java JDK not found! Install in standard location or set java_paths!'))
  72             sys.exit(1)
  73
  74         def _get_index(section, etag=None):
  75             url = _append_to_url_path(section)
  76             return index.download_repo_index(url, etag=etag)
  77     else:
  78         def _get_index(section, etag=None):
  79             import io
  80             import json
  81             import zipfile
  82             from . import net
  83             url = _append_to_url_path(section, 'index-v1.jar')
  84             content, etag = net.http_get(url)
  85             with zipfile.ZipFile(io.BytesIO(content)) as zip:
  86                 jsoncontents = zip.open('index-v1.json').read()
  87             data = json.loads(jsoncontents.decode('utf-8'))
  88             return data, etag
  89
  90     ip = None
  91     try:
  92         ip = ipaddress.ip_address(hostname)
  93     except ValueError:
  94         pass
  95     if hostname == 'f-droid.org' \
  96        or (ip is not None and hostname in socket.gethostbyname_ex('f-droid.org')[2]):
  97         print(_('ERROR: this command should never be used to mirror f-droid.org!\n'
  98                 'A full mirror of f-droid.org requires more than 200GB.'))
  99         sys.exit(1)
 100
 101     path = path.rstrip('/')
 102     if path.endswith('repo') or path.endswith('archive'):
 103         logging.warning(_('Do not include "{path}" in URL!')
 104                         .format(path=path.split('/')[-1]))
 105     elif not path.endswith('fdroid'):
 106         logging.warning(_('{url} does not end with "fdroid", check the URL path!')
 107                         .format(url=options.url))
 108
 109     icondirs = ['icons', ]
 110     for density in update.screen_densities:
 111         icondirs.append('icons-' + density)
 112
 113     if options.output_dir:
 114         basedir = options.output_dir
 115     else:
 116         basedir = os.path.join(os.getcwd(), hostname, path.strip('/'))
 117         os.makedirs(basedir, exist_ok=True)
 118
 119     if options.archive:
 120         sections = ('repo', 'archive')
 121     else:
 122         sections = ('repo', )
 123
 124     for section in sections:
 125         sectiondir = os.path.join(basedir, section)
 126
 127         data, etag = _get_index(section)
 128
 129         os.makedirs(sectiondir, exist_ok=True)
 130         os.chdir(sectiondir)
 131         for icondir in icondirs:
 132             os.makedirs(os.path.join(sectiondir, icondir), exist_ok=True)
 133
 134         urls = []
 135         for packageName, packageList in data['packages'].items():
 136             for package in packageList:
 137                 to_fetch = []
 138                 for k in ('apkName', 'srcname'):
 139                     if k in package:
 140                         to_fetch.append(package[k])
 141                     elif k == 'apkName':
 142                         logging.error(_('{appid} is missing {name}')
 143                                       .format(appid=package['packageName'], name=k))
 144                 for f in to_fetch:
 145                     if not os.path.exists(f) \
 146                        or (f.endswith('.apk') and os.path.getsize(f) != package['size']):
 147                         urls.append(_append_to_url_path(section, f))
 148                         urls.append(_append_to_url_path(section, f + '.asc'))
 149         _run_wget(sectiondir, urls)
 150
 151         for app in data['apps']:
 152             localized = app.get('localized')
 153             if localized:
 154                 for locale, d in localized.items():
 155                     urls = []
 156                     components = (section, app['packageName'], locale)
 157                     for k in update.GRAPHIC_NAMES:
 158                         f = d.get(k)
 159                         if f:
 160                             filepath_tuple = components + (f, )
 161                             urls.append(_append_to_url_path(*filepath_tuple))
 162                     _run_wget(os.path.join(basedir, *components), urls)
 163                     for k in update.SCREENSHOT_DIRS:
 164                         urls = []
 165                         filelist = d.get(k)
 166                         if filelist:
 167                             components = (section, app['packageName'], locale, k)
 168                             for f in filelist:
 169                                 filepath_tuple = components + (f, )
 170                                 urls.append(_append_to_url_path(*filepath_tuple))
 171                             _run_wget(os.path.join(basedir, *components), urls)
 172
 173         urls = dict()
 174         for app in data['apps']:
 175             if 'icon' not in app:
 176                 logging.error(_('no "icon" in {appid}').format(appid=app['packageName']))
 177                 continue
 178             icon = app['icon']
 179             for icondir in icondirs:
 180                 url = _append_to_url_path(section, icondir, icon)
 181                 if icondir not in urls:
 182                     urls[icondir] = []
 183                 urls[icondir].append(url)
 184
 185         for icondir in icondirs:
 186             _run_wget(os.path.join(basedir, section, icondir), urls[icondir])
 187
 188
 189 if __name__ == "__main__":
 190     main()