fdroidserver/mirror.py

   1 #!/usr/bin/env python3
   2
   3 import ipaddress
   4 import logging
   5 import os
   6 import posixpath
   7 import socket
   8 import subprocess
   9 import sys
  10 from argparse import ArgumentParser
  11 import urllib.parse
  12
  13 from . import _
  14 from . import common
  15 from . import index
  16 from . import update
  17
  18 options = None
  19
  20
  21 def main():
  22     global options
  23
  24     parser = ArgumentParser(usage=_("%(prog)s [options] url"))
  25     common.setup_global_opts(parser)
  26     parser.add_argument("url", nargs='?',
  27                         help=_('Base URL to mirror, can include the index signing key '
  28                                + 'using the query string: ?fingerprint='))
  29     parser.add_argument("--archive", action='store_true', default=False,
  30                         help=_("Also mirror the full archive section"))
  31     parser.add_argument("--output-dir", default=None,
  32                         help=_("The directory to write the mirror to"))
  33     options = parser.parse_args()
  34
  35     if options.url is None:
  36         logging.error(_('A URL is required as an argument!') + '\n')
  37         parser.print_help()
  38         sys.exit(1)
  39
  40     scheme, hostname, path, params, query, fragment = urllib.parse.urlparse(options.url)
  41     fingerprint = urllib.parse.parse_qs(query).get('fingerprint')
  42
  43     def _append_to_url_path(*args):
  44         '''Append the list of path components to URL, keeping the rest the same'''
  45         newpath = posixpath.join(path, *args)
  46         return urllib.parse.urlunparse((scheme, hostname, newpath, params, query, fragment))
  47
  48     if fingerprint:
  49         config = common.read_config(options)
  50         if not ('jarsigner' in config or 'apksigner' in config):
  51             logging.error(_('Java JDK not found! Install in standard location or set java_paths!'))
  52             sys.exit(1)
  53
  54         def _get_index(section, etag=None):
  55             url = _append_to_url_path(section)
  56             return index.download_repo_index(url, etag=etag)
  57     else:
  58         def _get_index(section, etag=None):
  59             import io
  60             import json
  61             import zipfile
  62             from . import net
  63             url = _append_to_url_path(section, 'index-v1.jar')
  64             content, etag = net.http_get(url)
  65             with zipfile.ZipFile(io.BytesIO(content)) as zip:
  66                 jsoncontents = zip.open('index-v1.json').read()
  67             data = json.loads(jsoncontents.decode('utf-8'))
  68             return data, etag
  69
  70     ip = None
  71     try:
  72         ip = ipaddress.ip_address(hostname)
  73     except ValueError:
  74         pass
  75     if hostname == 'f-droid.org' \
  76        or (ip is not None and hostname in socket.gethostbyname_ex('f-droid.org')[2]):
  77         print(_('ERROR: this command should never be used to mirror f-droid.org!\n'
  78                 'A full mirror of f-droid.org requires more than 200GB.'))
  79         sys.exit(1)
  80
  81     path = path.rstrip('/')
  82     if path.endswith('repo') or path.endswith('archive'):
  83         logging.warning(_('Do not include "{path}" in URL!')
  84                         .format(path=path.split('/')[-1]))
  85     elif not path.endswith('fdroid'):
  86         logging.warning(_('{url} does not end with "fdroid", check the URL path!')
  87                         .format(url=options.url))
  88
  89     icondirs = ['icons', ]
  90     for density in update.screen_densities:
  91         icondirs.append('icons-' + density)
  92
  93     if options.output_dir:
  94         basedir = options.output_dir
  95     else:
  96         basedir = os.path.join(os.getcwd(), hostname, path.strip('/'))
  97         os.makedirs(basedir, exist_ok=True)
  98
  99     if options.archive:
 100         sections = ('repo', 'archive')
 101     else:
 102         sections = ('repo', )
 103
 104     for section in sections:
 105         sectiondir = os.path.join(basedir, section)
 106
 107         data, etag = _get_index(section)
 108
 109         os.makedirs(sectiondir, exist_ok=True)
 110         os.chdir(sectiondir)
 111         for icondir in icondirs:
 112             os.makedirs(os.path.join(sectiondir, icondir), exist_ok=True)
 113
 114         urls = []
 115         for packageName, packageList in data['packages'].items():
 116             for package in packageList:
 117                 to_fetch = []
 118                 for k in ('apkName', 'srcname'):
 119                     if k in package:
 120                         to_fetch.append(package[k])
 121                     elif k == 'apkName':
 122                         logging.error(_('{appid} is missing {name}')
 123                                       .format(appid=package['packageName'], name=k))
 124                 for f in to_fetch:
 125                     if not os.path.exists(f) \
 126                        or (f.endswith('.apk') and os.path.getsize(f) != package['size']):
 127                         urls.append(_append_to_url_path(section, f))
 128                         urls.append(_append_to_url_path(section, f + '.asc'))
 129
 130         for app in data['apps']:
 131             localized = app.get('localized')
 132             if localized:
 133                 for locale, d in localized.items():
 134                     for k in update.GRAPHIC_NAMES:
 135                         f = d.get(k)
 136                         if f:
 137                             urls.append(_append_to_url_path(section, app['packageName'], locale, f))
 138                     for k in update.SCREENSHOT_DIRS:
 139                         filelist = d.get(k)
 140                         if filelist:
 141                             for f in filelist:
 142                                 urls.append(_append_to_url_path(section, app['packageName'], locale, k, f))
 143
 144         with open(urls_file, 'w') as fp:
 145             for url in urls:
 146                 fp.write(url.split('?')[0] + '\n')  # wget puts query string in the filename
 147         subprocess.call(['wget', '--continue', '--user-agent="fdroid mirror"',
 148                          '--input-file=' + urls_file])
 149         os.remove(urls_file)
 150
 151         urls = dict()
 152         for app in data['apps']:
 153             if 'icon' not in app:
 154                 logging.error(_('no "icon" in {appid}').format(appid=app['packageName']))
 155                 continue
 156             icon = app['icon']
 157             for icondir in icondirs:
 158                 url = _append_to_url_path(section, icondir, icon)
 159                 if icondir not in urls:
 160                     urls[icondir] = []
 161                 urls[icondir].append(url)
 162
 163         for icondir in icondirs:
 164             os.chdir(os.path.join(basedir, section, icondir))
 165             with open(urls_file, 'w') as fp:
 166                 for url in urls[icondir]:
 167                     fp.write(url.split('?')[0] + '\n')  # wget puts query string in the filename
 168             subprocess.call(['wget', '--continue', '--input-file=' + urls_file])
 169             os.remove(urls_file)
 170
 171
 172 if __name__ == "__main__":
 173     main()