fdroidserver/mirror.py

   1 #!/usr/bin/env python3
   2
   3 import ipaddress
   4 import logging
   5 import os
   6 import posixpath
   7 import socket
   8 import subprocess
   9 import sys
  10 from argparse import ArgumentParser
  11 import urllib.parse
  12
  13 from . import _
  14 from . import common
  15 from . import index
  16 from . import update
  17
  18 options = None
  19
  20
  21 def _run_wget(path, urls):
  22     if options.verbose:
  23         verbose = '--verbose'
  24     else:
  25         verbose = '--no-verbose'
  26
  27     os.chdir(path)
  28     urls_file = '.fdroid-mirror-wget-input-file'
  29     with open(urls_file, 'w') as fp:
  30         for url in urls:
  31             fp.write(url.split('?')[0] + '\n')  # wget puts query string in the filename
  32     subprocess.call(['wget', verbose, '--continue', '--user-agent="fdroid mirror"',
  33                      '--input-file=' + urls_file])
  34     os.remove(urls_file)
  35
  36
  37 def main():
  38     global options
  39
  40     parser = ArgumentParser(usage=_("%(prog)s [options] url"))
  41     common.setup_global_opts(parser)
  42     parser.add_argument("url", nargs='?',
  43                         help=_('Base URL to mirror, can include the index signing key '
  44                                + 'using the query string: ?fingerprint='))
  45     parser.add_argument("--archive", action='store_true', default=False,
  46                         help=_("Also mirror the full archive section"))
  47     parser.add_argument("--output-dir", default=None,
  48                         help=_("The directory to write the mirror to"))
  49     options = parser.parse_args()
  50
  51     if options.url is None:
  52         logging.error(_('A URL is required as an argument!') + '\n')
  53         parser.print_help()
  54         sys.exit(1)
  55
  56     scheme, hostname, path, params, query, fragment = urllib.parse.urlparse(options.url)
  57     fingerprint = urllib.parse.parse_qs(query).get('fingerprint')
  58
  59     def _append_to_url_path(*args):
  60         '''Append the list of path components to URL, keeping the rest the same'''
  61         newpath = posixpath.join(path, *args)
  62         return urllib.parse.urlunparse((scheme, hostname, newpath, params, query, fragment))
  63
  64     if fingerprint:
  65         config = common.read_config(options)
  66         if not ('jarsigner' in config or 'apksigner' in config):
  67             logging.error(_('Java JDK not found! Install in standard location or set java_paths!'))
  68             sys.exit(1)
  69
  70         def _get_index(section, etag=None):
  71             url = _append_to_url_path(section)
  72             return index.download_repo_index(url, etag=etag)
  73     else:
  74         def _get_index(section, etag=None):
  75             import io
  76             import json
  77             import zipfile
  78             from . import net
  79             url = _append_to_url_path(section, 'index-v1.jar')
  80             content, etag = net.http_get(url)
  81             with zipfile.ZipFile(io.BytesIO(content)) as zip:
  82                 jsoncontents = zip.open('index-v1.json').read()
  83             data = json.loads(jsoncontents.decode('utf-8'))
  84             return data, etag
  85
  86     ip = None
  87     try:
  88         ip = ipaddress.ip_address(hostname)
  89     except ValueError:
  90         pass
  91     if hostname == 'f-droid.org' \
  92        or (ip is not None and hostname in socket.gethostbyname_ex('f-droid.org')[2]):
  93         print(_('ERROR: this command should never be used to mirror f-droid.org!\n'
  94                 'A full mirror of f-droid.org requires more than 200GB.'))
  95         sys.exit(1)
  96
  97     path = path.rstrip('/')
  98     if path.endswith('repo') or path.endswith('archive'):
  99         logging.warning(_('Do not include "{path}" in URL!')
 100                         .format(path=path.split('/')[-1]))
 101     elif not path.endswith('fdroid'):
 102         logging.warning(_('{url} does not end with "fdroid", check the URL path!')
 103                         .format(url=options.url))
 104
 105     icondirs = ['icons', ]
 106     for density in update.screen_densities:
 107         icondirs.append('icons-' + density)
 108
 109     if options.output_dir:
 110         basedir = options.output_dir
 111     else:
 112         basedir = os.path.join(os.getcwd(), hostname, path.strip('/'))
 113         os.makedirs(basedir, exist_ok=True)
 114
 115     if options.archive:
 116         sections = ('repo', 'archive')
 117     else:
 118         sections = ('repo', )
 119
 120     for section in sections:
 121         sectiondir = os.path.join(basedir, section)
 122
 123         data, etag = _get_index(section)
 124
 125         os.makedirs(sectiondir, exist_ok=True)
 126         os.chdir(sectiondir)
 127         for icondir in icondirs:
 128             os.makedirs(os.path.join(sectiondir, icondir), exist_ok=True)
 129
 130         urls = []
 131         for packageName, packageList in data['packages'].items():
 132             for package in packageList:
 133                 to_fetch = []
 134                 for k in ('apkName', 'srcname'):
 135                     if k in package:
 136                         to_fetch.append(package[k])
 137                     elif k == 'apkName':
 138                         logging.error(_('{appid} is missing {name}')
 139                                       .format(appid=package['packageName'], name=k))
 140                 for f in to_fetch:
 141                     if not os.path.exists(f) \
 142                        or (f.endswith('.apk') and os.path.getsize(f) != package['size']):
 143                         urls.append(_append_to_url_path(section, f))
 144                         urls.append(_append_to_url_path(section, f + '.asc'))
 145
 146         for app in data['apps']:
 147             localized = app.get('localized')
 148             if localized:
 149                 for locale, d in localized.items():
 150                     for k in update.GRAPHIC_NAMES:
 151                         f = d.get(k)
 152                         if f:
 153                             urls.append(_append_to_url_path(section, app['packageName'], locale, f))
 154                     for k in update.SCREENSHOT_DIRS:
 155                         filelist = d.get(k)
 156                         if filelist:
 157                             for f in filelist:
 158                                 urls.append(_append_to_url_path(section, app['packageName'], locale, k, f))
 159
 160         _run_wget(sectiondir, urls)
 161
 162         urls = dict()
 163         for app in data['apps']:
 164             if 'icon' not in app:
 165                 logging.error(_('no "icon" in {appid}').format(appid=app['packageName']))
 166                 continue
 167             icon = app['icon']
 168             for icondir in icondirs:
 169                 url = _append_to_url_path(section, icondir, icon)
 170                 if icondir not in urls:
 171                     urls[icondir] = []
 172                 urls[icondir].append(url)
 173
 174         for icondir in icondirs:
 175             _run_wget(os.path.join(basedir, section, icondir), urls[icondir])
 176
 177
 178 if __name__ == "__main__":
 179     main()