fdroidserver/mirror.py

   1 #!/usr/bin/env python3
   2
   3 import io
   4 import ipaddress
   5 import json
   6 import logging
   7 import os
   8 import socket
   9 import subprocess
  10 import sys
  11 import zipfile
  12 from argparse import ArgumentParser
  13 from urllib.parse import urlparse
  14
  15 from . import _
  16 from . import common
  17 from . import net
  18 from . import update
  19
  20 options = None
  21
  22
  23 def main():
  24     global options
  25
  26     parser = ArgumentParser(usage=_("%(prog)s [options] url"))
  27     common.setup_global_opts(parser)
  28     parser.add_argument("url", nargs='?', help=_("Base URL to mirror"))
  29     parser.add_argument("--archive", action='store_true', default=False,
  30                         help=_("Also mirror the full archive section"))
  31     parser.add_argument("--output-dir", default=os.getcwd(),
  32                         help=_("The directory to write the mirror to"))
  33     options = parser.parse_args()
  34
  35     if options.url is None:
  36         logging.error(_('A URL is required as an argument!') + '\n')
  37         parser.print_help()
  38         sys.exit(1)
  39
  40     baseurl = options.url
  41     basedir = options.output_dir
  42
  43     url = urlparse(baseurl)
  44     hostname = url.netloc
  45     ip = None
  46     try:
  47         ip = ipaddress.ip_address(hostname)
  48     except ValueError:
  49         pass
  50     if hostname == 'f-droid.org' \
  51        or (ip is not None and hostname in socket.gethostbyname_ex('f-droid.org')[2]):
  52         print(_('ERROR: this command should never be used to mirror f-droid.org!\n'
  53                 'A full mirror of f-droid.org requires more than 200GB.'))
  54         sys.exit(1)
  55
  56     path = url.path.rstrip('/')
  57     if path.endswith('repo') or path.endswith('archive'):
  58         logging.error(_('Do not include "{path}" in URL!').format(path=path.split('/')[-1]))
  59         sys.exit(1)
  60     elif not path.endswith('fdroid'):
  61         logging.warning(_('{url} does not end with "fdroid", check the URL path!')
  62                         .format(url=baseurl))
  63
  64     icondirs = ['icons', ]
  65     for density in update.screen_densities:
  66         icondirs.append('icons-' + density)
  67
  68     if options.archive:
  69         sections = ('repo', 'archive')
  70     else:
  71         sections = ('repo', )
  72
  73     for section in sections:
  74         sectionurl = baseurl + '/' + section
  75         sectiondir = os.path.join(basedir, section)
  76         repourl = sectionurl + '/index-v1.jar'
  77
  78         content, etag = net.http_get(repourl)
  79         with zipfile.ZipFile(io.BytesIO(content)) as zip:
  80             jsoncontents = zip.open('index-v1.json').read()
  81
  82         os.makedirs(sectiondir, exist_ok=True)
  83         os.chdir(sectiondir)
  84         for icondir in icondirs:
  85             os.makedirs(os.path.join(sectiondir, icondir), exist_ok=True)
  86
  87         data = json.loads(jsoncontents.decode('utf-8'))
  88         urls = ''
  89         for packageName, packageList in data['packages'].items():
  90             for package in packageList:
  91                 to_fetch = []
  92                 for k in ('apkName', 'srcname'):
  93                     if k in package:
  94                         to_fetch.append(package[k])
  95                     elif k == 'apkName':
  96                         logging.error(_('{appid} is missing {name}')
  97                                       .format(appid=package['packageName'], name=k))
  98                 for f in to_fetch:
  99                     if not os.path.exists(f) \
 100                        or (f.endswith('.apk') and os.path.getsize(f) != package['size']):
 101                         url = sectionurl + '/' + f
 102                         urls += url + '\n'
 103                         urls += url + '.asc\n'
 104
 105         for app in data['apps']:
 106             localized = app.get('localized')
 107             if localized:
 108                 for locale, d in localized.items():
 109                     for k in update.GRAPHIC_NAMES:
 110                         f = d.get(k)
 111                         if f:
 112                             urls += '/'.join((sectionurl, locale, f)) + '\n'
 113                     for k in update.SCREENSHOT_DIRS:
 114                         filelist = d.get(k)
 115                         if filelist:
 116                             for f in filelist:
 117                                 urls += '/'.join((sectionurl, locale, k, f)) + '\n'
 118
 119         with open('.rsync-input-file', 'w') as fp:
 120             fp.write(urls)
 121         subprocess.call(['wget', '--continue', '--user-agent="fdroid mirror"',
 122                          '--input-file=.rsync-input-file'])
 123         os.remove('.rsync-input-file')
 124
 125         urls = dict()
 126         for app in data['apps']:
 127             if 'icon' not in app:
 128                 logging.error(_('no "icon" in {appid}').format(appid=app['packageName']))
 129                 continue
 130             icon = app['icon']
 131             for icondir in icondirs:
 132                 url = sectionurl + '/' + icondir + '/' + icon
 133                 if icondir not in urls:
 134                     urls[icondir] = ''
 135                 urls[icondir] += url + '\n'
 136
 137         for icondir in icondirs:
 138             os.chdir(os.path.join(basedir, section, icondir))
 139             with open('.rsync-input-file', 'w') as fp:
 140                 fp.write(urls[icondir])
 141             subprocess.call(['wget', '--continue', '--input-file=.rsync-input-file'])
 142             os.remove('.rsync-input-file')
 143
 144
 145 if __name__ == "__main__":
 146     main()