10 from argparse import ArgumentParser
21 def _run_wget(path, urls):
25 verbose = '--no-verbose'
29 logging.debug(_('Running wget in {path}').format(path=path))
30 os.makedirs(path, exist_ok=True)
32 urls_file = '.fdroid-mirror-wget-input-file'
33 with open(urls_file, 'w') as fp:
35 fp.write(url.split('?')[0] + '\n') # wget puts query string in the filename
36 subprocess.call(['wget', verbose, '--continue', '--user-agent="fdroid mirror"',
37 '--input-file=' + urls_file])
44 parser = ArgumentParser(usage=_("%(prog)s [options] url"))
45 common.setup_global_opts(parser)
46 parser.add_argument("url", nargs='?',
47 help=_('Base URL to mirror, can include the index signing key '
48 + 'using the query string: ?fingerprint='))
49 parser.add_argument("--archive", action='store_true', default=False,
50 help=_("Also mirror the full archive section"))
51 parser.add_argument("--output-dir", default=None,
52 help=_("The directory to write the mirror to"))
53 options = parser.parse_args()
55 if options.url is None:
56 logging.error(_('A URL is required as an argument!') + '\n')
60 scheme, hostname, path, params, query, fragment = urllib.parse.urlparse(options.url)
61 fingerprint = urllib.parse.parse_qs(query).get('fingerprint')
63 def _append_to_url_path(*args):
64 '''Append the list of path components to URL, keeping the rest the same'''
65 newpath = posixpath.join(path, *args)
66 return urllib.parse.urlunparse((scheme, hostname, newpath, params, query, fragment))
69 config = common.read_config(options)
70 if not ('jarsigner' in config or 'apksigner' in config):
71 logging.error(_('Java JDK not found! Install in standard location or set java_paths!'))
74 def _get_index(section, etag=None):
75 url = _append_to_url_path(section)
76 return index.download_repo_index(url, etag=etag)
78 def _get_index(section, etag=None):
83 url = _append_to_url_path(section, 'index-v1.jar')
84 content, etag = net.http_get(url)
85 with zipfile.ZipFile(io.BytesIO(content)) as zip:
86 jsoncontents = zip.open('index-v1.json').read()
87 data = json.loads(jsoncontents.decode('utf-8'))
92 ip = ipaddress.ip_address(hostname)
95 if hostname == 'f-droid.org' \
96 or (ip is not None and hostname in socket.gethostbyname_ex('f-droid.org')[2]):
97 print(_('ERROR: this command should never be used to mirror f-droid.org!\n'
98 'A full mirror of f-droid.org requires more than 200GB.'))
101 path = path.rstrip('/')
102 if path.endswith('repo') or path.endswith('archive'):
103 logging.warning(_('Do not include "{path}" in URL!')
104 .format(path=path.split('/')[-1]))
105 elif not path.endswith('fdroid'):
106 logging.warning(_('{url} does not end with "fdroid", check the URL path!')
107 .format(url=options.url))
109 icondirs = ['icons', ]
110 for density in update.screen_densities:
111 icondirs.append('icons-' + density)
113 if options.output_dir:
114 basedir = options.output_dir
116 basedir = os.path.join(os.getcwd(), hostname, path.strip('/'))
117 os.makedirs(basedir, exist_ok=True)
120 sections = ('repo', 'archive')
122 sections = ('repo', )
124 for section in sections:
125 sectiondir = os.path.join(basedir, section)
127 data, etag = _get_index(section)
129 os.makedirs(sectiondir, exist_ok=True)
131 for icondir in icondirs:
132 os.makedirs(os.path.join(sectiondir, icondir), exist_ok=True)
135 for packageName, packageList in data['packages'].items():
136 for package in packageList:
138 for k in ('apkName', 'srcname'):
140 to_fetch.append(package[k])
142 logging.error(_('{appid} is missing {name}')
143 .format(appid=package['packageName'], name=k))
145 if not os.path.exists(f) \
146 or (f.endswith('.apk') and os.path.getsize(f) != package['size']):
147 urls.append(_append_to_url_path(section, f))
148 urls.append(_append_to_url_path(section, f + '.asc'))
149 _run_wget(sectiondir, urls)
151 for app in data['apps']:
152 localized = app.get('localized')
154 for locale, d in localized.items():
156 components = (section, app['packageName'], locale)
157 for k in update.GRAPHIC_NAMES:
160 filepath_tuple = components + (f, )
161 urls.append(_append_to_url_path(*filepath_tuple))
162 _run_wget(os.path.join(basedir, *components), urls)
163 for k in update.SCREENSHOT_DIRS:
167 components = (section, app['packageName'], locale, k)
169 filepath_tuple = components + (f, )
170 urls.append(_append_to_url_path(*filepath_tuple))
171 _run_wget(os.path.join(basedir, *components), urls)
174 for app in data['apps']:
175 if 'icon' not in app:
176 logging.error(_('no "icon" in {appid}').format(appid=app['packageName']))
179 for icondir in icondirs:
180 url = _append_to_url_path(section, icondir, icon)
181 if icondir not in urls:
183 urls[icondir].append(url)
185 for icondir in icondirs:
186 _run_wget(os.path.join(basedir, section, icondir), urls[icondir])
189 if __name__ == "__main__":