3 # btlog.py - part of the FDroid server tools
4 # Copyright (C) 2017, Hans-Christoph Steiner <hans@eds.org>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU Affero General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU Affero General Public License for more details.
16 # You should have received a copy of the GNU Affero General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
19 # This is for creating a binary transparency log in a git repo for any
20 # F-Droid repo accessible via HTTP. It is meant to run very often,
21 # even once a minute in a cronjob, so it uses HEAD requests and the
22 # HTTP ETag to check if the file has changed. HEAD requests should
23 # not count against the download counts. This pattern of a HEAD then
24 # a GET is what fdroidclient uses to avoid ETags being abused as
25 # cookies. This also uses the same HTTP User Agent as the F-Droid
26 # client app so its not easy for the server to distinguish this from
39 import xml.dom.minidom
41 from argparse import ArgumentParser
46 from .exception import FDroidException
52 def make_binary_transparency_log(repodirs, btrepo='binary_transparency',
54 commit_title='fdroid update'):
55 '''Log the indexes in a standalone git repo to serve as a "binary
58 see: https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies
62 logging.info('Committing indexes to ' + btrepo)
63 if os.path.exists(os.path.join(btrepo, '.git')):
64 gitrepo = git.Repo(btrepo)
66 if not os.path.exists(btrepo):
68 gitrepo = git.Repo.init(btrepo)
71 url = common.config['repo_url'].rstrip('/')
72 with open(os.path.join(btrepo, 'README.md'), 'w') as fp:
74 # Binary Transparency Log for %s
76 This is a log of the signed app index metadata. This is stored in a
77 git repo, which serves as an imperfect append-only storage mechanism.
78 People can then check that any file that they received from that
79 F-Droid repository was a publicly released file.
81 For more info on this idea:
82 * https://wiki.mozilla.org/Security/Binary_Transparency
83 """ % url[:url.rindex('/')]) # strip '/repo'
84 gitrepo.index.add(['README.md', ])
85 gitrepo.index.commit('add README')
87 for repodir in repodirs:
88 cpdir = os.path.join(btrepo, repodir)
89 if not os.path.exists(cpdir):
91 for f in ('index.xml', 'index-v1.json'):
92 repof = os.path.join(repodir, f)
93 if not os.path.exists(repof):
95 dest = os.path.join(cpdir, f)
96 if f.endswith('.xml'):
97 doc = xml.dom.minidom.parse(repof)
98 output = doc.toprettyxml(encoding='utf-8')
99 with open(dest, 'wb') as f:
101 elif f.endswith('.json'):
102 with open(repof) as fp:
103 output = json.load(fp, object_pairs_hook=collections.OrderedDict)
104 with open(dest, 'w') as fp:
105 json.dump(output, fp, indent=2)
106 gitrepo.index.add([repof, ])
107 for f in ('index.jar', 'index-v1.jar'):
108 repof = os.path.join(repodir, f)
109 if not os.path.exists(repof):
111 dest = os.path.join(cpdir, f)
112 jarin = zipfile.ZipFile(repof, 'r')
113 jarout = zipfile.ZipFile(dest, 'w')
114 for info in jarin.infolist():
115 if info.filename.startswith('META-INF/'):
116 jarout.writestr(info, jarin.read(info.filename))
119 gitrepo.index.add([repof, ])
122 for root, dirs, files in os.walk(repodir):
124 output_files.append(os.path.relpath(os.path.join(root, f), repodir))
125 output = collections.OrderedDict()
126 for f in sorted(output_files):
127 repofile = os.path.join(repodir, f)
128 stat = os.stat(repofile)
137 fslogfile = os.path.join(cpdir, 'filesystemlog.json')
138 with open(fslogfile, 'w') as fp:
139 json.dump(output, fp, indent=2)
140 gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ])
142 for f in glob.glob(os.path.join(cpdir, '*.HTTP-headers.json')):
143 gitrepo.index.add([os.path.join(repodir, os.path.basename(f)), ])
145 gitrepo.index.commit(commit_title)
151 parser = ArgumentParser(usage="%(prog)s [options]")
152 common.setup_global_opts(parser)
153 parser.add_argument("--git-repo",
154 default=os.path.join(os.getcwd(), 'binary_transparency'),
155 help=_("Path to the git repo to use as the log"))
156 parser.add_argument("-u", "--url", default='https://f-droid.org',
157 help=_("The base URL for the repo to log (default: https://f-droid.org)"))
158 parser.add_argument("--git-remote", default=None,
159 help=_("Push the log to this git remote repository"))
160 options = parser.parse_args()
163 logging.getLogger("requests").setLevel(logging.INFO)
164 logging.getLogger("urllib3").setLevel(logging.INFO)
166 logging.getLogger("requests").setLevel(logging.WARNING)
167 logging.getLogger("urllib3").setLevel(logging.WARNING)
169 if not os.path.exists(options.git_repo):
170 raise FDroidException(
171 '"%s" does not exist! Create it, or use --git-repo' % options.git_repo)
173 session = requests.Session()
176 repodirs = ('repo', 'archive')
177 tempdirbase = tempfile.mkdtemp(prefix='.fdroid-btlog-')
178 for repodir in repodirs:
179 # TODO read HTTP headers for etag from git repo
180 tempdir = os.path.join(tempdirbase, repodir)
181 os.makedirs(tempdir, exist_ok=True)
182 gitrepodir = os.path.join(options.git_repo, repodir)
183 os.makedirs(gitrepodir, exist_ok=True)
184 for f in ('index.jar', 'index.xml', 'index-v1.jar', 'index-v1.json'):
185 dlfile = os.path.join(tempdir, f)
186 dlurl = options.url + '/' + repodir + '/' + f
187 http_headers_file = os.path.join(gitrepodir, f + '.HTTP-headers.json')
190 'User-Agent': 'F-Droid 0.102.3'
193 if os.path.exists(http_headers_file):
194 with open(http_headers_file) as fp:
195 etag = json.load(fp)['ETag']
197 r = session.head(dlurl, headers=headers, allow_redirects=False)
198 if r.status_code != 200:
199 logging.debug('HTTP Response (' + str(r.status_code) + '), did not download ' + dlurl)
201 if etag and etag == r.headers.get('ETag'):
202 logging.debug('ETag matches, did not download ' + dlurl)
205 r = session.get(dlurl, headers=headers, allow_redirects=False)
206 if r.status_code == 200:
207 with open(dlfile, 'wb') as f:
212 for k, v in r.headers.items():
214 with open(http_headers_file, 'w') as fp:
215 json.dump(dump, fp, indent=2, sort_keys=True)
219 os.chdir(tempdirbase)
220 make_binary_transparency_log(repodirs, options.git_repo, options.url, 'fdroid btlog')
221 if options.git_remote:
222 server.push_binary_transparency(options.git_repo, options.git_remote)
223 shutil.rmtree(tempdirbase, ignore_errors=True)
226 if __name__ == "__main__":