`fdroid btlog` external binary transparency logger

This complements the binary transparency logging that happens on the
server side !226.  Anyone can set up an efficient tracker of any
F-Droid repo which stores all index files that it sees.  It uses HEAD
requests and ETag checking to be as efficient as possible, so that
this can be automatically run at a frequent pace.

diff --git a/completion/bash-completion b/completion/bash-completion
index bdaa2cbb97dd261950b73e129ff2050aff254cf0..48352447a1d9faec79486a9e9c6dc889d8ade20f 100644 (file)
--- a/completion/bash-completion
+++ b/completion/bash-completion
@@ -236,6 +236,12 @@ __complete_verify() {
        esac
 }
 
+__complete_btlog() {
+       opts="-u"
+       lopts="--git-remote --git-repo --url"
+       __complete_options
+}
+
 __complete_stats() {
        opts="-v -q -d"
        lopts="--verbose --quiet --download"

diff --git a/fdroid b/fdroid
index feea104afa98a6142738ec1053d0e9fc29347b46..bc1655b9786feacc99c3bcdfaf22b6f8693eb180 100755 (executable)
--- a/fdroid
+++ b/fdroid
@@ -42,6 +42,7 @@ commands = {
     "stats": "Update the stats of the repo",
     "server": "Interact with the repo HTTP server",
     "signindex": "Sign indexes created using update --nosign",
+    "btlog": "Update the binary transparency log for a URL",
 }
 
 

diff --git a/fdroidserver/btlog.py b/fdroidserver/btlog.py
new file mode 100755 (executable)
index 0000000..1911a09
--- /dev/null
+++ b/fdroidserver/btlog.py
@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+#
+# btlog.py - part of the FDroid server tools
+# Copyright (C) 2017, Hans-Christoph Steiner <hans@eds.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# This is for creating a binary transparency log in a git repo for any
+# F-Droid repo accessible via HTTP.  It is meant to run very often,
+# even once a minute in a cronjob, so it uses HEAD requests and the
+# HTTP ETag to check if the file has changed.  HEAD requests should
+# not count against the download counts.  This pattern of a HEAD then
+# a GET is what fdroidclient uses to avoid ETags being abused as
+# cookies. This also uses the same HTTP User Agent as the F-Droid
+# client app so its not easy for the server to distinguish this from
+# the F-Droid client.
+
+import os
+import json
+import logging
+import requests
+import shutil
+import sys
+import tempfile
+from argparse import ArgumentParser
+
+from . import common
+
+
+options = None
+
+
+def main():
+    global options
+
+    parser = ArgumentParser(usage="%(prog)s [options]")
+    common.setup_global_opts(parser)
+    parser.add_argument("--git-repo",
+                        default=os.path.join(os.getcwd(), 'binary_transparency'),
+                        help="Path to the git repo to use as the log")
+    parser.add_argument("-u", "--url", default='https://f-droid.org',
+                        help="The base URL for the repo to log (default: https://f-droid.org)")
+    parser.add_argument("--git-remote", default=None,
+                        help="Create a repo signing key in a keystore")
+    options = parser.parse_args()
+
+    if options.verbose:
+        logging.getLogger("requests").setLevel(logging.INFO)
+        logging.getLogger("urllib3").setLevel(logging.INFO)
+    else:
+        logging.getLogger("requests").setLevel(logging.WARNING)
+        logging.getLogger("urllib3").setLevel(logging.WARNING)
+
+    if not os.path.exists(options.git_repo):
+        logging.error('"' + options.git_repo + '/" does not exist! Create it, or use --git-repo')
+        sys.exit(1)
+
+    session = requests.Session()
+
+    new_files = False
+    repodirs = ('repo', 'archive')
+    tempdirbase = tempfile.mkdtemp(prefix='.fdroid-btlog-')
+    for repodir in repodirs:
+        # TODO read HTTP headers for etag from git repo
+        tempdir = os.path.join(tempdirbase, repodir)
+        os.makedirs(tempdir, exist_ok=True)
+        gitrepodir = os.path.join(options.git_repo, repodir)
+        os.makedirs(gitrepodir, exist_ok=True)
+        for f in ('index.jar', 'index.xml', 'index-v1.jar', 'index-v1.json'):
+            dlfile = os.path.join(tempdir, f)
+            dlurl = options.url + '/' + repodir + '/' + f
+            http_headers_file = os.path.join(gitrepodir, f + '.HTTP-headers.json')
+
+            headers = {
+                'User-Agent': 'F-Droid 0.102.3'
+            }
+            if os.path.exists(http_headers_file):
+                with open(http_headers_file) as fp:
+                    etag = json.load(fp)['ETag']
+
+            r = session.head(dlurl, headers=headers, allow_redirects=False)
+            if r.status_code != 200:
+                logging.debug('HTTP Response (' + str(r.status_code) + '), did not download ' + dlurl)
+                continue
+            if etag and etag == r.headers.get('ETag'):
+                logging.debug('ETag matches, did not download ' + dlurl)
+                continue
+
+            r = session.get(dlurl, headers=headers, allow_redirects=False)
+            if r.status_code == 200:
+                with open(dlfile, 'wb') as f:
+                    for chunk in r:
+                        f.write(chunk)
+
+                dump = dict()
+                for k, v in r.headers.items():
+                    dump[k] = v
+                with open(http_headers_file, 'w') as fp:
+                    json.dump(dump, fp, indent=2, sort_keys=True)
+                new_files = True
+
+    if new_files:
+        os.chdir(tempdirbase)
+        common.make_binary_transparency_log(repodirs, options.git_repo, options.url,
+                                            'fdroid btlog')
+    shutil.rmtree(tempdirbase, ignore_errors=True)
+
+if __name__ == "__main__":
+    main()

diff --git a/fdroidserver/common.py b/fdroidserver/common.py
index 00f22c7acf77c29db10a0ad6bbcdf973bf69404a..be902c47870fd584c7c03272cb780f3c83a5933e 100644 (file)
--- a/fdroidserver/common.py
+++ b/fdroidserver/common.py
@@ -2353,7 +2353,9 @@ def is_repo_file(filename):
         ]
 
 
-def make_binary_transparency_log(repodirs):
+def make_binary_transparency_log(repodirs, btrepo='binary_transparency',
+                                 url=None,
+                                 commit_title='fdroid update'):
     '''Log the indexes in a standalone git repo to serve as a "binary
     transparency" log.
 
@@ -2362,7 +2364,6 @@ def make_binary_transparency_log(repodirs):
     '''
 
     import git
-    btrepo = 'binary_transparency'
     if os.path.exists(os.path.join(btrepo, '.git')):
         gitrepo = git.Repo(btrepo)
     else:
@@ -2371,10 +2372,11 @@ def make_binary_transparency_log(repodirs):
         gitrepo = git.Repo.init(btrepo)
 
         gitconfig = gitrepo.config_writer()
-        gitconfig.set_value('user', 'name', 'fdroid update')
+        gitconfig.set_value('user', 'name', commit_title)
         gitconfig.set_value('user', 'email', 'fdroid@' + platform.node())
 
-        url = config['repo_url'].rstrip('/')
+        if not url:
+            url = config['repo_url'].rstrip('/')
         with open(os.path.join(btrepo, 'README.md'), 'w') as fp:
             fp.write("""
 # Binary Transparency Log for %s
@@ -2388,11 +2390,16 @@ def make_binary_transparency_log(repodirs):
         if not os.path.exists(cpdir):
             os.mkdir(cpdir)
         for f in ('index.xml', 'index-v1.json'):
+            repof = os.path.join(repodir, f)
+            if not os.path.exists(repof):
+                continue
             dest = os.path.join(cpdir, f)
-            shutil.copyfile(os.path.join(repodir, f), dest)
-            gitrepo.index.add([os.path.join(repodir, f), ])
+            shutil.copyfile(repof, dest)
+            gitrepo.index.add([repof, ])
         for f in ('index.jar', 'index-v1.jar'):
             repof = os.path.join(repodir, f)
+            if not os.path.exists(repof):
+                continue
             dest = os.path.join(cpdir, f)
             jarin = ZipFile(repof, 'r')
             jarout = ZipFile(dest, 'w')
@@ -2424,4 +2431,7 @@ def make_binary_transparency_log(repodirs):
             json.dump(output, fp, indent=2)
         gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ])
 
-    gitrepo.index.commit('fdroid update')
+        for f in glob.glob(os.path.join(cpdir, '*.HTTP-headers.json')):
+            gitrepo.index.add([os.path.join(repodir, os.path.basename(f)), ])
+
+    gitrepo.index.commit(commit_title)