From 03ec590989aa7d54b1029d2ec0392cf681a75aaa Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Fri, 16 Dec 2016 12:19:07 +0100 Subject: [PATCH] new repo-wide config option for a Binary Transparency Log A Binary Transparency Log is a append only log of all binaries published by a repo. This is useful for people to find whether the binary they have matches what F-Droid has published, and also makes it more difficult for the published history to be changed without notice, or for a server to give specific users custom malware binaries. https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies --- examples/config.py | 7 ++++ fdroidserver/server.py | 21 ++++++++++++ fdroidserver/update.py | 73 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+) diff --git a/examples/config.py b/examples/config.py index eac1b995..20405b07 100644 --- a/examples/config.py +++ b/examples/config.py @@ -238,6 +238,13 @@ The repository of older versions of applications from the main demo repository. # wiki_user = "login" # wiki_password = "1234" +# Keep a log of all generated index files in a git repo to provide a +# "binary transparency" log for anyone to check the history of the +# binaries that are published. This is in the form of a "git remote", +# which this machine where `fdroid update` is run has already been +# configured to allow push access (e.g. ssh key, username/password, etc) +# binary_transparency_remote = "git@gitlab.com:fdroid/binary-transparency-log.git" + # Only set this to true when running a repository where you want to generate # stats, and only then on the master build servers, not a development # machine. diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 2213d788..998b80cf 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -279,6 +279,22 @@ def upload_to_virustotal(repo_section, vt_apikey): logging.info(response['verbose_msg'] + " " + response['permalink']) +def push_binary_transparency(binary_transparency_remote): + '''push the binary transparency git repo to the specifed remote''' + import git + + repo = git.Repo('binary_transparency_log') + pushremote = None + for remote in repo.remotes: + if remote.url == binary_transparency_remote: + pushremote = remote + break + + if not pushremote: + pushremote = repo.create_remote('fdroid_server_update', binary_transparency_remote) + pushremote.push('master') + + def main(): global config, options @@ -414,6 +430,11 @@ def main(): upload_to_android_observatory(repo_section) if config.get('virustotal_apikey'): upload_to_virustotal(repo_section, config.get('virustotal_apikey')) + + binary_transparency_remote = config.get('binary_transparency_remote') + if binary_transparency_remote: + push_binary_transparency(binary_transparency_remote) + sys.exit(0) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 5189633c..880b4338 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -23,6 +23,7 @@ import sys import os import shutil import glob +import json import re import socket import zipfile @@ -1439,6 +1440,75 @@ def add_apks_to_per_app_repos(repodir, apks): shutil.copy(apkascpath, apk['per_app_repo']) +def make_binary_transparency_log(repodirs): + '''Log the indexes in a standalone git repo to serve as a "binary + transparency" log. + + see: https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies + + ''' + + import git + btrepo = 'binary_transparency' + if os.path.exists(os.path.join(btrepo, '.git')): + gitrepo = git.Repo(btrepo) + else: + if not os.path.exists(btrepo): + os.mkdir(btrepo) + gitrepo = git.Repo.init(btrepo) + url = config['repo_url'].rstrip('/') + with open(os.path.join(btrepo, 'README.md'), 'w') as fp: + fp.write(""" +# Binary Transparency Log for %s + +""" % url[:url.rindex('/')]) # strip '/repo' + gitrepo.index.add(['README.md', ]) + gitrepo.index.commit('add README') + + for repodir in repodirs: + cpdir = os.path.join(btrepo, repodir) + if not os.path.exists(cpdir): + os.mkdir(cpdir) + for f in ('index.xml', ): + dest = os.path.join(cpdir, f) + shutil.copyfile(os.path.join(repodir, f), dest) + gitrepo.index.add([os.path.join(repodir, f), ]) + for f in ('index.jar', ): + repof = os.path.join(repodir, f) + dest = os.path.join(cpdir, f) + jarin = zipfile.ZipFile(repof, 'r') + jarout = zipfile.ZipFile(dest, 'w') + for info in jarin.infolist(): + if info.filename.startswith('META-INF/'): + jarout.writestr(info, jarin.read(info.filename)) + jarout.close() + jarin.close() + gitrepo.index.add([repof, ]) + + files = [] + for root, dirs, filenames in os.walk(repodir): + for f in filenames: + files.append(os.path.relpath(os.path.join(root, f), repodir)) + output = collections.OrderedDict() + for f in sorted(files): + repofile = os.path.join(repodir, f) + stat = os.stat(repofile) + output[f] = ( + stat.st_size, + stat.st_ctime_ns, + stat.st_mtime_ns, + stat.st_mode, + stat.st_uid, + stat.st_gid, + ) + fslogfile = os.path.join(cpdir, 'filesystemlog.json') + with open(fslogfile, 'w') as fp: + json.dump(output, fp, indent=2) + gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ]) + + gitrepo.index.commit('fdroid update') + + config = None options = None @@ -1678,6 +1748,9 @@ def main(): if len(repodirs) > 1: make_index(apps, sortedids, archapks, repodirs[1], True) + if config.get('binary_transparency_remote'): + make_binary_transparency_log(repodirs) + if config['update_stats']: # Update known apks info... -- 2.30.2