# wiki_user = "login"
# wiki_password = "1234"
+# Keep a log of all generated index files in a git repo to provide a
+# "binary transparency" log for anyone to check the history of the
+# binaries that are published. This is in the form of a "git remote",
+# which this machine where `fdroid update` is run has already been
+# configured to allow push access (e.g. ssh key, username/password, etc)
+# binary_transparency_remote = "git@gitlab.com:fdroid/binary-transparency-log.git"
+
# Only set this to true when running a repository where you want to generate
# stats, and only then on the master build servers, not a development
# machine.
re.S | re.M).group(1)
src = os.path.join(bindir, src)
elif omethod == 'raw':
- globpath = os.path.join(root_dir, build.output)
+ output_path = common.replace_build_vars(build.output, build)
+ globpath = os.path.join(root_dir, output_path)
apks = glob.glob(globpath)
if len(apks) > 1:
raise BuildException('Multiple apks match %s' % globpath, '\n'.join(apks))
env[n] = build.ndk_path()
+def replace_build_vars(cmd, build):
+ cmd = cmd.replace('$$COMMIT$$', build.commit)
+ cmd = cmd.replace('$$VERSION$$', build.versionName)
+ cmd = cmd.replace('$$VERCODE$$', build.versionCode)
+ return cmd
+
+
def replace_config_vars(cmd, build):
cmd = cmd.replace('$$SDK$$', config['sdk_path'])
cmd = cmd.replace('$$NDK$$', build.ndk_path())
cmd = cmd.replace('$$MVN3$$', config['mvn3'])
cmd = cmd.replace('$$QT$$', config['qt_sdk_path'] or '')
if build is not None:
- cmd = cmd.replace('$$COMMIT$$', build.commit)
- cmd = cmd.replace('$$VERSION$$', build.versionName)
- cmd = cmd.replace('$$VERCODE$$', build.versionCode)
+ cmd = replace_build_vars(cmd, build)
return cmd
trying to do the comparison.
"""
+ absapk1 = os.path.abspath(apk1)
+ absapk2 = os.path.abspath(apk2)
+
+ # try to find diffoscope in the path, if it hasn't been manually configed
+ if 'diffoscope' not in config:
+ tmp = find_command('diffoscope')
+ if tmp is not None:
+ config['diffoscope'] = tmp
+ if 'diffoscope' in config:
+ htmlfile = absapk1 + '.diffoscope.html'
+ textfile = absapk1 + '.diffoscope.txt'
+ if subprocess.call([config['diffoscope'],
+ '--max-report-size', '12345678', '--max-diff-block-lines', '100',
+ '--html', htmlfile, '--text', textfile,
+ absapk1, absapk2]) != 0:
+ return("Failed to unpack " + apk1)
+
apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
for d in [apk1dir, apk2dir]:
logging.info(response['verbose_msg'] + " " + response['permalink'])
+def push_binary_transparency(binary_transparency_remote):
+ '''push the binary transparency git repo to the specifed remote'''
+ import git
+
+ repo = git.Repo('binary_transparency_log')
+ pushremote = None
+ for remote in repo.remotes:
+ if remote.url == binary_transparency_remote:
+ pushremote = remote
+ break
+
+ if not pushremote:
+ pushremote = repo.create_remote('fdroid_server_update', binary_transparency_remote)
+ pushremote.push('master')
+
+
def main():
global config, options
upload_to_android_observatory(repo_section)
if config.get('virustotal_apikey'):
upload_to_virustotal(repo_section, config.get('virustotal_apikey'))
+
+ binary_transparency_remote = config.get('binary_transparency_remote')
+ if binary_transparency_remote:
+ push_binary_transparency(binary_transparency_remote)
+
sys.exit(0)
import os
import shutil
import glob
+import json
import re
import socket
import zipfile
import hashlib
import pickle
+import platform
import urllib.parse
from datetime import datetime, timedelta
from xml.dom.minidom import Document
shutil.copy(apkascpath, apk['per_app_repo'])
+def make_binary_transparency_log(repodirs):
+ '''Log the indexes in a standalone git repo to serve as a "binary
+ transparency" log.
+
+ see: https://www.eff.org/deeplinks/2014/02/open-letter-to-tech-companies
+
+ '''
+
+ import git
+ btrepo = 'binary_transparency'
+ if os.path.exists(os.path.join(btrepo, '.git')):
+ gitrepo = git.Repo(btrepo)
+ else:
+ if not os.path.exists(btrepo):
+ os.mkdir(btrepo)
+ gitrepo = git.Repo.init(btrepo)
+
+ gitconfig = gitrepo.config_writer()
+ gitconfig.set_value('user', 'name', 'fdroid update')
+ gitconfig.set_value('user', 'email', 'fdroid@' + platform.node())
+
+ url = config['repo_url'].rstrip('/')
+ with open(os.path.join(btrepo, 'README.md'), 'w') as fp:
+ fp.write("""
+# Binary Transparency Log for %s
+
+""" % url[:url.rindex('/')]) # strip '/repo'
+ gitrepo.index.add(['README.md', ])
+ gitrepo.index.commit('add README')
+
+ for repodir in repodirs:
+ cpdir = os.path.join(btrepo, repodir)
+ if not os.path.exists(cpdir):
+ os.mkdir(cpdir)
+ for f in ('index.xml', ):
+ dest = os.path.join(cpdir, f)
+ shutil.copyfile(os.path.join(repodir, f), dest)
+ gitrepo.index.add([os.path.join(repodir, f), ])
+ for f in ('index.jar', ):
+ repof = os.path.join(repodir, f)
+ dest = os.path.join(cpdir, f)
+ jarin = zipfile.ZipFile(repof, 'r')
+ jarout = zipfile.ZipFile(dest, 'w')
+ for info in jarin.infolist():
+ if info.filename.startswith('META-INF/'):
+ jarout.writestr(info, jarin.read(info.filename))
+ jarout.close()
+ jarin.close()
+ gitrepo.index.add([repof, ])
+
+ files = []
+ for root, dirs, filenames in os.walk(repodir):
+ for f in filenames:
+ files.append(os.path.relpath(os.path.join(root, f), repodir))
+ output = collections.OrderedDict()
+ for f in sorted(files):
+ repofile = os.path.join(repodir, f)
+ stat = os.stat(repofile)
+ output[f] = (
+ stat.st_size,
+ stat.st_ctime_ns,
+ stat.st_mtime_ns,
+ stat.st_mode,
+ stat.st_uid,
+ stat.st_gid,
+ )
+ fslogfile = os.path.join(cpdir, 'filesystemlog.json')
+ with open(fslogfile, 'w') as fp:
+ json.dump(output, fp, indent=2)
+ gitrepo.index.add([os.path.join(repodir, 'filesystemlog.json'), ])
+
+ gitrepo.index.commit('fdroid update')
+
+
config = None
options = None
if len(repodirs) > 1:
make_index(apps, sortedids, archapks, repodirs[1], True)
+ if config.get('binary_transparency_remote'):
+ make_binary_transparency_log(repodirs)
+
if config['update_stats']:
# Update known apks info...
Build:0.2.1,2000
commit=0.2.1
- output=app/build/distributions/FDroidPrivilegedExtensionFromBinaries-0.2.1.zip
+ output=app/build/distributions/FDroidPrivilegedExtensionFromBinaries-$$VERSION$$.zip
build=gradle assembleUpdateZipFromBinariesDebug
Auto Update Mode:Version %v
set -e
+#------------------------------------------------------------------------------#
+echo_header "copy tests/repo, update with binary transparency log"
+
+REPOROOT=`create_test_dir`
+GNUPGHOME=$REPOROOT/gnupghome
+KEYSTORE=$WORKSPACE/tests/keystore.jks
+cd $REPOROOT
+$fdroid init --keystore $KEYSTORE --repo-keyalias=sova
+cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $WORKSPACE/tests/stats $REPOROOT/
+echo 'keystorepass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
+echo 'keypass = "r9aquRHYoI8+dYz6jKrLntQ5/NJNASFBacJh7Jv2BlI="' >> config.py
+echo 'binary_transparency_remote = "git@gitlab.com:fdroid-continuous-integration/binary-transparency.git"' >> config.py
+echo "accepted_formats = ['json', 'txt', 'yml']" >> config.py
+$fdroid update --verbose --pretty
+test -e repo/index.xml
+test -e repo/index.jar
+grep -F '<application id=' repo/index.xml > /dev/null
+cd binary_transparency
+[ `git rev-list --count HEAD` == "2" ]
+
+
#------------------------------------------------------------------------------#
echo_header "setup a new repo with keystore with APK, update, then without key"
~ianmdlvl / fdroidserver.git / commitdiff