# files generated by tests
tmp/
tests/repo/icons*
+
+# files used in manual testing
+/config.py
+/tmp/
+/logs/
+/metadata/
+makebuildserver.config.py
+/tests/config.py
+/tests/fdroid-icon.png
+/unsigned/
# this doesn't actually work yet https://github.com/travis-ci/travis-ci/issues/5337
dist: trusty
- os: osx
+ osx_image: xcode8
env: ANDROID_HOME=/usr/local/opt/android-sdk
licenses:
sudo pip3 install pep8 pyflakes pylint;
sudo pip3 install -e .;
sudo rm -rf fdroidserver.egg-info;
- echo y | android --verbose update sdk --no-ui --all --filter platform-tools,build-tools-23.0.3;
+ echo y | android --verbose update sdk --no-ui --all --filter platform-tools,build-tools-24.0.0;
elif [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
sudo add-apt-repository ppa:guardianproject/fdroidserver -y;
sudo apt-get -q update -y;
.vagrant
up.log
cache/
-Vagrantfile
+Vagrantfile.yaml
--- /dev/null
+
+require 'yaml'
+configfile = YAML.load_file('Vagrantfile.yaml')
+
+Vagrant.configure("2") do |config|
+
+ if Vagrant.has_plugin?("vagrant-cachier")
+ config.cache.scope = :box
+ config.cache.auto_detect = false
+ config.cache.enable :apt
+ config.cache.enable :chef
+ end
+
+ config.vm.box = configfile['basebox']
+ config.vm.box_url = configfile['baseboxurl']
+
+ config.vm.provider "virtualbox" do |v|
+ v.customize ["modifyvm", :id, "--memory", configfile['memory']]
+ v.customize ["modifyvm", :id, "--cpus", configfile['cpus']]
+ v.customize ["modifyvm", :id, "--hwvirtex", configfile['hwvirtex']]
+ end
+
+ config.vm.boot_timeout = configfile['boot_timeout']
+
+ config.vm.provision :shell, :path => "fixpaths.sh"
+
+ if configfile.has_key? "aptproxy"
+ config.vm.provision :shell, path: "provision-apt-proxy",
+ args: [configfile["aptproxy"]]
+ end
+
+ # buildserver/ is shared to the VM's /vagrant by default so the old
+ # default does not need a custom mount
+ if configfile["cachedir"] != "buildserver/cache"
+ config.vm.synced_folder configfile["cachedir"], '/vagrant/cache',
+ owner: 'root', group: 'root', create: true
+ end
+
+ # cache .deb packages on the host via a mount trick
+ if configfile.has_key? "aptcachedir"
+ config.vm.synced_folder configfile["aptcachedir"], "/var/cache/apt/archives",
+ owner: 'root', group: 'root', create: true
+ end
+
+ config.vm.provision "shell", path: "setup-env-vars",
+ args: ["/home/vagrant/android-sdk"]
+ config.vm.provision "shell", path: "provision-apt-get-install",
+ args: [configfile['debian_mirror']]
+
+ config.vm.provision :chef_solo do |chef|
+ chef.cookbooks_path = "cookbooks"
+ chef.log_level = :debug
+ chef.add_recipe "kivy"
+ end
+
+ config.vm.provision "shell", path: "provision-android-sdk"
+ config.vm.provision "shell", path: "provision-android-ndk",
+ args: ["/home/vagrant/android-ndk"]
+ config.vm.provision "shell", path: "provision-pip",
+ args: ["compare-locales"]
+ config.vm.provision "shell", path: "provision-gradle"
+ config.vm.provision "file", source: "gradle",
+ destination: "/opt/gradle/bin/gradle"
+
+ # let Ubuntu/trusty's paramiko work with the VM instance
+ if `uname -v`.include? "14.04"
+ config.vm.provision "shell", path: "provision-ubuntu-trusty-paramiko"
+ end
+
+end
+++ /dev/null
-
-user = node[:settings][:user]
-debian_mirror = node[:settings][:debian_mirror]
-
-execute 'set_debian_mirror' do
- command "sed -i 's,http://ftp.uk.debian.org/debian/,#{debian_mirror},g' /etc/apt/sources.list"
-end
-
-execute "jessie_backports" do
- command "echo 'deb #{debian_mirror} jessie-backports main' > /etc/apt/sources.list.d/backports.list"
- only_if "grep jessie /etc/apt/sources.list"
-end
-
-execute "apt-get-update" do
- command "apt-get update"
-end
-
-%w{
- ant
- ant-contrib
- autoconf
- autoconf2.13
- automake1.11
- autopoint
- bison
- bzr
- cmake
- curl
- expect
- faketime
- flex
- gettext
- git-core
- git-svn
- gperf
- graphviz
- imagemagick
- inkscape
- javacc
- libarchive-zip-perl
- libexpat1-dev
- libglib2.0-dev
- liblzma-dev
- librsvg2-bin
- libsaxonb-java
- libssl-dev
- libssl1.0.0
- libtool
- libtool-bin
- make
- maven
- mercurial
- nasm
- openjdk-8-jdk-headless
- optipng
- p7zip
- pandoc
- perlmagick
- pkg-config
- python-gnupg
- python-magic
- python-setuptools
- python3-gnupg
- python3-requests
- python3-yaml
- qt5-default
- qtbase5-dev
- quilt
- realpath
- scons
- subversion
- swig
- texinfo
- transfig
- unzip
- vorbis-tools
- xsltproc
- yasm
- zip
- }.each do |pkg|
- package pkg do
- action :install
- end
-end
-
-if node['kernel']['machine'] == "x86_64"
- %w{libstdc++6:i386 libgcc1:i386 zlib1g:i386 libncurses5:i386}.each do |pkg|
- package pkg do
- action :install
- end
- end
-end
-
-easy_install_package "compare-locales" do
- options "-U"
- action :install
-end
-
-execute "set-default-java" do
- command "update-java-alternatives --set java-1.8.0-openjdk-i386"
-end
-
-# Ubuntu trusty 14.04's paramiko does not work with jessie's openssh's default settings
-# https://stackoverflow.com/questions/7286929/paramiko-incompatible-ssh-peer-no-acceptable-kex-algorithm/32691055#32691055
-execute "support-ubuntu-trusty-paramiko" do
- only_if { node[:settings][:ubuntu_trusty] == 'true' }
- command "echo Ciphers aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes128-ctr >> /etc/ssh/sshd_config"
- command "echo MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-512,hmac-sha2-256,hmac-ripemd160,hmac-sha1 >> /etc/ssh/sshd_config"
- command "echo KexAlgorithms diffie-hellman-group-exchange-sha256,diffie-hellman-group14-sha1,diffie-hellman-group-exchange-sha1 >> /etc/ssh/sshd_config"
-end
-user = node[:settings][:user]
-
%w{cython python-pygame python-pip python-virtualenv python-opengl python-gst0.10 python-enchant libgl1-mesa-dev libgles2-mesa-dev}.each do |pkg|
package pkg do
action :install
#!/bin/sh
+echo $0
+
fixit()
{
#Fix sudoers so the PATH gets passed through, otherwise chef
d_plugin_v=(2.12 2.12 2.4 2.4 2.3 2.2.1 2.2.1 2.1 2.1 1.12 1.12 1.12 1.11 1.10 1.9 1.8 1.6 1.6 1.4 1.4)
# All gradle versions we know about
-plugin_v=(2.13 2.12 2.11 2.10 2.9 2.8 2.7 2.6 2.5 2.4 2.3 2.2.1 2.1 1.12 1.11 1.10 1.9 1.8 1.7 1.6 1.4)
+plugin_v=(2.14 2.13 2.12 2.11 2.10 2.9 2.8 2.7 2.6 2.5 2.4 2.3 2.2.1 2.2 2.1 1.12 1.11 1.10 1.9 1.8 1.7 1.6 1.4)
# Find the highest version available
for v in ${plugin_v[*]}; do
#!/bin/bash
#
+echo $0
set -e
NDK_BASE=$1
#!/bin/bash
#
+echo $0
set -e
+set -x
if [ -z $ANDROID_HOME ]; then
echo "ANDROID_HOME env var must be set!"
cd /vagrant/cache
-# make hard links for `android update sdk` to use and delete
+# make links for `android update sdk` to use and delete
for f in android_*.zip android-[0-9]*.zip platform-[0-9]*.zip build-tools_r*-linux.zip; do
rm -f ${ANDROID_HOME}/temp/$f
ln -s /vagrant/cache/$f ${ANDROID_HOME}/temp/
cached=,build-tools-${ver}${cached}
done
-${ANDROID_HOME}/tools/android --silent update sdk --no-ui --all \
- --filter platform-tools,extra-android-m2repository${cached} > /dev/null <<EOH
+${ANDROID_HOME}/tools/android update sdk --no-ui --all \
+ --filter platform-tools,extra-android-m2repository${cached} <<EOH
y
EOH
--- /dev/null
+#!/bin/bash
+
+echo $0
+set -e
+set -x
+
+debian_mirror=$1
+
+sed -i "s,http://ftp.uk.debian.org/debian/,${debian_mirror},g" /etc/apt/sources.list
+
+if grep --quiet jessie /etc/apt/sources.list; then
+ echo "deb $debian_mirror jessie-backports main" > /etc/apt/sources.list.d/backports.list
+fi
+
+dpkg --add-architecture i386
+
+apt-get -y update
+apt-get -y upgrade
+
+packages="
+ ant
+ ant-contrib
+ autoconf
+ autoconf2.13
+ automake1.11
+ autopoint
+ bison
+ bzr
+ cmake
+ curl
+ expect
+ faketime
+ flex
+ gettext
+ git-core
+ git-svn
+ gperf
+ graphviz
+ imagemagick
+ inkscape
+ javacc
+ libarchive-zip-perl
+ libexpat1-dev
+ libgcc1:i386
+ libglib2.0-dev
+ liblzma-dev
+ libncurses5:i386
+ librsvg2-bin
+ libsaxonb-java
+ libssl-dev
+ libssl1.0.0
+ libstdc++6:i386
+ libtool
+ libtool-bin
+ make
+ maven
+ mercurial
+ nasm
+ openjdk-8-jdk-headless
+ optipng
+ p7zip
+ pandoc
+ perlmagick
+ pkg-config
+ python-gnupg
+ python-magic
+ python-setuptools
+ python3-gnupg
+ python3-requests
+ python3-yaml
+ qt5-default
+ qtbase5-dev
+ quilt
+ realpath
+ scons
+ subversion
+ swig
+ texinfo
+ transfig
+ unzip
+ vorbis-tools
+ xsltproc
+ yasm
+ zip
+ zlib1g:i386
+"
+
+apt-get install --yes --no-install-recommends $packages
+
+highestjava=`update-java-alternatives --list | sort -n | tail -1 | cut -d ' ' -f 1`
+update-java-alternatives --set $highestjava
--- /dev/null
+#!/bin/bash
+
+echo $0
+set -e
+
+rm -f /etc/apt/apt.conf.d/02proxy
+echo "Acquire::ftp::Proxy \"$1\";" >> /etc/apt/apt.conf.d/02proxy
+echo "Acquire::http::Proxy \"$1\";" >> /etc/apt/apt.conf.d/02proxy
+echo "Acquire::https::Proxy \"$1\";" >> /etc/apt/apt.conf.d/02proxy
+
+apt-get update
--- /dev/null
+#!/bin/bash
+
+echo $0
+set -e
+set -x
+
+# cache pypi downloads
+if [ -z $PIP_DOWNLOAD_CACHE ]; then
+ export PIP_DOWNLOAD_CACHE=$HOME/.pip_download_cache
+fi
+
+apt-get install --yes --no-install-recommends python-pip
+
+pip install --upgrade $@
--- /dev/null
+#!/bin/bash
+
+echo $0
+set -e
+
+# Ubuntu trusty 14.04's paramiko does not work with jessie's openssh's default settings
+# https://stackoverflow.com/questions/7286929/paramiko-incompatible-ssh-peer-no-acceptable-kex-algorithm/32691055#32691055
+
+if ! grep --quiet ^Ciphers /etc/ssh/sshd_config; then
+ echo Ciphers aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes128-ctr >> /etc/ssh/sshd_config
+fi
+
+if ! grep --quiet ^MACs /etc/ssh/sshd_config; then
+ echo MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-512,hmac-sha2-256,hmac-ripemd160,hmac-sha1 >> /etc/ssh/sshd_config
+fi
+
+if ! grep --quiet ^KexAlgorithms /etc/ssh/sshd_config; then
+ echo KexAlgorithms diffie-hellman-group-exchange-sha256,diffie-hellman-group14-sha1,diffie-hellman-group-exchange-sha1 >> /etc/ssh/sshd_config
+fi
# }
# Build tools version to be used
-# build_tools = "23.0.3"
+# build_tools = "24.0.0"
# Force all build to use the above version of build -tools, good for testing
# builds without having all of the possible build-tools installed.
# Get an SFTP connection...
ftp = sshs.open_sftp()
- ftp.get_channel().settimeout(15)
+ ftp.get_channel().settimeout(60)
# Put all the necessary files in place...
ftp.chdir(homedir)
if options.verbose:
cmdline += ' --verbose'
cmdline += " %s:%s" % (app.id, build.vercode)
- cmdline = '. /etc/profile && ' + cmdline
- chan.exec_command('bash -c "' + cmdline + '"')
+ chan.exec_command('bash --login -c "' + cmdline + '"')
output = bytes()
while not chan.exit_status_ready():
while chan.recv_ready():
'r9b': None,
'r10e': "$ANDROID_NDK",
},
- 'build_tools': "23.0.3",
+ 'build_tools': "24.0.0",
'force_build_tools': False,
'java_paths': None,
'ant': "ant",
set up the environment variables for the build environment
There is only a weak standard, the variables used by gradle, so also set
- up the most commonly used environment variables for SDK and NDK
+ up the most commonly used environment variables for SDK and NDK. Also, if
+ there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
'''
global env, orig_path
for k, v in config['java_paths'].items():
env['JAVA%s_HOME' % k] = v
+ missinglocale = True
+ for k, v in env.items():
+ if k == 'LANG' and v != 'C':
+ missinglocale = False
+ elif k == 'LC_ALL':
+ missinglocale = False
+ if missinglocale:
+ env['LANG'] = 'en_US.UTF-8'
+
if build is not None:
path = build.ndk_path()
paths = orig_path.split(os.pathsep)
if not os.path.isfile(iconpath):
return
+ fp = None
try:
- im = Image.open(iconpath)
+ fp = open(iconpath, 'rb')
+ im = Image.open(fp)
size = dpi_to_px(density)
if any(length > size for length in im.size):
except Exception as e:
logging.error("Failed resizing {0} - {1}".format(iconpath, e))
+ finally:
+ if fp:
+ fp.close()
+
def resize_all_icons(repodirs):
"""Resize all icons that exceed the max size
return hashlib.md5(hexlify(cert_encoded)).hexdigest()
+def get_icon_bytes(apkzip, iconsrc):
+ '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
+ try:
+ return apkzip.read(iconsrc)
+ except KeyError:
+ return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
+
+
+def sha256sum(filename):
+ '''Calculate the sha256 of the given file'''
+ sha = hashlib.sha256()
+ with open(filename, 'rb') as f:
+ while True:
+ t = f.read(16384)
+ if len(t) == 0:
+ break
+ sha.update(t)
+ return sha.hexdigest()
+
+
+def insert_obbs(repodir, apps, apks):
+ """Scans the .obb files in a given repo directory and adds them to the
+ relevant APK instances. OBB files have versionCodes like APK
+ files, and they are loosely associated. If there is an OBB file
+ present, then any APK with the same or higher versionCode will use
+ that OBB file. There are two OBB types: main and patch, each APK
+ can only have only have one of each.
+
+ https://developer.android.com/google/play/expansion-files.html
+
+ :param repodir: repo directory to scan
+ :param apps: list of current, valid apps
+ :param apks: current information on all APKs
+
+ """
+
+ def obbWarnDelete(f, msg):
+ logging.warning(msg + f)
+ if options.delete_unknown:
+ logging.error("Deleting unknown file: " + f)
+ os.remove(f)
+
+ obbs = []
+ java_Integer_MIN_VALUE = -pow(2, 31)
+ for f in glob.glob(os.path.join(repodir, '*.obb')):
+ obbfile = os.path.basename(f)
+ # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
+ chunks = obbfile.split('.')
+ if chunks[0] != 'main' and chunks[0] != 'patch':
+ obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
+ continue
+ if not re.match(r'^-?[0-9]+$', chunks[1]):
+ obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
+ continue
+ versioncode = int(chunks[1])
+ packagename = ".".join(chunks[2:-1])
+
+ highestVersionCode = java_Integer_MIN_VALUE
+ if packagename not in apps.keys():
+ obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
+ continue
+ for apk in apks:
+ if packagename == apk['id'] and apk['versioncode'] > highestVersionCode:
+ highestVersionCode = apk['versioncode']
+ if versioncode > highestVersionCode:
+ obbWarnDelete(f, 'OBB file has newer versioncode(' + str(versioncode)
+ + ') than any APK: ')
+ continue
+ obbsha256 = sha256sum(f)
+ obbs.append((packagename, versioncode, obbfile, obbsha256))
+
+ for apk in apks:
+ for (packagename, versioncode, obbfile, obbsha256) in sorted(obbs, reverse=True):
+ if versioncode <= apk['versioncode'] and packagename == apk['id']:
+ if obbfile.startswith('main.') and 'obbMainFile' not in apk:
+ apk['obbMainFile'] = obbfile
+ apk['obbMainFileSha256'] = obbsha256
+ elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
+ apk['obbPatchFile'] = obbfile
+ apk['obbPatchFileSha256'] = obbsha256
+ if 'obbMainFile' in apk and 'obbPatchFile' in apk:
+ break
+
+
def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
"""Scan the apks in the given repo directory.
icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
sdkversion_pat = re.compile(".*'([0-9]*)'.*")
- string_pat = re.compile(".*'([^']*)'.*")
+ string_pat = re.compile(".* name='([^']*)'.*")
for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
apkfilename = apkfile[len(repodir) + 1:]
logging.critical("Spaces in filenames are not allowed.")
sys.exit(1)
- # Calculate the sha256...
- sha = hashlib.sha256()
- with open(apkfile, 'rb') as f:
- while True:
- t = f.read(16384)
- if len(t) == 0:
- break
- sha.update(t)
- shasum = sha.hexdigest()
+ shasum = sha256sum(apkfile)
usecache = False
if apkfilename in apkcache:
# Check for debuggable apks...
if common.isApkDebuggable(apkfile, config):
- logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
+ logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
# Get the signature (or md5 of, to be precise)...
logging.debug('Getting signature of {0}'.format(apkfile))
try:
with open(icondest, 'wb') as f:
- f.write(apkzip.read(iconsrc))
+ f.write(get_icon_bytes(apkzip, iconsrc))
apk['icons'][density] = iconfilename
except:
iconpath = os.path.join(
get_icon_dir(repodir, '0'), iconfilename)
with open(iconpath, 'wb') as f:
- f.write(apkzip.read(iconsrc))
+ f.write(get_icon_bytes(apkzip, iconsrc))
try:
im = Image.open(iconpath)
dpi = px_to_dpi(im.size[0])
get_icon_dir(repodir, last_density), iconfilename)
iconpath = os.path.join(
get_icon_dir(repodir, density), iconfilename)
+ fp = None
try:
- im = Image.open(last_iconpath)
- except:
- logging.warn("Invalid image file at %s" % last_iconpath)
- continue
+ fp = open(last_iconpath, 'rb')
+ im = Image.open(fp)
- size = dpi_to_px(density)
+ size = dpi_to_px(density)
- im.thumbnail((size, size), Image.ANTIALIAS)
- im.save(iconpath, "PNG")
- empty_densities.remove(density)
+ im.thumbnail((size, size), Image.ANTIALIAS)
+ im.save(iconpath, "PNG")
+ empty_densities.remove(density)
+ except:
+ logging.warning("Invalid image file at %s" % last_iconpath)
+ finally:
+ if fp:
+ fp.close()
# Then just copy from the highest resolution available
last_density = None
addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel)
if 'maxSdkVersion' in apk:
addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
+ addElementNonEmpty('obbMainFile', apk.get('obbMainFile'), doc, apkel)
+ addElementNonEmpty('obbMainFileSha256', apk.get('obbMainFileSha256'), doc, apkel)
+ addElementNonEmpty('obbPatchFile', apk.get('obbPatchFile'), doc, apkel)
+ addElementNonEmpty('obbPatchFileSha256', apk.get('obbPatchFileSha256'), doc, apkel)
if 'added' in apk:
addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
help="Create skeleton metadata files that are missing")
parser.add_argument("--delete-unknown", action="store_true", default=False,
- help="Delete APKs without metadata from the repo")
+ help="Delete APKs and/or OBBs without metadata from the repo")
parser.add_argument("-b", "--buildreport", action="store_true", default=False,
help="Report on build data status")
parser.add_argument("-i", "--interactive", default=False, action="store_true",
if newmetadata:
apps = metadata.read_metadata()
+ insert_obbs(repodirs[0], apps, apks)
+
# Scan the archive repo for apks as well
if len(repodirs) > 1:
archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
PY_TEST_FILES="tests/*.TestCase"
SH_FILES="hooks/pre-commit"
BASH_FILES="fd-commit jenkins-build docs/update.sh completion/bash-completion buildserver/provision-*"
- RB_FILES="buildserver/cookbooks/*/recipes/*.rb"
+ RB_FILES="buildserver/cookbooks/*/recipes/*.rb buildserver/Vagrantfile"
else
# if actually committing right now, then only run on the files
# that are going to be committed at this moment
# make sure we have the vagrant box image cached
test -e ~/.cache/fdroidserver || mkdir -p ~/.cache/fdroidserver
cd ~/.cache/fdroidserver
-wget --tries=1 --timeout=5 --continue https://f-droid.org/jessie32.box || true
-echo "ff6b0c0bebcb742783becbc51a9dfff5a2a0a839bfcbfd0288dcd3113f33e533 jessie32.box" > jessie32.box.sha256
-sha256sum -c jessie32.box.sha256
+wget --tries=1 --timeout=5 --continue https://f-droid.org/jessie64.box || true
+echo "de3e3c4a9c13e8c015e30edeea0f583b195d1ee8ff9ad4814e933bbfb560200f jessie64.box" > jessie64.box.sha256
+sha256sum -c jessie64.box.sha256
# redirect homes to be in the git repo, so they'll get cleaned and reset
export XDG_CONFIG_HOME=$WORKSPACE
cd fdroiddata
echo "build_server_always = True" > config.py
+# if it can't build fdroid, then its really broken
+../fdroid build --verbose --stop --latest org.fdroid.fdroid
# Gradle, JNI, preassemble
../fdroid build --stop org.adaway:55
# Maven
-../fdroid build --stop org.quantumbadger.redreader:55
+#../fdroid build --stop org.quantumbadger.redreader:55
# Custom build (make)
-../fdroid build --stop com.amaze.filemanager:29
+#../fdroid build --stop com.amaze.filemanager:29
# Uses verification
-../fdroid build --stop info.guardianproject.checkey:101
+#../fdroid build --stop info.guardianproject.checkey:101
# Gradle with retrolambda (JDK7 and JDK8)
-../fdroid build --stop com.moez.QKSMS:124
+#../fdroid build --stop com.moez.QKSMS:124
import subprocess
import time
import hashlib
+import yaml
from clint.textui import progress
from optparse import OptionParser
# set up default config
cachedir = os.path.join(os.getenv('HOME'), '.cache', 'fdroidserver')
config = {
- 'arch64': False,
- 'basebox': 'jessie32',
+ 'basebox': 'jessie64',
# TODO in py3, convert this to pathlib.Path(absolute_path_string).as_uri()
'baseboxurl': [
- 'file://' + os.path.join(cachedir, 'jessie32.box'),
- 'https://f-droid.org/jessie32.box',
+ 'file://' + os.path.join(cachedir, 'jessie64.box'),
+ 'https://f-droid.org/jessie64.box',
],
'debian_mirror': 'http://http.debian.net/debian/',
'apt_package_cache': False,
'cachedir': cachedir,
'cpus': 1,
'memory': 1024,
+ 'hwvirtex': 'off',
}
# load config file, if present
elif os.path.exists('makebs.config.py'):
# this is the old name for the config file
exec(compile(open('makebs.config.py').read(), 'makebs.config.py', 'exec'), config)
+del(config['__builtins__']) # added by compile/exec
if not os.path.exists('makebuildserver') or not os.path.exists(serverdir):
print('This must be run from the correct directory!')
if not os.path.exists(cachedir):
os.makedirs(cachedir, 0o755)
+if config['apt_package_cache']:
+ config['aptcachedir'] = cachedir + '/apt/archives'
+
cachefiles = [
('https://dl.google.com/android/repository/tools_r25.1.7-linux.zip',
'3ca053600a86a5a64d5571edfbb1dad27f2bda3bfd2d38e2fe54322610b1ef0b'),
- ('https://dl.google.com/android/repository/android_m2repository_r32.zip',
- 'a6a8d7ffb153161f26d5fdebfa9aa1c9c84b29c62851fffff2cdfad9e094b13b'),
+ ('https://dl.google.com/android/repository/platform-tools_r24-linux.zip',
+ '076368b337d042d163364594dda63b7e778835f636fafb2c8af4d4a604175c32'),
+ ('https://dl.google.com/android/repository/android_m2repository_r33.zip',
+ 'be9bb4a27aeefb1c9adb0cade8771f764447c4cbde74426303db2ac6bde1879c'),
('https://dl.google.com/android/repository/android-1.5_r04-linux.zip',
'85b6c8f9797e56aa415d3a282428bb640c96b0acb17c11d41621bb2a5302fe64'),
('https://dl.google.com/android/repository/android-1.6_r03-linux.zip',
'45eb581bbe53c9256f34c26b2cea919543c0079140897ac721cf88c0b9f6789e'),
('https://dl.google.com/android/repository/platform-23_r03.zip',
'4b4bcddead3319708275c54c76294707bfaa953d767e34f1a5b599f3edd0076c'),
+ ('https://dl.google.com/android/repository/platform-24_r01.zip',
+ '292e556e9ec84b4134e4126c835e4ac467f6950924fec3684792af0fcf426091'),
('https://dl.google.com/android/repository/build-tools_r17-linux.zip',
'4c8444972343a19045236f6924bd7f12046287c70dace96ab88b2159c8ec0e74'),
('https://dl.google.com/android/repository/build-tools_r18.0.1-linux.zip',
'82754f551a6e36eaf516fbdd00c95ff0ccd19f81d1e134125b6ac4916f7ed9b6'),
('https://dl.google.com/android/repository/build-tools_r23.0.3-linux.zip',
'd961663d4a9e128841751c0156548a347c882c081c83942e53788d8949bf34e1'),
+ ('https://dl.google.com/android/repository/build-tools_r24-linux.zip',
+ 'b4871f357224c5f660fd2bbee04d8c7d1c187eeddfd9702cc84503529e3b3724'),
# the binaries that Google uses are here:
# https://android.googlesource.com/platform/tools/external/gradle/+/studio-1.5/
('https://services.gradle.org/distributions/gradle-1.4-bin.zip',
'8734b13a401f4311ee418173ed6ca8662d2b0a535be8ff2a43ecb1c13cd406ea'),
('https://services.gradle.org/distributions/gradle-2.1-bin.zip',
'3eee4f9ea2ab0221b89f8e4747a96d4554d00ae46d8d633f11cfda60988bf878'),
+ ('https://services.gradle.org/distributions/gradle-2.2-bin.zip',
+ '91e5655fe11ef414449f218c4fa2985b3a49b7903c57556da109c84fa26e1dfb'),
('https://services.gradle.org/distributions/gradle-2.2.1-bin.zip',
'420aa50738299327b611c10b8304b749e8d3a579407ee9e755b15921d95ff418'),
('https://services.gradle.org/distributions/gradle-2.3-bin.zip',
'e77064981906cd0476ff1e0de3e6fef747bd18e140960f1915cca8ff6c33ab5c'),
('https://services.gradle.org/distributions/gradle-2.13-bin.zip',
'0f665ec6a5a67865faf7ba0d825afb19c26705ea0597cec80dd191b0f2cbb664'),
+ ('https://services.gradle.org/distributions/gradle-2.14-bin.zip',
+ '993b4f33b652c689e9721917d8e021cab6bbd3eae81b39ab2fd46fdb19a928d5'),
('https://pypi.python.org/packages/source/K/Kivy/Kivy-1.7.2.tar.gz',
'0485e2ef97b5086df886eb01f8303cb542183d2d71a159466f99ad6c8a1d03f1'),
+ ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
+ '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
+ ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
+ '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
+ ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
+ 'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1'),
]
-if config['arch64']:
- cachefiles.extend([
- ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86_64.bin',
- '102d6723f67ff1384330d12c45854315d6452d6510286f4e5891e00a5a8f1d5a'),
- ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64.tar.bz2',
- '8956e9efeea95f49425ded8bb697013b66e162b064b0f66b5c75628f76e0f532'),
- ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86_64-legacy-toolchains.tar.bz2',
- 'de93a394f7c8f3436db44568648f87738a8d09801a52f459dcad3fc047e045a1')])
-else:
- cachefiles.extend([
- ('https://dl.google.com/android/ndk/android-ndk-r10e-linux-x86.bin',
- '92b07d25aaad9b341a7f2b2a62402d508e948bf2dea3ee7b65a6aeb18bca7df5'),
- ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86.tar.bz2',
- '748104b829dd12afb2fdb3044634963abb24cdb0aad3b26030abe2e9e65bfc81'),
- ('https://dl.google.com/android/ndk/android-ndk-r9b-linux-x86-legacy-toolchains.tar.bz2',
- '606aadf815ae28cc7b0154996247c70d609f111b14e44bcbcd6cad4c87fefb6f')])
-
def sha256_for_file(path):
with open(path, 'rb') as f:
print("\t...shasum verified for " + local_filename)
else:
print("Invalid shasum of '" + v + "' detected for " + local_filename)
+ os.remove(local_filename)
sys.exit(1)
-# allow specifying a list/tuple that includes cached local copy
-if type(config['baseboxurl']) in (list, tuple) or config['baseboxurl'][0] in ('(', '['):
- baseboxurl = config['baseboxurl']
-else:
- baseboxurl = '"{0}"'.format(config['baseboxurl'])
-
# use VirtualBox software virtualization if hardware is not available,
# like if this is being run in kvm or some other VM platform, like
# http://jenkins.debian.net, the values are 'on' or 'off'
-hwvirtex = 'off'
if sys.platform.startswith('darwin'):
# all < 10 year old Macs work, and OSX servers as VM host are very
# rare, but this could also be auto-detected if someone codes it
- hwvirtex = 'on'
+ config['hwvirtex'] = 'on'
elif os.path.exists('/proc/cpuinfo'):
with open('/proc/cpuinfo') as f:
contents = f.read()
if 'vmx' in contents or 'svm' in contents:
- hwvirtex = 'on'
-
-# Generate an appropriate Vagrantfile for the buildserver, based on our
-# settings...
-vagrantfile = """
-Vagrant.configure("2") do |config|
-
- if Vagrant.has_plugin?("vagrant-cachier")
- config.cache.scope = :box
- config.cache.auto_detect = false
- config.cache.enable :apt
- config.cache.enable :chef
- end
-
- config.vm.box = "{0}"
- config.vm.box_url = {1}
-
- config.vm.provider "virtualbox" do |v|
- v.customize ["modifyvm", :id, "--memory", "{2}"]
- v.customize ["modifyvm", :id, "--cpus", "{3}"]
- v.customize ["modifyvm", :id, "--hwvirtex", "{4}"]
- end
+ config['hwvirtex'] = 'on'
- config.vm.boot_timeout = {5}
-
- config.vm.provision :shell, :path => "fixpaths.sh"
-""".format(config['basebox'],
- baseboxurl,
- config['memory'],
- config.get('cpus', 1),
- hwvirtex,
- config['boot_timeout'])
-if 'aptproxy' in config and config['aptproxy']:
- vagrantfile += """
- config.vm.provision :shell, :inline => 'sudo echo "Acquire::http {{ Proxy \\"{0}\\"; }};" > /etc/apt/apt.conf.d/02proxy && sudo apt-get update'
-""".format(config['aptproxy'])
-
-# buildserver/ is shared to the VM's /vagrant by default so the old default
-# does not need a custom mount
-if cachedir != 'buildserver/cache':
- vagrantfile += """
- config.vm.synced_folder '{0}', '/vagrant/cache',
- owner: 'root', group: 'root', create: true
-""".format(cachedir)
-
-# cache .deb packages on the host via a mount trick
-if config['apt_package_cache']:
- aptcachedir = cachedir + '/apt/archives'
- vagrantfile += """
- config.vm.synced_folder "{0}", "/var/cache/apt/archives",
- owner: 'root', group: 'root', create: true
-""".format(aptcachedir)
-
-vagrantfile += """
-
- config.vm.provision "shell", path: "setup-env-vars",
- args: ["/home/vagrant/android-sdk"]
-
- config.vm.provision :chef_solo do |chef|
- chef.cookbooks_path = "cookbooks"
- chef.log_level = :debug
- chef.json = {
- :settings => {
- :debian_mirror => "%s",
- :ubuntu_trusty => "%s",
- :user => "vagrant"
- }
- }
- chef.add_recipe "fdroidbuild-general"
- chef.add_recipe "kivy"
- end
-
- config.vm.provision "shell", path: "provision-android-sdk"
- config.vm.provision "shell", path: "provision-android-ndk",
- args: ["/home/vagrant/android-ndk"]
- config.vm.provision "shell", path: "provision-gradle"
- config.vm.provision "file", source: "gradle",
- destination: "/opt/gradle/bin/gradle"
-
-end
-""" % (config['debian_mirror'],
- str('14.04' in os.uname()[3]).lower())
-
-# Check against the existing Vagrantfile, and if they differ, we need to
-# create a new box:
-vf = os.path.join(serverdir, 'Vagrantfile')
+# Check against the existing Vagrantfile.yaml, and if they differ, we
+# need to create a new box:
+vf = os.path.join(serverdir, 'Vagrantfile.yaml')
writevf = True
if os.path.exists(vf):
vagrant(['halt'], serverdir)
- with open(vf, 'r') as f:
- oldvf = f.read()
- if oldvf != vagrantfile:
+ with open(vf, 'r', encoding='utf-8') as f:
+ oldconfig = yaml.load(f)
+ if config != oldconfig:
print("Server configuration has changed, rebuild from scratch is required")
vagrant(['destroy', '-f'], serverdir)
else:
else:
print("No existing server - building from scratch")
if writevf:
- with open(vf, 'w') as f:
- f.write(vagrantfile)
-
+ with open(vf, 'w', encoding='utf-8') as f:
+ yaml.dump(config, f)
print("Configuring build server VM")
returncode, out = vagrant(['up', '--provision'], serverdir, printout=True)
--- /dev/null
+Categories:Development
+License:GPLv3
+Source Code:https://github.com/eighthave/urzip
+Bitcoin:1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk
+
+Auto Name:OBB Main Old Version
+
+Repo Type:git
+Repo:https://github.com/eighthave/urzip.git
+
+
+Current Version Code:99999999
--- /dev/null
+Categories:Development
+License:GPLv3
+Source Code:https://github.com/eighthave/urzip
+Bitcoin:1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk
+
+Auto Name:OBB Two Versions
+
+Repo Type:git
+Repo:https://github.com/eighthave/urzip.git
+
+
+Current Version Code:99999999
--- /dev/null
+Categories:Development
+License:GPLv3
+Source Code:https://github.com/eighthave/urzip
+Bitcoin:1Fi5xUHiAPRKxHvyUGVFGt9extBe8Srdbk
+
+Auto Name:OBB Main+Patch Current Version
+
+Repo Type:git
+Repo:https://github.com/eighthave/urzip.git
+
+
+Current Version Code:99999999
$fdroid update
+#------------------------------------------------------------------------------#
+echo_header "copy tests/repo, generate a keystore, and update"
+
+REPOROOT=`create_test_dir`
+cd $REPOROOT
+$fdroid init
+cp -a $WORKSPACE/tests/metadata $WORKSPACE/tests/repo $REPOROOT/
+echo "accepted_formats = ['json', 'txt', 'xml', 'yml']" >> config.py
+$fdroid update --verbose
+test -e repo/index.xml
+test -e repo/index.jar
+grep -F '<application id=' repo/index.xml > /dev/null
+
+
#------------------------------------------------------------------------------#
echo_header "test metadata checks"
pysig = fdroidserver.update.getsig(apkfile)
self.assertIsNone(pysig, "python sig should be None: " + str(sig))
- def testScanApks(self):
+ def testScanApksAndObbs(self):
os.chdir(os.path.dirname(__file__))
if os.path.basename(os.getcwd()) != 'tests':
raise Exception('This test must be run in the "tests/" subdir')
fdroidserver.update.options = type('', (), {})()
fdroidserver.update.options.clean = True
+ fdroidserver.update.options.delete_unknown = True
- alltestapps = fdroidserver.metadata.read_metadata(xref=True)
- apps = dict()
- apps['info.guardianproject.urzip'] = alltestapps['info.guardianproject.urzip']
+ apps = fdroidserver.metadata.read_metadata(xref=True)
knownapks = fdroidserver.common.KnownApks()
apks, cachechanged = fdroidserver.update.scan_apks(apps, {}, 'repo', knownapks, False)
- self.assertEqual(len(apks), 1)
+ self.assertEqual(len(apks), 6)
apk = apks[0]
self.assertEqual(apk['minSdkVersion'], '4')
self.assertEqual(apk['targetSdkVersion'], '18')
self.assertFalse('maxSdkVersion' in apk)
+ fdroidserver.update.insert_obbs('repo', apps, apks)
+ for apk in apks:
+ if apk['id'] == 'obb.mainpatch.current':
+ self.assertEqual(apk.get('obbMainFile'), 'main.1619.obb.mainpatch.current.obb')
+ self.assertEqual(apk.get('obbPatchFile'), 'patch.1619.obb.mainpatch.current.obb')
+ elif apk['id'] == 'obb.main.oldversion':
+ self.assertEqual(apk.get('obbMainFile'), 'main.1434483388.obb.main.oldversion.obb')
+ self.assertIsNone(apk.get('obbPatchFile'))
+ elif apk['id'] == 'obb.main.twoversions':
+ self.assertIsNone(apk.get('obbPatchFile'))
+ if apk['versioncode'] == 1101613:
+ self.assertEqual(apk.get('obbMainFile'), 'main.1101613.obb.main.twoversions.obb')
+ elif apk['versioncode'] == 1101615:
+ self.assertEqual(apk.get('obbMainFile'), 'main.1101615.obb.main.twoversions.obb')
+ elif apk['versioncode'] == 1101617:
+ self.assertEqual(apk.get('obbMainFile'), 'main.1101615.obb.main.twoversions.obb')
+ else:
+ self.assertTrue(False)
+ elif apk['id'] == 'info.guardianproject.urzip':
+ self.assertIsNone(apk.get('obbMainFile'))
+ self.assertIsNone(apk.get('obbPatchFile'))
+
if __name__ == "__main__":
parser = optparse.OptionParser()
~ianmdlvl / fdroidserver.git / commitdiff