- ./complete-ci-tests
# Test that the parsing of the .txt format didn't change from last
-# released version. Ensure that the official tags are included when
-# running these tests on forks as well.
+# released version. This uses the commit ID of the release tags,
+# rather than the release tag itself so that contributor forks do not
+# need to include the tags in them for this test to work.
+#
+# The COMMIT_ID should be bumped after each release, so that the list
+# of sed hacks needed does not continuously grow.
metadata_v0:
image: registry.gitlab.com/fdroid/ci-images-server:latest
+ variables:
+ RELEASE_COMMIT_ID: 6d69dcddd95fe08ffe431e305932cfdeafd6fc9d # 1.0.0
script:
- - git fetch https://gitlab.com/fdroid/fdroidserver 0.8
+ - git fetch https://gitlab.com/fdroid/fdroidserver $RELEASE_COMMIT_ID
- cd tests
- export GITCOMMIT=`git describe`
- - git checkout 0.8 # bump after release
+ - git checkout $RELEASE_COMMIT_ID
- cd ..
- git clone --depth 1 https://gitlab.com/fdroid/fdroiddata
- cd fdroiddata
- git checkout $GITCOMMIT
- cd fdroiddata
- ../tests/dump_internal_metadata_format.py
- - sed -i -e '/LiberapayID/d'
- metadata/dump_*/*.yaml
- diff -uw metadata/dump_*
debian_testing:
printf 'APT::Acquire::Retries "20";\n' \
> /etc/apt/apt.conf.d/99acquire-retries
+printf 'APT::Periodic::Update-Package-Lists "0";\nAPT::Periodic::Unattended-Upgrade "0";\n' \
+ > /etc/apt/apt.conf.d/99no-auto-updates
+
if echo $debian_mirror | grep '^https' 2>&1 > /dev/null; then
apt-get -y update
apt-get -y install apt-transport-https
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-import sys
import os
import shutil
import glob
import subprocess
import re
import resource
+import sys
import tarfile
import traceback
import time
try:
cmd_stdout = chan.makefile('rb', 1024)
output = bytes()
- output += get_android_tools_version_log(build.ndk_path()).encode()
+ output += common.get_android_tools_version_log(build.ndk_path()).encode()
while not chan.exit_status_ready():
line = cmd_stdout.readline()
if line:
path)
-def _get_build_timestamp():
- return time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime())
-
-
def transform_first_char(string, method):
"""Uses method() on the first character of string."""
if len(string) == 0:
log_path = os.path.join(log_dir,
common.get_toolsversion_logname(app, build))
with open(log_path, 'w') as f:
- f.write(get_android_tools_version_log(build.ndk_path()))
+ f.write(common.get_android_tools_version_log(build.ndk_path()))
else:
if build.sudo:
logging.warning('%s:%s runs this on the buildserver with sudo:\n\t%s'
return True
-def get_android_tools_versions(ndk_path=None):
- '''get a list of the versions of all installed Android SDK/NDK components'''
-
- global config
- sdk_path = config['sdk_path']
- if sdk_path[-1] != '/':
- sdk_path += '/'
- components = []
- if ndk_path:
- ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT')
- if os.path.isfile(ndk_release_txt):
- with open(ndk_release_txt, 'r') as fp:
- components.append((os.path.basename(ndk_path), fp.read()[:-1]))
-
- pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE)
- for root, dirs, files in os.walk(sdk_path):
- if 'source.properties' in files:
- source_properties = os.path.join(root, 'source.properties')
- with open(source_properties, 'r') as fp:
- m = pattern.search(fp.read())
- if m:
- components.append((root[len(sdk_path):], m.group(1)))
-
- return components
-
-
-def get_android_tools_version_log(ndk_path):
- '''get a list of the versions of all installed Android SDK/NDK components'''
- log = '== Installed Android Tools ==\n\n'
- components = get_android_tools_versions(ndk_path)
- for name, version in sorted(components):
- log += '* ' + name + ' (' + version + ')\n'
-
- return log
-
-
def parse_commandline():
"""Parse the command line. Returns options, parser."""
options = None
config = None
buildserverid = None
-starttime = _get_build_timestamp()
+fdroidserverid = None
+start_timestamp = time.gmtime()
def main():
- global options, config, buildserverid
+ global options, config, buildserverid, fdroidserverid
options, parser = parse_commandline()
for build in app.builds:
wikilog = None
- build_starttime = _get_build_timestamp()
+ build_starttime = common.get_wiki_timestamp()
tools_version_log = ''
if not options.onserver:
- tools_version_log = get_android_tools_version_log(build.ndk_path())
+ tools_version_log = common.get_android_tools_version_log(build.ndk_path())
try:
# For the first build of a particular app, we need to set up
f.write('versionCode: %s\nversionName: %s\ncommit: %s\n' %
(build.versionCode, build.versionName, build.commit))
f.write('Build completed at '
- + _get_build_timestamp() + '\n')
+ + common.get_wiki_timestamp() + '\n')
f.write('\n' + tools_version_log + '\n')
f.write(str(e))
logging.error("Could not build app %s: %s" % (appid, e))
newpage = site.Pages[lastbuildpage]
with open(os.path.join('tmp', 'fdroidserverid')) as fp:
fdroidserverid = fp.read().rstrip()
- txt = "* build session started at " + starttime + '\n' \
+ txt = "* build session started at " + common.get_wiki_timestamp(start_timestamp) + '\n' \
+ "* this build started at " + build_starttime + '\n' \
- + "* this build completed at " + _get_build_timestamp() + '\n' \
+ + "* this build completed at " + common.get_wiki_timestamp() + '\n' \
+ '* fdroidserverid: [https://gitlab.com/fdroid/fdroidserver/commit/' \
+ fdroidserverid + ' ' + fdroidserverid + ']\n\n'
if buildserverid:
logging.info(ngettext("{} build failed",
"{} builds failed", len(failed_apps)).format(len(failed_apps)))
+ if options.wiki:
+ wiki_page_path = 'build_' + time.strftime('%s', start_timestamp)
+ newpage = site.Pages[wiki_page_path]
+ txt = ''
+ txt += "* command line: <code>%s</code>\n" % ' '.join(sys.argv)
+ txt += "* started at %s\n" % common.get_wiki_timestamp(start_timestamp)
+ txt += "* completed at %s\n" % common.get_wiki_timestamp()
+ if buildserverid:
+ txt += ('* buildserverid: [https://gitlab.com/fdroid/fdroidserver/commit/{id} {id}]\n'
+ .format(id=buildserverid))
+ if fdroidserverid:
+ txt += ('* fdroidserverid: [https://gitlab.com/fdroid/fdroidserver/commit/{id} {id}]\n'
+ .format(id=fdroidserverid))
+ if os.cpu_count():
+ txt += "* host processors: %d\n" % os.cpu_count()
+ if os.path.isfile('/proc/meminfo') and os.access('/proc/meminfo', os.R_OK):
+ with open('/proc/meminfo') as fp:
+ for line in fp:
+ m = re.search(r'MemTotal:\s*([0-9].*)', line)
+ if m:
+ txt += "* host RAM: %s\n" % m.group(1)
+ break
+ txt += "* successful builds: %d\n" % len(build_succeeded)
+ txt += "* failed builds: %d\n" % len(failed_apps)
+ txt += "\n\n"
+ newpage.save(txt, summary='Run log')
+ newpage = site.Pages['build']
+ newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
+
# hack to ensure this exits, even is some threads are still running
sys.stdout.flush()
sys.stderr.flush()
import urllib.error
import time
import subprocess
+import sys
from argparse import ArgumentParser
import traceback
import html
raise FDroidException("Git commit failed")
+def update_wiki(gplaylog, locallog):
+ if config.get('wiki_server') and config.get('wiki_path'):
+ try:
+ import mwclient
+ site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
+ path=config['wiki_path'])
+ site.login(config['wiki_user'], config['wiki_password'])
+
+ # Write a page with the last build log for this version code
+ wiki_page_path = 'checkupdates_' + time.strftime('%s', start_timestamp)
+ newpage = site.Pages[wiki_page_path]
+ txt = ''
+ txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
+ txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
+ txt += "* completed at " + common.get_wiki_timestamp() + '\n'
+ txt += "\n\n"
+ txt += common.get_android_tools_version_log()
+ txt += "\n\n"
+ if gplaylog:
+ txt += '== --gplay check ==\n\n'
+ txt += gplaylog
+ if locallog:
+ txt += '== local source check ==\n\n'
+ txt += locallog
+ newpage.save(txt, summary='Run log')
+ newpage = site.Pages['checkupdates']
+ newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
+ except Exception as e:
+ logging.error(_('Error while attempting to publish log: %s') % e)
+
+
config = None
options = None
+start_timestamp = time.gmtime()
def main():
apps = common.read_app_args(options.appid, allapps, False)
+ gplaylog = ''
if options.gplay:
for appid, app in apps.items():
+ gplaylog += '* ' + appid + '\n'
version, reason = check_gplay(app)
if version is None:
if reason == '404':
else:
logging.info("{0} has the same version {1} on the Play Store"
.format(common.getappname(app), version))
+ update_wiki(gplaylog, None)
return
+ locallog = ''
for appid, app in apps.items():
if options.autoonly and app.AutoUpdateMode in ('None', 'Static'):
logging.debug(_("Nothing to do for {appid}.").format(appid=appid))
continue
- logging.info(_("Processing {appid}").format(appid=appid))
+ msg = _("Processing {appid}").format(appid=appid)
+ logging.info(msg)
+ locallog += '* ' + msg + '\n'
try:
checkupdates_app(app)
except Exception as e:
- logging.error(_("...checkupdate failed for {appid} : {error}")
- .format(appid=appid, error=e))
+ msg = _("...checkupdate failed for {appid} : {error}").format(appid=appid, error=e)
+ logging.error(msg)
+ locallog += msg + '\n'
+
+ update_wiki(None, locallog)
logging.info(_("Finished"))
examplesdir = prefix + '/examples'
return examplesdir
+
+
+def get_wiki_timestamp(timestamp=None):
+ """Return current time in the standard format for posting to the wiki"""
+
+ if timestamp is None:
+ timestamp = time.gmtime()
+ return time.strftime("%Y-%m-%d %H:%M:%SZ", timestamp)
+
+
+def get_android_tools_versions(ndk_path=None):
+ '''get a list of the versions of all installed Android SDK/NDK components'''
+
+ global config
+ sdk_path = config['sdk_path']
+ if sdk_path[-1] != '/':
+ sdk_path += '/'
+ components = []
+ if ndk_path:
+ ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT')
+ if os.path.isfile(ndk_release_txt):
+ with open(ndk_release_txt, 'r') as fp:
+ components.append((os.path.basename(ndk_path), fp.read()[:-1]))
+
+ pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE)
+ for root, dirs, files in os.walk(sdk_path):
+ if 'source.properties' in files:
+ source_properties = os.path.join(root, 'source.properties')
+ with open(source_properties, 'r') as fp:
+ m = pattern.search(fp.read())
+ if m:
+ components.append((root[len(sdk_path):], m.group(1)))
+
+ return components
+
+
+def get_android_tools_version_log(ndk_path=None):
+ '''get a list of the versions of all installed Android SDK/NDK components'''
+ log = '== Installed Android Tools ==\n\n'
+ components = get_android_tools_versions(ndk_path)
+ for name, version in sorted(components):
+ log += '* ' + name + ' (' + version + ')\n'
+
+ return log
config = None
options = None
+start_timestamp = time.gmtime()
BINARY_TRANSPARENCY_DIR = 'binary_transparency'
origin.push('master')
+def update_wiki():
+ try:
+ import mwclient
+ site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
+ path=config['wiki_path'])
+ site.login(config['wiki_user'], config['wiki_password'])
+
+ # Write a page with the last build log for this version code
+ wiki_page_path = 'deploy_' + time.strftime('%s', start_timestamp)
+ newpage = site.Pages[wiki_page_path]
+ txt = ''
+ txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
+ txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
+ txt += "* completed at " + common.get_wiki_timestamp() + '\n'
+ txt += "\n\n"
+ newpage.save(txt, summary='Run log')
+ newpage = site.Pages['deploy']
+ newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
+ except Exception as e:
+ logging.error(_('Error while attempting to publish log: %s') % e)
+
+
def main():
global config, options
push_binary_transparency(BINARY_TRANSPARENCY_DIR,
binary_transparency_remote)
+ if config.get('wiki_server') and config.get('wiki_path'):
+ update_wiki()
+
sys.exit(0)
logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
# Purge server cache to ensure counts are up to date
- site.pages['Repository Maintenance'].purge()
+ site.Pages['Repository Maintenance'].purge()
+
+ # Write a page with the last build log for this version code
+ wiki_page_path = 'update_' + time.strftime('%s', start_timestamp)
+ newpage = site.Pages[wiki_page_path]
+ txt = ''
+ txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
+ txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
+ txt += "* completed at " + common.get_wiki_timestamp() + '\n'
+ txt += "\n\n"
+ txt += common.get_android_tools_version_log()
+ newpage.save(txt, summary='Run log')
+ newpage = site.Pages['update']
+ newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
def delete_disabled_builds(apps, apkcache, repodirs):
config = None
options = None
+start_timestamp = time.gmtime()
def main():
# when everything is copied over to run on SIGN machine
../fdroid publish
../fdroid gpgsign
-# when everything is copied over to run on BUILD machine
+# when everything is copied over to run on BUILD machine,
+# which does not have a keyring, only a cached pubkey
+echo "repo_pubkey = '308204e1308202c9a003020102020434597643300d06092a864886f70d01010b050030213110300e060355040b1307462d44726f6964310d300b06035504031304736f7661301e170d3136303931333230313930395a170d3434303133303230313930395a30213110300e060355040b1307462d44726f6964310d300b06035504031304736f766130820222300d06092a864886f70d01010105000382020f003082020a028202010086ef94b5aacf2ba4f38c875f4194b44f5644392e3715575d7c92828577e692c352b567172823851c8c72347fbc9d99684cd7ca3e1db3e4cca126382c53f2a5869fb4c19bdec989b2930501af3e758ff40588915fe96b10076ce3346a193a0277d79e83e30fd8657c20e35260dd085aa32eac7c4b85786ffefbf1555cafe2bc928443430cdbba48cfbe701e12ae86e676477932730d4fc7c00af820aef85038a5b4df084cf6470d110dc4c49ea1b749b80b34709d199b3db516b223625c5de4501e861f7d261b3838f8f616aa78831d618d41d25872dc810c9b2087b5a9e146ca95be740316dcdbcb77314e23ab87d4487913b800b1113c0603ea2294188b71d3e49875df097b56f9151211fc6832f9790c5c83d17481f14ad37915fd164f4fd713f6732a15f4245714b84cd665bdbd085660ea33ad7d7095dcc414f09e3903604a40facc2314a115c0045bb50e9df38efb57e1b8e7cc105f340a26eeb46aba0fa6672953eee7f1f92dcb408e561909bbd4bdf4a4948c4d57c467d21aa238c34ba43be050398be963191fa2b49828bc1e4eeed224b40dbe9dc3e570890a71a974a2f4527edb1b07105071755105edcb2af2f269facfb89180903a572a99b46456e80d4a01685a80b233278805f2c876678e731f4ec4f52075aeef6b2b023efbb8a3637ef507c4c37c27e428152ec1817fcba640ad601cb09f72f0fbe2d274a2410203010001a321301f301d0603551d0e04160414c28bf33dd5a9a17338e5b1d1a6edd8c7d141ed0b300d06092a864886f70d01010b0500038202010084e20458b2aafd7fc27146b0986f9324f4260f244920417a77c9bf15e2e2d22d2725bdd8093ec261c3779c3ca03312516506f9410075b90595b41345956d8eb2786fb5994f195611382c2b99dba13381b0100a30bc9e6e47248bf4325e2f6eec9d789216dc7536e753bf1f4be603d9fa2e6f5e192b4eb988b8cdb0bb1e8668a9225426f7d4636479f73ed24ad1d2657c31e63c93d9679b9080171b3bd1bf10a3b92b80bd790fbf62d3644900cd08eae8b9bf9c2567be98dc8cdd2ae19a8d57a3e3e2de899f81f1279f578989e6af906f80c8c2b67651730ee7e568c1af5bcb845b6d685dc55332a9984aeceaea3b7e883447edf1c76b155d95253e39b9710eaa22efa6c81468829702b5dce7126538f3ca70c2f0ad9a5795435fdb1f715f20d60359ef9a9926c7050116e802df651727447848827815f70bd82af3cedd08783156102d2d8ce995c4c43b8e47e91a3e6927f3505a5d395e6bebb84542c570903eeab4382a1c2151f1471c7a06a34dc4d268d8fa72e93bdcd2dccc4302ecac47b9e7e3d8bc9b46d21cd097874a24d529548018dc190ff568c6aa428f0a5eedff1a347730931c74f19277538e49647a4ad7254f4c1ec7d4da12cce9e1fad9607534e66ab40a56b473d9d7e3d563fd03cad2052bad365c5a29f8ae54f09b60dbca3ea768d7767cbe1c133ca08ce725c1c1370f4aab8e5b6e286f52dc0be8d0982b5a'" >> config.py
../fdroid update --nosign
+sed -i '/^repo_pubkey = /d' config.py
# when everything is copied over to run on SIGN machine
../fdroid signindex --verbose
~ianmdlvl / fdroidserver.git / commitdiff