From b94830d9593ca8c243fef1bc0b29eb751f935cde Mon Sep 17 00:00:00 2001
Message-Id: <b94830d9593ca8c243fef1bc0b29eb751f935cde.1716649272.git.mdw@distorted.org.uk>
From: Mark Wooding <mdw@chiark.greenend.org.uk>
Date: Wed, 10 Feb 2016 02:24:42 +0000
Subject: [PATCH] New tools for building Debian packages using sbuild(1).
Organization: Straylight/Edgeware

From: Mark Wooding <mdw@distorted.org.uk>

There's now a service, which can do parallel builds of a package for
multiple platforms, and a client for it which can upload packages to be
built and download the results.  And there's integration with the
existing `mdw-build' script.
---
 bin/mdw-build         |  13 ++-
 bin/mdw-sbuild        | 110 +++++++++++++++++
 bin/mdw-sbuild-server | 266 ++++++++++++++++++++++++++++++++++++++++++
 dot/mdw-build.conf    |   1 +
 mdw.conf              |   1 +
 setup                 |   2 +-
 6 files changed, 389 insertions(+), 4 deletions(-)
 create mode 100755 bin/mdw-sbuild
 create mode 100755 bin/mdw-sbuild-server

diff --git a/bin/mdw-build b/bin/mdw-build
index 572aa75..24ae04a 100755
--- a/bin/mdw-build
+++ b/bin/mdw-build
@@ -59,6 +59,7 @@ set -e
 unset checkout checkoutrev
 unset setup setupcmd
 unset sign signkey
+unset sbuild sbuildsrv
 unset upload uploadpath
 unset dput dputtarget
 unset build distcheck debian clean vpath native
@@ -82,6 +83,7 @@ default_depends () {
 : ${clean=yes}
 : ${vpath=yes}
 : ${native=yes}
+default_depends sbuild sbuildsrv
 default_depends sign signkey
 default_depends upload uploadpath
 default_depends dput dputtarget
@@ -107,6 +109,7 @@ Build options:
   [no]dput[=TARGET]
   [no]clean
   [no]vpath
+  [no]sbuild[=SERVER]
   [no]sign[=KEYID]
   [no]native
 EOF
@@ -144,6 +147,8 @@ for opt; do
     upload=*)	upload=yes uploadpath=${opt#*=} ;;
     sign)	maybe_set sign signkey ;;
     sign=*)	sign=yes signkey=${opt#*=} ;;
+    sbuild)	maybe_set sbuild sbuildsrv ;;
+    sbuild=*)	sbuild=yes sbuildsrv=${opt#*=} ;;
     dput)	maybe_set dput dputtarget ;;
     dput=*)	dput=yes dputtarget=${opt#*=} ;;
 
@@ -151,7 +156,7 @@ for opt; do
       eval "$opt=yes"
       ;;
     nocheckout | nosetup | nodistcheck | nodebian | \
-    noupload | nodput | noclean | novpath | nonative | nosign)
+    noupload | nodput | noclean | novpath | nonative | nosbuild | nosign)
       eval "${opt#no}=no"
       ;;
     *)
@@ -400,8 +405,10 @@ EOF
 	mv debian/changelog.new debian/changelog
 	;;
     esac
-    case $build,$sign in
-      release,yes) run dpkg-buildpackage -k$signkey ;;
+    case $sbuild,$build,$sign in
+      yes,release,yes) run mdw-sbuild -k$signkey $sbuildsrv ;;
+      yes,*) run mdw-sbuild $sbuildsrv ;;
+      no,release,yes) run dpkg-buildpackage -k$signkey ;;
       no,*) run dpkg-buildpackage -us -uc ;;
     esac
     ;;
diff --git a/bin/mdw-sbuild b/bin/mdw-sbuild
new file mode 100755
index 0000000..84c24d8
--- /dev/null
+++ b/bin/mdw-sbuild
@@ -0,0 +1,110 @@
+#! /bin/sh -e
+###
+### Build a Debian package on an sbuild server.
+###
+### (c) 2016 Mark Wooding
+###
+
+###----- Licensing notice ---------------------------------------------------
+###
+### This program is free software; you can redistribute it and/or modify
+### it under the terms of the GNU General Public License as published by
+### the Free Software Foundation; either version 2 of the License, or
+### (at your option) any later version.
+###
+### This program is distributed in the hope that it will be useful,
+### but WITHOUT ANY WARRANTY; without even the implied warranty of
+### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+### GNU General Public License for more details.
+###
+### You should have received a copy of the GNU General Public License
+### along with this program; if not, write to the Free Software Foundation,
+### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+###--------------------------------------------------------------------------
+### Some utilities.
+
+prog=${0##*/}
+
+fail () { echo >&2 "$prog: $*"; exit 1; }
+usage () { echo "usage: $prog [-ain] [-k KEYID] [-t TARGET] HOST"; }
+fail_usage () { usage >&2; exit 1; }
+
+###--------------------------------------------------------------------------
+### Parse options.
+
+bogusp=nil noactp=nil
+unset buildopts keyid
+while getopts "haik:nt:" opt; do
+  case $opt in
+    h)
+      usage
+      cat <<EOF
+
+Options:
+	-h		Show this help text.
+	-a		Build only architecture-dependent packages.
+	-i		Build only architecture-neutral packages.
+	-k KEYID	Sign the result using KEYID.
+	-n		Don't actually do the build.
+	-t TARGET	Build in TARGET build environment.
+EOF
+      exit 0
+      ;;
+    a) buildopts="${buildopts+$buildopts }-a" ;;
+    i) buildopts="${buildopts+$buildopts }-i" ;;
+    k) keyid=$OPTARG ;;
+    n) buildopts="${buildopts+$buildopts }-n" noactp=t ;;
+    t) buildopts="${buildopts+$buildopts }-t$OPTARG" ;;
+    *) bogusp=t ;;
+  esac
+done
+shift $(( $OPTIND - 1 ))
+case $# in
+  1) host=$1 ;;
+  *) bogusp=t ;;
+esac
+case $bogusp in t) fail_usage ;; esac
+
+###--------------------------------------------------------------------------
+### Main program.
+
+## Figure out the package name and version number.
+unset pkg ver
+while read tag value; do
+  case $tag in
+    Source:) pkg=$value ;;
+    Version:) ver=$value ;;
+  esac
+done <<EOF
+$(dpkg-parsechangelog)
+EOF
+case ${pkg+t} in t) ;; *) fail "can't figure out the package name" ;; esac
+case ${ver+t} in t) ;; *) fail "can't figure out the package version" ;; esac
+
+## Build a Debian source package.  If we're signing, use `dpkg-buildpackage'
+## for this so that we get an uploadable `_source.changes' file out the end
+## of it.
+case ${keyid+t},$noactp in
+  t,nil)
+    dpkg-buildpackage -S -k"$keyid"
+    cd ..
+    ;;
+  *)
+    dir=$(pwd); base=${dir##*/}
+    cd ..
+    dpkg-source -b "$base"
+    ;;
+esac
+dsc=${pkg}_${ver}.dsc
+[ -f "$dsc" ] || fail "where is my \`.dsc' file?"
+
+builddir=$(ssh "$host" mdw-sbuild-server dir "$pkg/$ver")
+dcmd rsync -a "$dsc" "$host:$builddir/"
+set +e; ssh "$host" mdw-sbuild-server $buildopts build "$builddir"
+rc=$?; set -e
+rsync -a "$host:$builddir/" ./
+case $rc in 0) ;; *) exit $rc ;; esac
+case $?,${keyid+t},$noactp in
+  0,t,nil) debsign -k"$keyid" "${pkg}_${ver}_"*.changes ;;
+esac
diff --git a/bin/mdw-sbuild-server b/bin/mdw-sbuild-server
new file mode 100755
index 0000000..a1355b8
--- /dev/null
+++ b/bin/mdw-sbuild-server
@@ -0,0 +1,266 @@
+#! /bin/sh -e
+###
+### Build a Debian package on supported architectures
+###
+### (c) 2016 Mark Wooding
+###
+
+###----- Licensing notice ---------------------------------------------------
+###
+### This program is free software; you can redistribute it and/or modify
+### it under the terms of the GNU General Public License as published by
+### the Free Software Foundation; either version 2 of the License, or
+### (at your option) any later version.
+###
+### This program is distributed in the hope that it will be useful,
+### but WITHOUT ANY WARRANTY; without even the implied warranty of
+### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+### GNU General Public License for more details.
+###
+### You should have received a copy of the GNU General Public License
+### along with this program; if not, write to the Free Software Foundation,
+### Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+###--------------------------------------------------------------------------
+### Configuration.
+
+unset buildroot default_targets parallel
+for i in \
+  "/etc/mdw-sbuild.conf" \
+  "${XDG_CONFIG_HOME-$HOME/.config}/mdw-sbuild.conf"
+do
+  if [ -f "$i" ]; then . "$i"; fi
+done
+: ${buildroot=$HOME/build}
+: ${default_targets="wheezy-amd64 wheezy-i386"}
+: ${parallel=-j3}
+: ${DEB_BUILD_OPTIONS=parallel=4}; export DEB_BUILD_OPTIONS
+
+###--------------------------------------------------------------------------
+### Some utilities.
+
+prog=${0##*/}
+
+fail () { echo >&2 "$prog: $*"; exit 1; }
+usage () { echo "usage: $prog [-ain] [-t TARGET] COMMAND [ARGUMENTS ...]"; }
+fail_usage () { usage >&2; exit 1; }
+
+want_1 () {
+  what=$1 pat=$2 type=$3; shift 3
+  for i in "$@"; do
+    [ $type "$i" ] || fail "$what not found: \`$i'"
+  done
+  case $# in
+    1) ;;
+    *) fail "expected exactly one $what matching \`$pat', but found $#" ;;
+  esac
+  echo "$1"
+}
+
+###--------------------------------------------------------------------------
+### Parse options.
+
+bogusp=nil archp=nil indepp=nil makeopts=""
+unset targets
+
+while getopts "haint:" opt; do
+  case $opt in
+    h)
+      usage
+      cat <<EOF
+
+Options:
+	-h		Show this help text.
+	-a		Build only architecture-dependent packages.
+	-i		Build only architecture-neutral packages.
+	-n		Don't actually do the build.
+	-t TARGET	Build in TARGET build environment.
+
+Commands available:
+
+	dir PROJECT/VERSION
+		Return a freshly-made directory for the source code to
+		go in.
+
+	build BUILDDIR
+		Build the package placed in BUILDDIR, which should contain
+		exactly one \`.dsc' file, and whatever source archive files
+		are necessary.
+EOF
+      exit
+      ;;
+    a) archp=t ;;
+    i) indepp=t ;;
+    n) makeopts="${makeopts+$makeopts }-n" ;;
+    t) targets="${targets+$targets }$OPTARG" ;;
+    *) bogusp=nil ;;
+  esac
+done
+shift $(( $OPTIND - 1 ))
+
+case $bogusp in t) fail_usage ;; esac
+case $archp,$indepp in nil,nil) archp=t indepp=t ;; esac
+case ${targets+t} in t) ;; *) targets=$default_targets ;; esac
+
+###--------------------------------------------------------------------------
+### Main work.
+
+case "$#,$1" in
+  0,*) fail_usage ;;
+  *,*,*) fail "bad command name \`$1'" ;;
+
+  2,dir)
+    ## dirname PROJECT/VERSION
+
+    ## Try to create a fresh build directory.
+    dist=$2
+    case "$dist" in */*/*) fail "bad distribution name \`$dist'" ;; esac
+    proj=${dist%/*} ver=${dist#*/}
+    cd "$buildroot"
+    mkdir -p "$proj"
+    cd "$proj"
+    i=0
+    winp=nil
+    while [ $i -lt 50 ]; do
+      i=$(( $i + 1 ))
+
+      ## Find a sequence number different from all of the existing builds of
+      ## this version.
+      nn=1
+      for j in "$ver#"*; do
+	case "$j" in "$ver#*") break ;; esac
+	n=${j##*\#}
+	if [ $nn -le $n ]; then nn=$(( $n + 1 )); fi
+      done
+
+      ## Try to make the build directory.  This might not work if we're
+      ## racing with another process, but that's why we're trying in a loop.
+      if mkdir "$ver#$nn" >/dev/null 2>&1; then
+	winp=t
+	break
+      fi
+
+      ## Make sure it actually failed because a directory appeared, rather
+      ## than for some other reason.
+      [ -e "$ver#$nn" ] || \
+	fail "unexpectedly couldn't create \`$buildroot/$dist#$nn'"
+    done
+
+    ## Make sure we actually succeeded.
+    case $winp in t) ;; *) fail "failed to create build directory" ;; esac
+
+    ## Done.
+    echo "$buildroot/$dist#$nn"
+    ;;
+
+  *,dir)
+    echo >&2 "usage: $prog dir PROJECT/VERSION"; exit 1 ;;
+
+  2,build)
+    ## build BUILDDIR
+
+    ## Track down the build directory.
+    builddir=$2
+    cd "$builddir"
+    dsc=$(want_1 "file" "*.dsc" -f *.dsc)
+
+    ## Figure out which targets need building.  If the `.dsc' file isn't
+    ## telling, assume it needs building everywhere and let sbuild(1) sort
+    ## out the mess.
+    os=$(dpkg-architecture -qDEB_HOST_ARCH_OS)
+    unset first rest; anyp=nil depp=nil allp=nil
+    wantarchs=$(sed -n '/^[Aa]rchitecture:/ s/^[^:]*: *//p' "$dsc")
+    : ${wantarchs:=any}
+
+    ## Work through the available targets assigning builds to them.  This is
+    ## actually a little tricky.
+    for t in $targets; do
+
+      ## Work through the architectures which we can build.
+      for arch in $wantarchs; do
+	case $arch in
+	  all)
+	    ## Package suitable for all architectures.
+
+	    ## If we don't want to build architecture-neutral packages then
+	    ## there's nothing to do.
+	    case $indepp in nil) continue ;; esac
+
+	    ## Pick this up if nobody has volunteered.  However, we should be
+	    ## ready to let some other architecture build this if it's going
+	    ## to build some architecture-dependent package too.
+	    case $anyp in nil) first=$t anyp=t allp=t ;; esac
+	    ;;
+	  *)
+	    ## There's at least one architecture-specific package.
+
+	    ## If we don't want to build architecture-specific package then
+	    ## there's nothing to do.
+	    case $archp in nil) continue ;; esac
+
+	    ## If we can't build it then we shouldn't try.
+	    if ! dpkg-architecture -a"$os-${t#*-}" -i"$arch"; then
+	      continue
+	    fi
+
+	    ## Decide whether we should take responsibility for the
+	    ## architecture-neutral packages.  If nobody's claimed them yet,
+	    ## or the previous claimant wasn't building architecture-specific
+	    ## packages, we should take over.
+	    case $depp in
+	      nil) first=$t depp=t anyp=t ;;
+	      t) rest="${rest+$rest }$t" ;;
+	    esac
+	    ;;
+	esac
+      done
+    done
+
+    ## If we never found a match then we can't do anything.
+    case $anyp in nil) echo "$prog: no packages to build"; exit 0 ;; esac
+
+    ## Figure out the right options to use.
+    case $indepp in
+      t) firstopt="--arch-all" ;;
+      nil) firstopt="--no-arch-all" ;;
+    esac
+    case $archp in
+      t) ;;
+      nil) firstopt="$firstopt --debbuildopt=-A" ;;
+    esac
+
+    ## Build a cheesy makefile to run these in parallel.
+    cat >build.mk <<EOF
+### -*-makefile-*-
+DSC = $dsc
+FIRST = $first
+REST = $rest
+SBUILD = t=\$@; sbuild \\
+	--dist=\$\${t%-*} --arch=\$\${t\#*-} \\
+	--chroot=\$@ --verbose
+TAGLINES = \\
+	while IFS= read -r line; do printf "%s: %s\n" "\$@" "\$\$line"; done
+all: \$(FIRST) \$(REST)
+\$(FIRST):; \$(SBUILD) $firstopt \$(DSC) | \$(TAGLINES)
+\$(REST):; \$(SBUILD) --no-arch-all \$(DSC) | \$(TAGLINES)
+EOF
+
+    ## And we're ready to go.
+    mkfifo pipeout
+    cat pipeout& catpid=$!
+    set +e; make -fbuild.mk $parallel $makeopts -k all >pipeout
+    rc=$?; set -e
+    wait $!
+    rm build.mk pipeout
+    find . -maxdepth 1 -type l -exec rm {} \;
+    exit $rc
+    ;;
+  build,*)
+    echo >&2 "usage: $prog build BUILDDIR"; exit 1 ;;
+
+  *)
+    fail "unknown command \`$1'"
+    ;;
+esac
+
+###----- That's all, folks --------------------------------------------------
diff --git a/dot/mdw-build.conf b/dot/mdw-build.conf
index 9c3dd64..db3fbd1 100644
--- a/dot/mdw-build.conf
+++ b/dot/mdw-build.conf
@@ -14,6 +14,7 @@ setup=yes setupcmd=mdw-setup
 
 set_from_mdw_conf uploadpath upload-target
 set_from_mdw_conf dputtarget dput-target
+set_from_mdw_conf sbuildsrv sbuild-server
 set_from_mdw_conf signkey releasekey
 
 DEB_BUILD_OPTIONS=parallel=4
diff --git a/mdw.conf b/mdw.conf
index 439b83c..5dac443 100644
--- a/mdw.conf
+++ b/mdw.conf
@@ -14,3 +14,4 @@ x-ctype = en_GB.utf8
 releasekey = E359CA55
 upload-target = ftp.distorted.org.uk:~ftp/pub/mdw/
 dput-target = distorted
+sbuild-server = universe.distorted.org.uk
diff --git a/setup b/setup
index 514a4cb..e0cf105 100755
--- a/setup
+++ b/setup
@@ -258,7 +258,7 @@ scripts="
   mdw-editor
   mdw-pager
   mdw-conf
-  mdw-build
+  mdw-build mdw-sbuild mdw-sbuild-server
   update-buildable-branch
   emacsclient-hack
   movemail-hack
-- 
[mdw]