[distorted-chroot] / bin / update-cross-tools

#! /bin/sh -e
###
### Fetch native versions of tools for insertion into foreign chroots
###
### (c) 2018 Mark Wooding
###

###----- Licensing notice ---------------------------------------------------
###
### This file is part of the distorted.org.uk chroot maintenance tools.
###
### distorted-chroot is free software: you can redistribute it and/or
### modify it under the terms of the GNU General Public License as
### published by the Free Software Foundation; either version 2 of the
### License, or (at your option) any later version.
###
### distorted-chroot is distributed in the hope that it will be useful,
### but WITHOUT ANY WARRANTY; without even the implied warranty of
### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
### General Public License for more details.
###
### You should have received a copy of the GNU General Public License
### along with distorted-chroot.  If not, write to the Free Software
### Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
### USA.

. state/config.sh # @@@config@@@

###--------------------------------------------------------------------------
### Utilities.

chase_link () {
  p=$1 d=
  ## Copy an absolute path P from the donor tree `$root/' into the
  ## cross-tools tree `$crossnew/'.  If resolving P involves traversing a
  ## symbolic link, then ensure that the pieces of filesystem it directs us
  ## to are also copied.

  ## Work through the remaining components of the path.
  while :; do

    ## Analyse the first remaining component of the path P.
    case $p in

      ## It's empty.  We're done.
      "") break ;;

      ## A redundant `/' or `./'.  Skip it.
      "/"*) p=${p#/} ;;
      "./"*) p=${p#./} ;;

      ## A `../'.  Strip off the trailing component of D.
      "../"*)
	p=${p#../}
	case $d in */*) d=${d%/*} ;; *) d= ;; esac
	;;

      ## Something else.  Transfer the component name to D.
      *)
	case $p in */*) f=${p%%/*} p=${p#*/} ;; *) f=$p p="" ;; esac
	d=${d:+$d/}$f
	;;
    esac

    ## If D doesn't refer to a file in the cross-tools tree, then maybe it
    ## refers to something in the donor tree.  Find out what, and copy it
    ## into the cross-tools tree.
    if ! [ -e "$crossnew$d" ] && ! [ -L "$crossnew$d" ]; then
      if [ -d "$root/$d" ] && ! [ -L "$root/$d" ]; then
	mkdir "$crossnew$d"
      else
	echo >&2 "$0: copy /$d to satisfy symlinks"
	rsync -aHR $root/./$d $crossnew
      fi
    fi

    ## If D refers to a symbolic link, then append the link target to P, so
    ## that we can make sure we copy the target.
    if [ -L "$crossnew$d" ]; then
      t=$(readlink "$crossnew$d")
      case $t in /*) t=${t#/} d= ;; esac
      case $d in */*) d=${d%/*} ;; *) d= ;; esac
      p=$t${p:+/$p}
    fi
  done
}

###--------------------------------------------------------------------------
### Main program.

## Parse the command line.
badp=nil
case $# in 2) ;; *) badp=t ;; esac
case $badp in t) echo >&2 "usage: $0 DIST MYARCH"; exit 2 ;; esac
d=$1 myarch=$2

## Keep track of our original stdout.
exec 3>&1

## Figure out derived architecture names.
mymulti=$(dpkg-architecture -a$myarch -qDEB_HOST_MULTIARCH)

## First, set `cross_archs' as a list of GNUish names for our supported
## foreign architectures.
cross_archs="arm-linux-gnueabi arm-linux-gnueabihf aarch64-linux-gnu"

## Make a list of extra packages we'll need to install to obtain our tools.
cross_pkgs="
	apt bash ccache coreutils dash eatmydata fakeroot findutils
	gpgv gzip m4 make mawk qemu-user-static sed tar xz-utils"
for a in $cross_archs; do
  for i in gcc g++ binutils; do
    cross_pkgs="$cross_pkgs $i-$a"
  done
done
cross_pkgs=$(echo $cross_pkgs)

## Make an enormous shopping list of paths.
##
## The `wanted' list consists of two kinds of items: an absolute path names a
## prefix (not necessarily a directory name) to be attached to the following
## relative names, up to the end of the list or the next absolute path.
wanted="
	/usr/bin/ apt apt-cache apt-config apt-get apt-key apt-mark
	/usr/lib/apt/ methods/ solvers/

	/bin/ cat chgrp chown cp date dd df dir echo false ln ls mkdir
		mknod mktemp mv pwd readlink rm rmdir sleep stty sync touch
		true uname vdir
	/usr/bin/ [ arch b2sum base32 base64 basename chcon cksum comm
		csplit cut dircolors dirname du env expand expr factor fmt
		fold groups head hostid id install join link logname md5sum
		mkfifo nice nl nohup nproc numfmt od paste pathchk pinky pr
		printenv printf ptx realpath runcon seq sha1sum sha224sum
		sha256sum sha384sum sha512sum shred shuf sort split stat
		stdbuf sum tac tail tee test timeout tr truncate tsort tty
		unexpand uniq unlink users wc who whoami yes
	/usr/lib/$mymulti/ coreutils/

	/lib/$mymulti/ libnss_*.so.*

	/usr/bin/ gpgv

	/usr/bin/ qemu-*-static

	/bin/ bash dash gzip sed tar
	/usr/bin/ ccache find m4 make mawk xargs xz
	/usr/lib/$mymulti/ libeatmydata.so* libfakeroot/

	/etc/ld.so.conf.d/ $mymulti.conf fakeroot*.conf"

for a in $cross_archs; do
  wanted="$wanted

	/usr/bin/$a- addr2line ar as c++filt dwp elfedit gprof ld ld.*
		nm objcopy objdump ranlib readelf size strings strip

	/usr/bin/$a- cpp gcc g++ gcov gcov-dump gcov-tool gprof
		gcc-ar gcc-nm gcc-ranlib
	/usr/lib/gcc-cross/$a/ ..."
done
wanted=$(echo $wanted)

## Figure out how to recognize dynamic executables.
case $myarch in
  i386) elfsig=7f454c46010101??0000000000000000????0300 ;;
  amd64) elfsig=7f454c46020101??0000000000000000????3e00 ;;
  *) echo >&2 "$0: unsupported local arch \`$myarch'"; exit 2 ;;
esac

## Open a session to the donor chroot.
echo >&2 "$0: create $d snapshot"
sess=$(schroot -bc$LVPREFIX$d-$myarch 3>&-)

## Make sure the donor tree is up-to-date, and install the extra packages we
## need.
schroot -uroot -rc$sess -- eatmydata sh -ec "
	apt-get update
	apt-get -y upgrade
	apt-get -y install $cross_pkgs"

## Establish some pathnames.  Prepare a place for our cross-tools tree.
crossdir=$LOCAL/cross/$d-$myarch/
crossold=${crossdir%/}.old/ crossnew=${crossdir%/}.new/
root=/schroot/$sess/fs
rm -rf $crossnew; mkdir -p $crossnew

## Work through the shopping list, copying the things it names into the
## cross-tools tree.
dir=/
for i in $wanted; do
  case $i in
    /*)
      dir=$i
      ;;
    *)
      case $i in ...) f=$dir ;; *) f=$dir$i ;; esac
      echo >&2 "$0: copy $f"
      rsync -aHR $root/.$f $crossnew
      ;;
  esac
done

## Chase links in the new tree, copying extra stuff that we'll need.
find $crossnew -xtype l -print | while read i; do
  chase_link ${i#$crossnew}
done

## Search the new tree for ELF binaries, and build a list of them in
## `QUEUE.in'.
find $crossnew -type f -print | while read i; do
  sig=$(head -c20 "$i" | bincode -e -m0 -flowerc hex)
  case $sig in $elfsig) echo "$i" ;; esac
done >$root/private/QUEUE.in

while [ -s $root/private/QUEUE.in ]; do
  ## Work through the ELF binaries in `QUEUE.in', determining which shared
  ## libraries they'll need.  Write the list of dependencies to `QUEUE.out'
  schroot -uroot -rc$sess -- eatmydata sh -ec '
	prog=$1
	while read i; do
	echo >&2 "$prog: scanning binary $i"
	  ldd "$i" | while read a b c d; do
	    case $a:$b:$c:$d in
	      not:a:dynamic:executable) ;;
	      statically:linked::) ;;
	      /*) echo "$a" ;;
	      *:=\>:/*) echo "$c" ;;
	      linux-*) ;;
	      *) echo >&2 "$i: unable to find $a"; exit 2 ;;
	    esac
	  done
	done </private/QUEUE.in >/private/QUEUE.out' - "$0"

  ## Work through the shared libraries in `QUEUE.out', copying them to the
  ## cross-tools tree if they're not there already.  Add the new ones to a
  ## new `QUEUE.in' file to scan them in turn.
  while read i; do
    if [ -e "$crossnew$i" ] || [ -L "$crossnew$i" ]
    then continue; fi
    if [ -d "$root$i" ]; then continue; fi
    echo >&2 "$0: copy $i"
    rsync -aHR $root/.$i $crossnew >&3
    chase_link $i >&3
    sig=$(head -c20 $crossnew$i | bincode -e -m0 -flowerc hex)
    case $sig in $elfsig) echo "$i" ;; esac
  done <$root/private/QUEUE.out >$root/private/QUEUE.in
done

## Set up the cross-compiler.  This is rather hairy.
echo >&2 "$0: establish TOOLCHAIN"
for a in $cross_archs; do
  tooldir=$crossnew/TOOLCHAIN/$a
  mkdir -p $tooldir
  for i in $crossnew/usr/bin/$a-*; do
    t=${i#$crossnew/usr/bin/}
    mv $i $tooldir/$t
    ln -s $t $tooldir/${t#$a-}
  done
done
mkdir $crossnew/TOOLCHAIN/lib
ln -s ../../usr/lib/gcc-cross $crossnew/TOOLCHAIN/lib/

## Set up the emulator.
echo >&2 "$0: establish QEMU"
mkdir $crossnew/QEMU
mv $crossnew/usr/bin/qemu-*-static $crossnew/QEMU/

## We're done.  Remove the snapshot, and replace the old cross-tools tree
## with our new one.
echo >&2 "$0: remove snapshot"
schroot -ec$sess 3>&-
if [ -d $crossdir ]; then mv $crossdir $crossold; fi
mv $crossnew $crossdir; rm -rf $crossold
echo >&2 "$0: committed $crossdir"

###----- That's all, folkd --------------------------------------------------
Commit	Line	Data
3e5b03e2 MW	1	#! /bin/sh -e
	2	###
	3	### Fetch native versions of tools for insertion into foreign chroots
	4	###
	5	### (c) 2018 Mark Wooding
	6	###
	7
	8	###----- Licensing notice ---------------------------------------------------
	9	###
	10	### This file is part of the distorted.org.uk chroot maintenance tools.
	11	###
	12	### distorted-chroot is free software: you can redistribute it and/or
	13	### modify it under the terms of the GNU General Public License as
	14	### published by the Free Software Foundation; either version 2 of the
	15	### License, or (at your option) any later version.
	16	###
	17	### distorted-chroot is distributed in the hope that it will be useful,
	18	### but WITHOUT ANY WARRANTY; without even the implied warranty of
	19	### MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	20	### General Public License for more details.
	21	###
	22	### You should have received a copy of the GNU General Public License
	23	### along with distorted-chroot. If not, write to the Free Software
	24	### Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
	25	### USA.
	26
	27	. state/config.sh # @@@config@@@
	28
	29	###--------------------------------------------------------------------------
	30	### Utilities.
	31
	32	chase_link () {
	33	p=$1 d=
	34	## Copy an absolute path P from the donor tree `$root/' into the
	35	## cross-tools tree `$crossnew/'. If resolving P involves traversing a
	36	## symbolic link, then ensure that the pieces of filesystem it directs us
	37	## to are also copied.
	38
	39	## Work through the remaining components of the path.
	40	while :; do
	41
	42	## Analyse the first remaining component of the path P.
	43	case $p in
	44
	45	## It's empty. We're done.
	46	"") break ;;
	47
	48	## A redundant `/' or `./'. Skip it.
	49	"/"*) p=${p#/} ;;
	50	"./"*) p=${p#./} ;;
	51
	52	## A `../'. Strip off the trailing component of D.
	53	"../"*)
	54	p=${p#../}
	55	case $d in /) d=${d%/} ;; ) d= ;; esac
	56	;;
	57
	58	## Something else. Transfer the component name to D.
	59	*)
	60	case $p in /) f=${p%%/} p=${p#/} ;; *) f=$p p="" ;; esac
	61	d=${d:+$d/}$f
	62	;;
	63	esac
	64
65	## If D doesn't refer to a file in the cross-tools tree, then maybe it
66	## refers to something in the donor tree. Find out what, and copy it
67	## into the cross-tools tree.
68	if ! [ -e "$crossnew$d" ] && ! [ -L "$crossnew$d" ]; then
69	if [ -d "$root/$d" ] && ! [ -L "$root/$d" ]; then
70	mkdir "$crossnew$d"
71	else
72	echo >&2 "$0: copy /$d to satisfy symlinks"
73	rsync -aHR $root/./$d $crossnew
74	fi
75	fi
76
77	## If D refers to a symbolic link, then append the link target to P, so
78	## that we can make sure we copy the target.
79	if [ -L "$crossnew$d" ]; then
80	t=$(readlink "$crossnew$d")
81	case $t in /*) t=${t#/} d= ;; esac
82	case $d in /) d=${d%/} ;; ) d= ;; esac
83	p=$t${p:+/$p}
84	fi
85	done
86	}
87
88	###--------------------------------------------------------------------------
89	### Main program.
90
91	## Parse the command line.
92	badp=nil
93	case $# in 2) ;; *) badp=t ;; esac
94	case $badp in t) echo >&2 "usage: $0 DIST MYARCH"; exit 2 ;; esac
95	d=$1 myarch=$2
96
97	## Keep track of our original stdout.
98	exec 3>&1
99
100	## Figure out derived architecture names.
101	mymulti=$(dpkg-architecture -a$myarch -qDEB_HOST_MULTIARCH)
102
103	## First, set `cross_archs' as a list of GNUish names for our supported
104	## foreign architectures.
105	cross_archs="arm-linux-gnueabi arm-linux-gnueabihf aarch64-linux-gnu"
106
107	## Make a list of extra packages we'll need to install to obtain our tools.
108	cross_pkgs="
109	apt bash ccache coreutils dash eatmydata fakeroot findutils
6b608a0f	110	gpgv gzip m4 make mawk qemu-user-static sed tar xz-utils"
3e5b03e2 MW	111	for a in $cross_archs; do
	112	for i in gcc g++ binutils; do
	113	cross_pkgs="$cross_pkgs $i-$a"
	114	done
	115	done
	116	cross_pkgs=$(echo $cross_pkgs)
	117
	118	## Make an enormous shopping list of paths.
	119	##
	120	## The `wanted' list consists of two kinds of items: an absolute path names a
	121	## prefix (not necessarily a directory name) to be attached to the following
	122	## relative names, up to the end of the list or the next absolute path.
	123	wanted="
	124	/usr/bin/ apt apt-cache apt-config apt-get apt-key apt-mark
	125	/usr/lib/apt/ methods/ solvers/
	126
	127	/bin/ cat chgrp chown cp date dd df dir echo false ln ls mkdir
	128	mknod mktemp mv pwd readlink rm rmdir sleep stty sync touch
	129	true uname vdir
	130	/usr/bin/ [ arch b2sum base32 base64 basename chcon cksum comm
	131	csplit cut dircolors dirname du env expand expr factor fmt
	132	fold groups head hostid id install join link logname md5sum
	133	mkfifo nice nl nohup nproc numfmt od paste pathchk pinky pr
	134	printenv printf ptx realpath runcon seq sha1sum sha224sum
	135	sha256sum sha384sum sha512sum shred shuf sort split stat
	136	stdbuf sum tac tail tee test timeout tr truncate tsort tty
	137	unexpand uniq unlink users wc who whoami yes
	138	/usr/lib/$mymulti/ coreutils/
	139
	140	/lib/$mymulti/ libnss_.so.
	141
6b608a0f	142	/usr/bin/ gpgv
3e5b03e2 MW	143
	144	/usr/bin/ qemu-*-static
	145
	146	/bin/ bash dash gzip sed tar
	147	/usr/bin/ ccache find m4 make mawk xargs xz
	148	/usr/lib/$mymulti/ libeatmydata.so* libfakeroot/
	149
	150	/etc/ld.so.conf.d/ $mymulti.conf fakeroot*.conf"
	151
	152	for a in $cross_archs; do
	153	wanted="$wanted
	154
	155	/usr/bin/$a- addr2line ar as c++filt dwp elfedit gprof ld ld.*
	156	nm objcopy objdump ranlib readelf size strings strip
	157
	158	/usr/bin/$a- cpp gcc g++ gcov gcov-dump gcov-tool gprof
	159	gcc-ar gcc-nm gcc-ranlib
	160	/usr/lib/gcc-cross/$a/ ..."
	161	done
	162	wanted=$(echo $wanted)
	163
	164	## Figure out how to recognize dynamic executables.
	165	case $myarch in
	166	i386) elfsig=7f454c46010101??0000000000000000????0300 ;;
	167	amd64) elfsig=7f454c46020101??0000000000000000????3e00 ;;
	168	*) echo >&2 "$0: unsupported local arch \`$myarch'"; exit 2 ;;
	169	esac
	170
	171	## Open a session to the donor chroot.
	172	echo >&2 "$0: create $d snapshot"
	173	sess=$(schroot -bc$LVPREFIX$d-$myarch 3>&-)
	174
	175	## Make sure the donor tree is up-to-date, and install the extra packages we
	176	## need.
	177	schroot -uroot -rc$sess -- eatmydata sh -ec "
	178	apt-get update
	179	apt-get -y upgrade
	180	apt-get -y install $cross_pkgs"
	181
	182	## Establish some pathnames. Prepare a place for our cross-tools tree.
	183	crossdir=$LOCAL/cross/$d-$myarch/
	184	crossold=${crossdir%/}.old/ crossnew=${crossdir%/}.new/
	185	root=/schroot/$sess/fs
	186	rm -rf $crossnew; mkdir -p $crossnew
	187
	188	## Work through the shopping list, copying the things it names into the
	189	## cross-tools tree.
	190	dir=/
	191	for i in $wanted; do
	192	case $i in
	193	/*)
	194	dir=$i
	195	;;
	196	*)
	197	case $i in ...) f=$dir ;; *) f=$dir$i ;; esac
	198	echo >&2 "$0: copy $f"
	199	rsync -aHR $root/.$f $crossnew
	200	;;
	201	esac
	202	done
	203
	204	## Chase links in the new tree, copying extra stuff that we'll need.
	205	find $crossnew -xtype l -print \| while read i; do
	206	chase_link ${i#$crossnew}
207	done
208
209	## Search the new tree for ELF binaries, and build a list of them in
210	## `QUEUE.in'.
211	find $crossnew -type f -print \| while read i; do
212	sig=$(head -c20 "$i" \| bincode -e -m0 -flowerc hex)
213	case $sig in $elfsig) echo "$i" ;; esac
214	done >$root/private/QUEUE.in
215
216	while [ -s $root/private/QUEUE.in ]; do
217	## Work through the ELF binaries in `QUEUE.in', determining which shared
218	## libraries they'll need. Write the list of dependencies to `QUEUE.out'
219	schroot -uroot -rc$sess -- eatmydata sh -ec '
220	prog=$1
221	while read i; do
222	echo >&2 "$prog: scanning binary $i"
223	ldd "$i" \| while read a b c d; do
224	case $a:$b:$c:$d in
225	not:a:dynamic:executable) ;;
226	statically:linked::) ;;
227	/*) echo "$a" ;;
228	:=\>:/) echo "$c" ;;
229	linux-*) ;;
230	*) echo >&2 "$i: unable to find $a"; exit 2 ;;
231	esac
232	done
233	done </private/QUEUE.in >/private/QUEUE.out' - "$0"
234
235	## Work through the shared libraries in `QUEUE.out', copying them to the
236	## cross-tools tree if they're not there already. Add the new ones to a
237	## new `QUEUE.in' file to scan them in turn.
238	while read i; do
239	if [ -e "$crossnew$i" ] \|\| [ -L "$crossnew$i" ]
240	then continue; fi
241	if [ -d "$root$i" ]; then continue; fi
242	echo >&2 "$0: copy $i"
243	rsync -aHR $root/.$i $crossnew >&3
244	chase_link $i >&3
245	sig=$(head -c20 $crossnew$i \| bincode -e -m0 -flowerc hex)
246	case $sig in $elfsig) echo "$i" ;; esac
247	done <$root/private/QUEUE.out >$root/private/QUEUE.in
248	done
249
250	## Set up the cross-compiler. This is rather hairy.
251	echo >&2 "$0: establish TOOLCHAIN"
252	for a in $cross_archs; do
253	tooldir=$crossnew/TOOLCHAIN/$a
254	mkdir -p $tooldir
255	for i in $crossnew/usr/bin/$a-*; do
256	t=${i#$crossnew/usr/bin/}
257	mv $i $tooldir/$t
258	ln -s $t $tooldir/${t#$a-}
259	done
260	done
261	mkdir $crossnew/TOOLCHAIN/lib
262	ln -s ../../usr/lib/gcc-cross $crossnew/TOOLCHAIN/lib/
263
264	## Set up the emulator.
265	echo >&2 "$0: establish QEMU"
266	mkdir $crossnew/QEMU
267	mv $crossnew/usr/bin/qemu-*-static $crossnew/QEMU/
268
269	## We're done. Remove the snapshot, and replace the old cross-tools tree
270	## with our new one.
271	echo >&2 "$0: remove snapshot"
272	schroot -ec$sess 3>&-
273	if [ -d $crossdir ]; then mv $crossdir $crossold; fi
274	mv $crossnew $crossdir; rm -rf $crossold
275	echo >&2 "$0: committed $crossdir"
276
277	###----- That's all, folkd --------------------------------------------------