X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=blobdiff_plain;f=nailing-cargo;h=a4d2d4a177579bc1fd2ba994b9f701291499f70a;hb=d853cf6dfe15770c1adb77e2223d20ba16291412;hp=cfb97f3d8bc028d260fe6c3a8b1dbdc3bdc5498f;hpb=f5a1481ebbeedd95edf3821c7e0ee0be539c8a54;p=nailing-cargo.git diff --git a/nailing-cargo b/nailing-cargo index cfb97f3..a4d2d4a 100755 --- a/nailing-cargo +++ b/nailing-cargo @@ -18,9 +18,8 @@ # along with this program. If not, see . # example usages: -# ../nailing-cargo/nailing-caretwgo make +# ../nailing-cargo/nailing-cargo make # ../nailing-cargo/nailing-cargo cargo build -# CARGO='../nailing-cargo/nailing-cargo cargo' make # Why do we need this ? # @@ -28,16 +27,158 @@ # https://stackoverflow.com/questions/33025887/how-to-use-a-local-unpublished-crate # https://github.com/rust-lang/cargo/issues/1481 -# Needs libtoml-perl - -#: Cargo.nail: +# Options: +# -v Increase verbosity. Default is 1. +# -q Set verbosity ot 0. +# -D Increase amount of debugging dump. +# +# -n "No action": stop after writing Cargo.toml.nailing~ +# everywhere, and do not run any build command. +# +# -T --target= +# Specify target architecture. If starts with a +# capital ascii letter, is an alias for some other arch +# looked up in Cargo.nail and then in the builtin list: +# RPI arm-unknown-linux-gnueabihf +# Translates to a --target= option to the ultimate command, +# unless that is a cargo subcommand which would reject it. +# +# -u | --cargo-lock-update +# -U | --no-cargo-lock-update +# Arranges to do a dance to allow Cargo.lock (or +# alternative) to be updated in the source directory. +# +# The Cargo.lock and Cargo.toml are copied to the build +# directory along with a skeleton just big enough to fool +# cargo. After cargo has run, the resulting Cargo.lock is +# copied back to the source tree. +# +# Makes no sense with in-tree builds. +# +# Default is no update unless the ultimate command is a +# cargo subcommand which we know needs it. +# +# -m | --cargo-manifest-args +# -M | --no-cargo-manifest-args +# Controls whether we add cargo command line options +# relating to finding Cargo.toml, to the command to +# run. +# +# Default is true if we are doing an out-of- tree build, +# unless we are updating the Cargo.lock (in which case the +# only relevant files are to be found in the build directory). +# +# The arguments are +# --manifest-path= +# --locked +# --target-dir=target +# +# -T | --no-cargo-target-arg +# -t | --cargo-target-arg +# Suppress --target (or un-suppress it). Only useful with -m. +# Done automatically when nailing-cargo sees that the cargo +# subcommand is one which needs it, eg `fetch'. +# +# --online | -o +# --offline | -O +# Whether to allow cargo to make network access. +# (nailing-cargo always passes --offline to cargo, unless +# --online is in force). The default is offline, +# unless the cargo subcommand is one which implies +# online (currently, `fetch'). # +# Cargo.nail: +# +# # Adds each to the list of directories whose +# # Cargo.toml is to be nailed, and also arranges to use +# # the package found there for other directories if +# # no other source of that package is evident in Cargo.nail. +# subdirs = [ +# "" +# ... +# ] +# or: +# subdirs = """ +# "" +# ... +# """ +# +# # Adds to the list of directories whose Cargo.toml +# # is to be nailed, and overrides any other nailing for # [packages] -# package = subdir -# package = { subdir = ... } +# = +# = { = ... } +# +# Configuration is read from +# +# /etc/nailing-cargo/cfg.toml +# ~/.nailing-cargo.toml +# ./.nailing-cargo.toml +# ../Nailing-Cargo.toml +# ../Cargo.nail +# +# To control use of alternative Cargo.lock filename: +# [alt_cargolock] +# +# file = true # equivalent to "Cargo.lock.example" +# file = false # disables this feature +# file = "" +# +# force = false # default, uses alt file only if it already exists +# force = true # always uses alt file; creation would make Cargo.lock +# +# (you can also specify just alt_cargo_lock instead of alt_cargo_lock.file) +# +# To enable out of tree builds: +# [oot] +# dir = "" # default is Build, if use is specified +# and then +# use = "really" +# user = "" +# or +# use = "ssh" +# user = "@host" # NB must still share a filesystem! +# or +# use = "command_args" +# command = ["", "", "nice"] +# or +# use = "command_sh" +# command = ["", "", "sh -c"] +# or +# use = "null" +# +# Other settings: +# [misc] +# online = true # forces default to be --online +# online = false # forces default to be --offline +# +# Limitations: +# +# Always dirties everyone's Cargo.toml, but tries to put them +# back (if not, running it again should fix it). Cannnot be +# fixed without changes to cargo. # -# [subdirs] -# subdir +# Out of tree builds require a unified filesystem view: eg, +# different users on the same host, NFS, or something. This +# could be improved. +# +# Alternative Cargo.lock file must currently be a leafname. +# I think this just involves review to check other values work. +# +# Alternative Cargo.lock file must be on smae filesystem. +# This is not so easy; we would want the existing algorithm but +# a fallback for this case. +# +# Cargo.nail unconditionally looked for in .. +# Ideally should be configurable, and also perhaps be able +# to combine multiple Cargo.nail files ? +# +# Env vars we pass to the command: +# NAILINGCARGO_WORKSPHERE absolute path of invocation .. +# NAILINGCARGO_MANIFEST_DIR absolute path of invocation . +# NAILINGCARGO_BUILDSPHERE only if out of tree: abs parent of build dir +# NAILINGCARGO_BUILD_DIR absolute path of build dir (even if = src) + our $self; @@ -46,7 +187,7 @@ use POSIX; use Types::Serialiser; our %archmap = ( -# RPI => 'arm-unknown-linux-gnueabihf', xxx + RPI => 'arm-unknown-linux-gnueabihf', ); BEGIN { @@ -82,6 +223,9 @@ our $lockfile = "../.nailing-cargo.lock"; our $cargo_lock_update; our $cargo_manifest_args; +our $cargo_target_arg=1; +our $alt_cargo_lock; +our $online; our @configs; our $verbose=1; @@ -99,9 +243,17 @@ sub read_or_enoent ($) { $r; } +sub stat_exists ($$) { + my ($fn, $what) = @_; + if (stat $fn) { return 1; } + $!==ENOENT or die "$self: stat $what: $fn: $!\n"; + return 0; +} + sub toml_or_enoent ($$) { my ($f,$what) = @_; my $toml = read_or_enoent($f) // return; + print STDERR "Read TOML from $f\n" if $dump; my ($v,$e) = from_toml($toml); if (!defined $v) { chomp $e; @@ -118,23 +270,20 @@ sub load1config ($) { } sub loadconfigs () { - my $cfgleaf = ".nailing-cargo-cfg.toml"; + my $dotfile = ".nailing-cargo.toml"; + load1config("../Nailing-Cargo.toml"); + load1config($dotfile); + load1config("$ENV{HOME}/$dotfile") if defined $ENV{HOME}; load1config("/etc/nailing-cargo/cfg.toml"); - load1config("$worksphere/$cfgleaf"); - load1config("$ENV{HOME}/$cfgleaf") if defined $ENV{HOME}; -} - -sub getcfg ($$) { - my ($k, $def) = @_; - foreach my $cfg (@configs) { - my $v = $cfg->{$k}; - return $v if defined $v; - } - return $def; } sub unlink_or_enoent ($) { unlink $_[0] or $!==ENOENT; } +sub same_file ($$) { + my ($x,$y) = @_; + "@$x[0..5]" eq "@$y[0..5]"; +} + sub takelock () { for (;;) { open LOCK, ">", $lockfile or die "$self: open/create $lockfile: $!\n"; @@ -145,7 +294,7 @@ sub takelock () { next if $! == ENOENT; die "$self: stat $lockfile: $!\n"; } - last if "@fstat[0..5]" eq "@stat[0..5]"; + last if same_file(\@fstat,\@stat); } } sub unlock () { @@ -161,16 +310,19 @@ sub badcfg { } sub cfg_uc { - my $v = $nail; - foreach my $k (@_) { - last unless defined $v; - ref($v) eq 'HASH' or badcfg @_, "parent key \`$k' is not a hash"; - $v = $v->{$k}; + foreach my $cfg (@configs) { + my $v = $cfg; + foreach my $k (@_) { + last unless defined $v; + ref($v) eq 'HASH' or badcfg @_, "parent key \`$k' is not a hash"; + $v = $v->{$k}; + } + return $v if defined $v; } - return $v; + return undef; } -sub cfg { +sub cfge { my $exp = shift @_; my $v = cfg_uc @_; my $got = ref($v) || 'scalar'; @@ -181,14 +333,20 @@ sub cfg { sub cfgn { my $exp = shift @_; - (cfg $exp, @_) // badcfg @_, "missing"; + (cfge $exp, @_) // badcfg @_, "missing"; } -sub cfgs { cfg 'scalar', @_ } +sub cfgs { cfge 'scalar', @_ } sub cfgsn { cfgn 'scalar', @_ } +sub cfg_bool { + my $v = cfg_uc @_; + return $v if !defined($v) || Types::Serialiser::is_bool $v; + badcfg @_, "expected boolean"; +} + sub cfgn_list { - my $l = cfg 'ARRAY', @_; + my $l = cfge 'ARRAY', @_; foreach my $x (@$l) { !ref $x or badcfg @_, "list contains non-scalar element"; } @@ -225,6 +383,8 @@ sub readnail () { } die "$e ?" if length $e; + $nail->{subdirs} //= [ ]; + if (!ref $nail->{subdirs}) { $nail->{subdirs} = [ grep /^[^\#]/, @@ -233,6 +393,42 @@ sub readnail () { $nail->{subdirs} ]; } + + unshift @configs, $nail; +} + +our @alt_cargo_lock_stat; + +sub consider_alt_cargo_lock () { + my @ck = qw(alt_cargo_lock); + # User should *either* have Cargo.lock in .gitignore, + # or expect to commit Cargo.lock.example ($alt_cargo_lock) + + $alt_cargo_lock = (cfg_uc @ck); + + my $force = 0; + if (defined($alt_cargo_lock) && ref($alt_cargo_lock) eq 'HASH') { + $force = cfg_bool qw(alt_cargo_lock force); + my @ck = qw(alt_cargo_lock file); + $alt_cargo_lock = cfg_uc @ck; + } + $alt_cargo_lock //= Types::Serialiser::true; + + if (Types::Serialiser::is_bool $alt_cargo_lock) { + if (!$alt_cargo_lock) { $alt_cargo_lock = undef; return; } + $alt_cargo_lock = 'Cargo.lock.example'; + } + + if (ref($alt_cargo_lock) || $alt_cargo_lock =~ m{/}) { + badcfg @ck, "expected boolean, or leafname"; + } + + if (!stat_exists $alt_cargo_lock, "alt_cargo_lock") { + $alt_cargo_lock = undef unless $force; + return; + } + + @alt_cargo_lock_stat = stat _; } our $oot_dir; # oot.dir or "Build" @@ -241,7 +437,9 @@ sub consider_oot () { $oot_dir = cfgs qw(oot dir); my $use = cfgs qw(oot use); unless (defined($oot_dir) || defined($use)) { - $cargo_lock_update//=0; + die "$self: specified --cargo-lock-update but not out-of-tree build!\n" + if $cargo_lock_update; + $cargo_lock_update=0; return; } $oot_dir //= 'Build'; @@ -322,28 +520,40 @@ sub calculate () { } sub addargs () { + $online //= cfg_bool qw(misc online); + if (@ARGV>=2 && - $ARGV[0] =~ m{\bcargo\b} && - $ARGV[1] =~ m/generate-lockfile|update/) { - $cargo_lock_update //= 1; - } else { - $cargo_lock_update //= 0; + $ARGV[0] =~ m{\bcargo\b}) { + if ($ARGV[1] =~ m/^(?:generate-lockfile|update)$/) { + $cargo_lock_update //= 1; + $target = undef; + } + if ($ARGV[1] =~ m/^(?:fetch)$/) { + $cargo_target_arg=0; + $online //= 1; + } } + $cargo_lock_update //= 0; $cargo_manifest_args //= (defined $oot_dir) && !$cargo_lock_update; + $online //= 0; + if ($cargo_manifest_args) { push @ARGV, "--manifest-path=${src_absdir}/Cargo.toml", - qw(--locked --target-dir=target); + qw(--locked); + push @ARGV, qw(--target-dir=target) if $cargo_target_arg; } if (defined $target) { if ($target =~ m{^[A-Z]}) { - $target = (cfg 'arch', $target) // $archmap{$target} + $target = (cfgs 'arch', $target) // $archmap{$target} // die "$self: --target=$target alias specified; not in cfg or map\n"; } push @ARGV, "--target=$target"; } + + push @ARGV, "--offline" unless $online; } our $oot_absdir; @@ -360,21 +570,29 @@ sub oot_massage_cmdline () { my @xargs; if (!$cargo_lock_update) { push @xargs, $build_absdir; - ($pre, $post) = ('cd "$1"; shift;', ''); + ($pre, $post) = ('cd "$1"; shift; ', ''); } else { push @xargs, $oot_absdir, $subdir, $src_absdir; - ($pre, $post) = (<<'END', <<'END'); + $pre = <<'END'; cd "$1"; shift; mkdir -p -- "$1"; cd "$1"; shift; - cp -- "$1"/Cargo.toml "$1"/Cargo.lock .; shift; - mkdir -p src; >src/lib.rs; -END - rm -r src Cargo.toml; + cp -- "$1"/Cargo.toml END - $pre =~ s/^\s+//mg; $pre =~ s/^\s+\n/ /g; - $post =~ s/^\s+//mg; $post =~ s/^\s+\n/ /g; + $pre .= <<'ENDLK' if stat_exists 'Cargo.lock', 'working cargo lockfile'; + "$1"/Cargo.lock +ENDLK + $pre .= <<'ENDCP'; + .; +ENDCP + $pre .= <<'ENDPRE'; + shift; + mkdir -p src; >src/lib.rs; >build.rs +ENDPRE + $post = <<'ENDPOST'; + rm -r src Cargo.toml build.rs; +ENDPOST } - my $addpath = (cfg qw(oot path_add)) // + my $addpath = (cfg_uc qw(oot path_add)) // $use eq 'really' ? Types::Serialiser::true : Types::Serialiser::false; $addpath = !Types::Serialiser::is_bool $addpath ? $addpath : @@ -386,15 +604,17 @@ END export PATH; END } + $pre =~ s/^\s+//mg; $pre =~ s/\s+/ /g; + $post =~ s/^\s+//mg; $post =~ s/\s+/ /g; my $getuser = sub { cfgsn qw(oot user) }; my @command; my $xe = $verbose >= 2 ? 'xe' : 'e'; my $sh_ec = sub { if (!length $post) { - @command = (@_, 'sh',"-${xe}c",$pre.' exec "$@"','--',@xargs); + @command = (@_, 'sh',"-${xe}c",$pre.'exec "$@"','--',@xargs); } else { - @command = (@_, 'sh',"-${xe}c",$pre.' "$@"; '.$post,'--',@xargs); + @command = (@_, 'sh',"-${xe}c",$pre.'"$@"; '.$post,'--',@xargs); } push @command, @ARGV; }; @@ -452,14 +672,42 @@ END { foreach my $mf (keys %manifests) { eval { uninstall1($mf,1); 1; } or warn "$@"; } + eval { unaltcargolock(1); 1; } or warn "$@"; } } +our $cleanup_cargo_lock; sub makebackups () { foreach my $mf (keys %manifests) { link "$mf", "$mf.unnailed" or $!==EEXIST or die "$self: make backup link $mf.unnailed: $!\n"; } + + if (defined($alt_cargo_lock)) { + if (@alt_cargo_lock_stat) { + print STDERR "$self: using alt_cargo_lock `$alt_cargo_lock'..." + if $verbose>=3; + if (link $alt_cargo_lock, 'Cargo.lock') { + print STDERR " linked\n" if $verbose>=3; + } elsif ($! != EEXIST) { + print STDERR "\n" if $verbose>=3; + die "$self: make \`Cargo.lock' available as \`$alt_cargo_lock': $!\n"; + } else { + print STDERR "checking quality." if $verbose>=3; + my @lock_stat = stat 'Cargo.lock' + or die "$self: stat Cargo.lock (for alt check: $!\n"; + same_file(\@alt_cargo_lock_stat, \@lock_stat) + or die +"$self: \`Cargo.lock' and alt file \`$alt_cargo_lock' both exist and are not the same file!\n"; + } + $cleanup_cargo_lock = 1; + } else { + $cleanup_cargo_lock = 1; + # If Cargo.lock exists and alt doesn't, that means either + # that a previous run was interrupted, or that the user has + # messed up. + } + } } sub nailed ($) { @@ -469,7 +717,18 @@ sub nailed ($) { } sub install () { + my @our_unfound_stab = stat_exists('Cargo.toml', 'local Cargo.toml') + ? (stat _) : (); foreach my $mf (keys %manifests) { + if (@our_unfound_stab) { + if (stat_exists $mf, "manifest in to-be-nailed directory") { + my @mf_stab = stat _ ; + if ("@mf_stab[0..1]" eq "@our_unfound_stab[0..1]") { + @our_unfound_stab = (); + } + } + } + my $nailing = "$mf.nailing~"; my $nailed = nailed($mf); my ($use, $rm); @@ -492,6 +751,11 @@ sub install () { unlink_or_enoent $rm or die "$self: remove old $rm: $!\n"; print STDERR "$self: nailed $mf\n" if $verbose>=3; } + + if (@our_unfound_stab) { + print STDERR + "$self: *WARNING* cwd is not in Cargo.nail thbough it has Cargo.toml!\n"; + } } sub invoke () { @@ -511,12 +775,13 @@ sub invoke () { } sub cargo_lock_update_after () { - return unless $cargo_lock_update; - # avoids importing File::Copy and the error handling is about as good - $!=0; $?=0; - my $r= system qw(cp --), "$build_absdir/Cargo.lock", "Cargo.lock"; - die "$self: run cp: $! $?" if $r<0 || $r & 0xff; - die "$self: failed to update local Cargo.lock (wait status $r)\n" if $r; + if ($cargo_lock_update) { + # avoids importing File::Copy and the error handling is about as good + $!=0; $?=0; + my $r= system qw(cp --), "$build_absdir/Cargo.lock", "Cargo.lock"; + die "$self: run cp: $! $?" if $r<0 || $r & 0xff; + die "$self: failed to update local Cargo.lock (wait status $r)\n" if $r; + } } sub uninstall1 ($$) { @@ -526,12 +791,29 @@ sub uninstall1 ($$) { or die "$self: failed to restore: rename $unnailed back to $mf: $!\n"; } +sub unaltcargolock ($) { + my ($enoentok) = @_; + return unless $cleanup_cargo_lock; + die 'internal error!' unless defined $alt_cargo_lock; + + # we ignore $enoentok because we don't know if one was supposed to + # have been created. + + rename('Cargo.lock', $alt_cargo_lock) or $!==ENOENT or die + "$self: cleanup: rename possibly-updated \`Cargo.lock' to \`$alt_cargo_lock': $!\n"; + + unlink 'Cargo.lock' or $!==ENOENT or die + "$self: cleanup: remove \`Cargo.lock' in favour of \`$alt_cargo_lock': $!\n"; + # ^ this also helps clean up the stupid rename() corner case +} + sub uninstall () { foreach my $mf (keys %manifests) { my $nailed = nailed($mf); link $mf, $nailed or die "$self: preserve (link) $mf as $nailed: $!\n"; uninstall1($mf,0); } + unaltcargolock(0); } while (@ARGV && $ARGV[0] =~ m/^-/) { @@ -550,17 +832,27 @@ while (@ARGV && $ARGV[0] =~ m/^-/) { } elsif (s{^-T(.+)}{-}s) { $target = $1; } elsif (s{^-([uU])}{-}) { - $cargo_lock_update= $1=~m/[A-Z]/; + $cargo_lock_update= $1=~m/[a-z]/; } elsif (s{^-([mM])}{-}) { - $cargo_manifest_args= $1=~m/[A-Z]/; + $cargo_manifest_args= $1=~m/[a-z]/; + } elsif (s{^-([tT])}{-}) { + $cargo_target_arg= $1=~m/[a-z]/; + } elsif (s{^-([oO])}{-}) { + $online= $1=~m/[a-z]/; } else { die "$self: unknown short option(s) $_\n"; } } } elsif (s{^--target=}{}) { $target = $_; - } elsif (m{^--no-cargo-lock-update}) { - $cargo_lock_update=0; + } elsif (m{^--(no-)?cargo-lock-update}) { + $cargo_lock_update= !!$1; + } elsif (m{^--(no-)?cargo-manifest-args}) { + $cargo_manifest_args= !!$1; + } elsif (m{^--(no-)?cargo-target-arg}) { + $cargo_target_arg= !!$1; + } elsif (m{^--(on|off)line$}) { + $online = $1 eq 'on'; } else { die "$self: unknown long option $_\n"; } @@ -568,8 +860,10 @@ while (@ARGV && $ARGV[0] =~ m/^-/) { die "$self: need command to run\n" unless @ARGV || $noact; +loadconfigs(); takelock(); readnail(); +consider_alt_cargo_lock(); consider_oot(); readorigs(); calculate(); @@ -598,18 +892,19 @@ $want_uninstall = 1; makebackups(); install(); -printf STDERR "$self: nailed (%s manifests, %s packages)\n", - (scalar keys %manifests), (scalar keys %packagemap) +printf STDERR "$self: nailed (%s manifests, %s packages)%s\n", + (scalar keys %manifests), (scalar keys %packagemap), + (defined($alt_cargo_lock) and ", using `$alt_cargo_lock'") if $verbose; print STDERR "$self: invoking: @display_cmd\n" if $verbose; my $estatus = invoke(); -uninstall(); -$want_uninstall = 1; - cargo_lock_update_after(); +uninstall(); +$want_uninstall = 0; + print STDERR "$self: unnailed. status $estatus.\n" if $verbose; exit $estatus;