X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ian/git?p=xfonts-traditional.git;a=blobdiff_plain;f=update-xfonts-traditional;h=494d637969ddcf53db31e94c9201ccc45d3518c0;hp=df71fce7628331f0389e0e886ade5d4c360bd1bd;hb=fbafd43c60babe5532e3b18ee5d1abdd412fbee9;hpb=be2a62a61efbfa8beab7807b4de28c250d1054f6;ds=sidebyside diff --git a/update-xfonts-traditional b/update-xfonts-traditional index df71fce..494d637 100755 --- a/update-xfonts-traditional +++ b/update-xfonts-traditional @@ -7,6 +7,7 @@ use File::Glob qw(:glob); use Data::Dumper; use IO::Pipe; use File::Find; +use Sys::CPU; our $prefix="/usr/local"; our $package='xfonts-traditional'; @@ -15,17 +16,27 @@ our @fonttrees=qw(/usr/share/fonts/X11 /usr/local/share/fonts/X11); our $donefile="$package.done"; our $logfile="$package.log"; our $fontprefix="trad--"; +our $bdfnorm="bdfnorm"; our @rulespath; our $mode; our %foundrymap; our $verbose=0; our $reportfh; +our $foundryinfo; +our %props; +our $tolerate_bad_fonts=1; +our $always_reprocess; +our $wanted_parallel; sub reportloaded { return unless $verbose; print $reportfh @_,"\n" or die $!; } +sub statsummary () { + return join ' ', ((stat _)[1,7,9,10]); +} + sub loadrules ($) { my ($key) = @_; our %cache; @@ -45,19 +56,22 @@ sub loadrules ($) { } sub processbdf ($$$$) { - my ($inbdf,$outbdf,$logfile,$what) = @_; + my ($inbdf,$outbdf,$logfh,$what) = @_; my $state='idle'; my ($foundry,$font); my ($w,$h,$xo,$yo,$y,$bitmap,$glyph); my $modified=0; + %props = (); + my $anyinput=0; while (<$inbdf>) { + $anyinput=1; if ($state eq 'bitmap' && $y==$h) { $glyph = uc $glyph; $glyph =~ s/\;$//; local ($_) = $glyph; my $key= sprintf "%s,%d,%d,%d,%d", $foundry,$w,$h,$xo,$yo; my $rules= loadrules($key); - return (0,'no rules') if !$rules; + return 'no rules' if !$rules; $rules->(); $modified += ($_ ne $glyph); print $outbdf $_,"\n" or die $! @@ -73,25 +87,29 @@ sub processbdf ($$$$) { } if ($state eq 'idle' && m/^FOUNDRY\s+/) { die if defined $foundry; - return (0,'foundry syntax') unless m/^FOUNDRY\s+\"(\w+)\"\s+/; + return 'foundry syntax' unless m/^FOUNDRY\s+\"(\w+)\"\s+/; $foundry = $foundrymap{lc $1}; - return (0,'no foundry') unless defined $foundry; + return 'no foundry' unless defined $foundry; $_ = "FOUNDRY \"$foundry\"\n"; } if ($state eq 'idle' && m/^FONT\s+/) { die if defined $font; - return (0,'simple font name') unless m/^(FONT\s+)\-(\w+)\-/; + return 'simple font name' unless m/^(FONT\s+)\-(\w+)\-/; $font = $foundrymap{lc $2}; - return (0,'no foundry') unless defined $font; + return 'no foundry' unless defined $font; $_ = "FONT -$font-$'"; } if ($state eq 'idle' && m/^STARTCHAR\s/) { die unless defined $foundry; die unless defined $font; - return (0,'foundry != font') unless $foundry eq $font; + return 'foundry != font' unless $foundry eq $font; $state='startchar'; $w=undef; } + if (($state eq 'idle' || $state eq 'startchar') && + m/^([A-Z_]+)\s+(.*\S)\s+$/) { + $props{$1}=$2; + } if ($state eq 'startchar') { if (m/^BBX\s+(\+?\d+)\s+(\+?\d+)\s+([-+]?\d+)\s+([-+]?\d+)\s+$/) { ($w,$h,$xo,$yo) = ($1,$2,$3,$4); @@ -101,6 +119,9 @@ sub processbdf ($$$$) { $y=0; $glyph=''; $state='bitmap'; + $props{' 7bit'}= + ($props{'CHARSET_REGISTRY'} =~ m/iso8859|utf|iso10646/i && + $props{'ENCODING'} <= 127); } } print $outbdf $_ or die $!; @@ -108,18 +129,27 @@ sub processbdf ($$$$) { die $! if $inbdf->error; die $! if $outbdf->error or !$outbdf->flush; die unless $state eq 'idle'; + return 'no bdf data' # also special cased in processpcfgz + if !$anyinput; if ($modified) { - printf $logfile "%s: %d glyphs changed\n", $what, $modified + printf $logfh "%s: %d glyphs changed\n", $what, $modified or die $!; } else { - printf $logfile "%s: unchanged - no rules matched\n", $what + printf $logfh "%s: unchanged - no rules matched\n", $what or die $!; } return $modified; } sub loadfoundries () { + $foundryinfo = ''; foreach my $path (@rulespath) { + if (!stat $path) { + die "$path $!" unless $!==&ENOENT; + next; + } + $foundryinfo .= statsummary().' '.$path."\0\n"; + my $p = "$path/foundries"; my $f = new IO::File $p; if (!$f) { @@ -127,10 +157,11 @@ sub loadfoundries () { print $reportfh "foundries: none in $p\n" or die $! if $verbose; next; } + stat $f or die $!; while (<$f>) { s/^\s*//; s/\s+$//; next if m/^\#/; - m/^(\w+)\s+(\w+)$/ or die; + m/^([.0-9A-Za-z]+)\s+([.0-9A-Za-z]+)$/ or die; my $k = lc $1; next if exists $foundrymap{$k}; $foundrymap{$k}=$2; @@ -141,15 +172,24 @@ sub loadfoundries () { die "no foundry maps\n" unless %foundrymap; } -sub processpcfgz ($$$$) { - my ($inpcfgz,$outpcfgz,$logfile,$what) = @_; +our %ch; - my $current = new IO::File $inpcfgz, '<' or die "$inpcfgz $!"; - my ($usread,$uswrite); +sub filter_st_isok ($) { + my ($ch) = @_; + my $st = $ch->{St}; + return !$st || $ch->{SigOK}{($st & ~128)}; +} + +sub process_filter ($$$$$$$$) { + my ($rr, $input, $output,$what,$logfh, + $procs, $sigpipeok, $after_hook) = @_; my ($reader,$writer); my @children; - foreach my $proc (['gunzip'], ['pcf2bdf'], [], - ['bdftopcf'],['',qw(gzip -1)]) { + my ($usread,$uswrite); + + my $current = $input; + + foreach my $proc (@$procs) { my $isfinal = (@$proc && $proc->[0] eq ''); if (!$isfinal) { $reader = new IO::Handle or die $!; @@ -158,7 +198,7 @@ sub processpcfgz ($$$$) { } else { shift @$proc; $reader = undef; - $writer = new IO::File $outpcfgz, '>' or die "$outpcfgz $!"; + $writer = $output; } if (@$proc) { my $exe = $proc->[0]; @@ -173,7 +213,14 @@ sub processpcfgz ($$$$) { close $uswrite or die $! if $uswrite; exec $exe @$proc or die "$exe $!"; } - push @children, [ $child, $exe, defined $usread ]; + my $ch = { + Pid => $child, + Exe => $exe, + Stage => (!$exe ? 'self' : defined $usread ? 'out' : 'in'), + SigOK => { }, + }; + push @children, $ch; + $ch{$exe} = $ch; close $current or die $!; close $writer or die $!; $current = $reader; @@ -183,28 +230,85 @@ sub processpcfgz ($$$$) { $current = $reader; } } - my $r = processbdf($usread,$uswrite,$logfile,$what); - my $none = $r !~ m/^\d/; - if ($none) { + $$rr = processbdf($usread,$uswrite,$logfh,$what); + my $none = $$rr !~ m/^\d/; + + $ch{$_}{SigOK}{13} = 1 foreach @$sigpipeok; + + if ($none || !$$rr) { + # We're not going to install or use this so we can kill our + # input and output filters. We kill the input filters so that + # we don't risk waiting for them. (If the input filter died + # for some other reason then sending it a KILL now won't + # affect its exit status.) We kill the output filters (before + # we close the output pipe) so we don't produce messages from + # our output filters about corrupted data. And we tolerate + # SIGPIPE in all the input filters. flush $uswrite or die $!; - } else { - close $uswrite or die $!; - } - close $usread or die $!; - foreach my $chinfo (@children) { - my ($child,$exe,$isoutput)=@$chinfo; - my $sigok = 0; - if ($none) { - if ($isoutput) { - $sigok = 9; - kill 9, $child or die "$child $!"; + + my $filterkind = 'input'; + foreach my $ch (@children) { + if ($ch->{Stage} ne 'self') { + kill 9, $ch->{Pid} or die "$ch->{Pid} $ch->{Exe} $!"; + $ch->{SigOK}{9} = 1; } else { - $sigok = 13; + $filterkind = 'output'; + next; } + if ($filterkind eq 'input') { + $ch->{SigOK}{13} = 1; + } + } + # ... we might not have read all the output from pcf2bdf, which is OK + } + + close $uswrite or die $!; + close $usread or die $!; + + foreach my $ch (@children) { + $!=0; waitpid($ch->{Pid}, 0) == $ch->{Pid} or + die "$ch->{Pid} $ch->{Exe} $!"; + $ch->{St} = $?; + } + + $after_hook->(); + + foreach my $ch (@children) { + if (!filter_st_isok($ch)) { + die "update-xfonts-traditional:". + " $ch->{Exe} [$ch->{Pid}] for $what". + " failed $ch->{St}". + " (".(join ' ', keys %{ $ch->{SigOK} })." ok)\n"; } - $!=0; waitpid($child, 0) == $child or die "$child $!"; - !$? or ($?&~128)==$sigok or die "$exe [$child] $sigok $?"; } +} + +sub processpcfgz ($$$$) { + my ($inpcfgz,$outpcfgz,$logfh,$what) = @_; + print $reportfh "processing $inpcfgz to $outpcfgz\n" if $verbose>=2; + my $input = new IO::File $inpcfgz, '<' or die "$inpcfgz $!"; + my $output = new IO::File $outpcfgz, '>' or die "$outpcfgz $!"; + + my $r; + process_filter(\$r, $input, $output, $inpcfgz, $logfh, + [ + ['gunzip'], ['pcf2bdf'], [$bdfnorm], + [], + ['bdftopcf'],['',qw(gzip -1 -n)] + ], + [qw(gunzip)], + # ... we never care if pcf2bdf didn't want all the data + sub { + if ($tolerate_bad_fonts && + $r eq 'no bdf data' && + filter_st_isok($ch{'gunzip'}) && + ($ch{'pcf2bdf'}{St} & ~128) == 6) + { + $r = "pcf2bdf failed ($ch{'pcf2bdf'}{St})"; + print STDERR "warning: $r: skipping $inpcfgz\n"; + $ch{'pcf2bdf'}{SigOK}{6} = 1; + } + }); return $r; } @@ -215,20 +319,62 @@ sub processfontdir ($) { return; } my $changed = 0; - my $olddone = do "$fontdir/$donefile"; + my $olddone; + if (!$always_reprocess) { + $olddone = do "$fontdir/$donefile"; + if (!$olddone) { + die "$fontdir $! $@ " unless $!==&ENOENT; + } elsif ($olddone->{''} ne $foundryinfo) { + our $repro_reported; + print $reportfh "reprocessing fonts (rules updated)\n" or die $! + unless $repro_reported++; + $olddone = undef; + } + } if (!$olddone) { - die "$fontdir $! $@ " unless $!==&ENOENT; $olddone = { }; $changed = 1; } - my $newdone = { }; - my $log = new IO::File "$fontdir/$logfile", "w" - or die "$fontdir/$logfile $!"; + my $newdone = { '' => $foundryinfo }; my %outfiles; # bitmask: 1 /*exists*/ | 2 /*wanted*/ my $updated=0; my $reported=0; my $anypcfs=0; + my $logpath = "$fontdir/$logfile"; + unlink "$logpath" or $!==&ENOENT or die "$logpath $!"; + my $logfh = new IO::File $logpath, ">>" or die "$logpath $!"; + + if (!$wanted_parallel) { + $wanted_parallel = Sys::CPU::cpu_count(); + printf $reportfh "parallelism: %d\n", $wanted_parallel if $verbose>=2; + } + $wanted_parallel = 1 if $wanted_parallel < 1; + + our %inprogress; + + my $await = sub { + my $child = wait; + die $! unless defined $child; + my $job = $inprogress{$child}; + die $child unless $job; + + my $dent = $job->{Dent}; + my $outdent = $job->{Outdent}; + my $stats = $job->{Stats}; + if ($?==0) { + $updated++; + $outfiles{$outdent} |= 3; + } elsif ($?==2*256) { + } else { + die "update-xfonts-traditional: processing of". + " $fontdir/$dent [$child] failed ($?)\n"; + } + $newdone->{$dent} = $stats; + $changed = 1; + delete $inprogress{$child}; + }; + flush $reportfh or die $!; while (my $dent = scalar readdir FD) { if ($dent =~ m/^\Q$fontprefix\E.*\.new$/) { @@ -249,7 +395,7 @@ sub processfontdir ($) { die "$fontdir $dent" unless -f _; $anypcfs++; - my $stats = join ' ', ((stat _)[1,7,9,10]); + my $stats = statsummary(); my $tdone = $olddone->{$dent}; my $outdent = $fontprefix.$dent; if (defined $tdone && $tdone eq $stats) { @@ -258,21 +404,34 @@ sub processfontdir ($) { next; } - my $r = processpcfgz("$fontdir/$dent", - "$fontdir/$outdent.new", - $log, $dent); - if ($r !~ m/^\d/) { - printf $log "%s: unchanged - %s\n", $dent, $r; - unlink "$fontdir/$outdent.new" or die "$fontdir $outdent $!"; - } else { - rename "$fontdir/$outdent.new", "$fontdir/$outdent" - or die "$fontdir $outdent $!"; - $updated++; - $outfiles{$outdent} |= 3; + $await->() while scalar keys %inprogress >= $wanted_parallel; + + my $child = fork; die unless defined $child; + if (!$child) { + my $r = processpcfgz("$fontdir/$dent", + "$fontdir/$outdent.new", + $logfh, $dent); + my $rc; + if ($r !~ m/^\d/) { + printf $logfh "%s: unchanged - %s\n", $dent, $r; + unlink "$fontdir/$outdent.new" or die "$fontdir $outdent $!"; + $rc = 2; + } else { + rename "$fontdir/$outdent.new", "$fontdir/$outdent" + or die "$fontdir $outdent $!"; + $rc = 0; + } + $logfh->flush or die "$logpath $!"; + exit $rc; } - $newdone->{$dent} = $stats; - $changed = 1; + $inprogress{$child} = { + Dent => $dent, + Outdent => $outdent, + Stats => $stats, + }; } + $await->() while scalar keys %inprogress; + my $affected=0; foreach my $olddent (keys %outfiles) { my $state = $outfiles{$olddent}; @@ -291,7 +450,7 @@ sub processfontdir ($) { die "$fontdir $? $!" if $? or $!; } if (!$anypcfs) { - unlink "$fontdir/$logfile" or die "$fontdir $!"; + unlink "$logpath" or die "$fontdir $!"; unlink "$fontdir/$donefile" or $!==&ENOENT or die "$fontdir $!"; } elsif ($changed) { my $newdoneh = new IO::File "$fontdir/$donefile.new", 'w' @@ -328,6 +487,9 @@ our (@options)=( 'R|rules-include=s@' => \@rulespath, 'share-dir=s' => \$sharedir, 'verbose|v+' => \$verbose, + 'j|parallel=i' => \$wanted_parallel, + 'always-reprocess!' => \$always_reprocess, + 'tolerate-bad-fonts!' => \$tolerate_bad_fonts, ); sub define_mode ($$) { @@ -387,6 +549,7 @@ Getopt::Long::Configure(qw(bundling)); GetOptions(@options) or exit 127; push @rulespath, "$sharedir/rules"; +$bdfnorm = "$sharedir/$bdfnorm"; die "need a mode\n" unless $mode;