update-xfonts-traditional: process fonts in parallel

Exploit multicore systems to speed up the runtime when processing
needs to be done.

Some timing results on a dual-core 1.50GHz Atom N550 laptop:

-j1
  real    0m43.035s
  user    0m43.319s
  sys     0m37.830s

-j2
  real    0m23.113s
  user    0m40.643s
  sys     0m30.486s

-j3
  real    0m21.910s
  user    0m43.707s
  sys     0m30.674s

-j4
  real    0m22.038s
  user    0m44.695s
  sys     0m31.058s

This system has hyperthreading so looks like 4 cores.  Based on this
I have set the number of processing threads to the number of cpus.

diff --git a/update-xfonts-traditional b/update-xfonts-traditional
index 65d1d086c333112bdeb5f0727f2e7a687fd49197..8e04e603196197b4a964a174784d4a7f68c82c95 100755 (executable)
--- a/update-xfonts-traditional
+++ b/update-xfonts-traditional
@@ -7,6 +7,7 @@ use File::Glob qw(:glob);
 use Data::Dumper;
 use IO::Pipe;
 use File::Find;
 use Data::Dumper;
 use IO::Pipe;
 use File::Find;

+use Sys::CPU;

 
 our $prefix="/usr/local";
 our $package='xfonts-traditional';
 
 our $prefix="/usr/local";
 our $package='xfonts-traditional';


@@ -22,6 +23,7 @@ our $verbose=0;
 our $reportfh;
 our $foundryinfo;
 our %props;
 our $reportfh;
 our $foundryinfo;
 our %props;

+our $wanted_parallel;

 
 sub reportloaded {
     return unless $verbose;
 
 sub reportloaded {
     return unless $verbose;


@@ -251,13 +253,44 @@ sub processfontdir ($) {
        $changed = 1;
     }
     my $newdone = { '' => $foundryinfo };
        $changed = 1;
     }
     my $newdone = { '' => $foundryinfo };

-    my $log = new IO::File "$fontdir/$logfile", "w" 
-       or die "$fontdir/$logfile $!";

     my %outfiles; # bitmask: 1 /*exists*/ | 2 /*wanted*/
     my $updated=0;
     my $reported=0;
     my $anypcfs=0;
 
     my %outfiles; # bitmask: 1 /*exists*/ | 2 /*wanted*/
     my $updated=0;
     my $reported=0;
     my $anypcfs=0;
 

+    my $logpath = "$fontdir/$logfile";
+    unlink "$logpath" or $!==&ENOENT or die "$logpath $!";
+    my $log = new IO::File $logpath, ">>" or die "$logpath $!";
+
+    if (!$wanted_parallel) {
+       $wanted_parallel = Sys::CPU::cpu_count();
+       printf $reportfh "parallelism: %d\n", $wanted_parallel if $verbose>=2;
+    }
+    $wanted_parallel = 1 if $wanted_parallel < 1;
+
+    our %inprogress;
+
+    my $await = sub {
+       my $child = wait;
+       die $! unless defined $child;
+       my $job = $inprogress{$child};
+       die $child unless $job;
+
+       my $dent = $job->{Dent};
+       my $outdent = $job->{Outdent};
+       my $stats = $job->{Stats};
+       if ($?==0) {
+           $updated++;
+           $outfiles{$outdent} |= 3;
+       } elsif ($?==2*256) {
+       } else {
+           die "[$child] $dent $outdent $?";
+       }
+       $newdone->{$dent} = $stats;
+       $changed = 1;
+       delete $inprogress{$child};
+    };
+

     flush $reportfh or die $!;
     while (my $dent = scalar readdir FD) {
        if ($dent =~ m/^\Q$fontprefix\E.*\.new$/) {
     flush $reportfh or die $!;
     while (my $dent = scalar readdir FD) {
        if ($dent =~ m/^\Q$fontprefix\E.*\.new$/) {


@@ -287,21 +320,34 @@ sub processfontdir ($) {
            next;
        }
 
            next;
        }
 

-       my $r = processpcfgz("$fontdir/$dent",
-                            "$fontdir/$outdent.new",
-                            $log, $dent);
-       if ($r !~ m/^\d/) {
-           printf $log "%s: unchanged - %s\n", $dent, $r;
-           unlink "$fontdir/$outdent.new" or die "$fontdir $outdent $!";
-       } else {
-           rename "$fontdir/$outdent.new", "$fontdir/$outdent"
-               or die "$fontdir $outdent $!";
-           $updated++;
-           $outfiles{$outdent} |= 3;
+       $await->() while scalar keys %inprogress >= $wanted_parallel;
+
+       my $child = fork;  die unless defined $child;
+       if (!$child) {
+           my $r = processpcfgz("$fontdir/$dent",
+                                "$fontdir/$outdent.new",
+                                $log, $dent);
+           my $rc;
+           if ($r !~ m/^\d/) {
+               printf $log "%s: unchanged - %s\n", $dent, $r;
+               unlink "$fontdir/$outdent.new" or die "$fontdir $outdent $!";
+               $rc = 2;
+           } else {
+               rename "$fontdir/$outdent.new", "$fontdir/$outdent"
+                   or die "$fontdir $outdent $!";
+               $rc = 0;
+           }
+           $log->flush or die "$logpath $!";
+           exit $rc;

        }
        }

-       $newdone->{$dent} = $stats;
-       $changed = 1;
+       $inprogress{$child} = {
+           Dent => $dent,
+           Outdent => $outdent,
+           Stats => $stats,
+       };

     }
     }

+    $await->() while scalar keys %inprogress;
+

     my $affected=0;
     foreach my $olddent (keys %outfiles) {
        my $state = $outfiles{$olddent};
     my $affected=0;
     foreach my $olddent (keys %outfiles) {
        my $state = $outfiles{$olddent};


@@ -320,7 +366,7 @@ sub processfontdir ($) {
        die "$fontdir $? $!" if $? or $!;
     }
     if (!$anypcfs) {
        die "$fontdir $? $!" if $? or $!;
     }
     if (!$anypcfs) {

-       unlink "$fontdir/$logfile" or die "$fontdir $!";
+       unlink "$logpath" or die "$fontdir $!";

        unlink "$fontdir/$donefile" or $!==&ENOENT or die "$fontdir $!";
     } elsif ($changed) {
        my $newdoneh = new IO::File "$fontdir/$donefile.new", 'w' 
        unlink "$fontdir/$donefile" or $!==&ENOENT or die "$fontdir $!";
     } elsif ($changed) {
        my $newdoneh = new IO::File "$fontdir/$donefile.new", 'w' 


@@ -357,6 +403,7 @@ our (@options)=(
     'R|rules-include=s@' => \@rulespath,
     'share-dir=s' => \$sharedir,
     'verbose|v+' => \$verbose,
     'R|rules-include=s@' => \@rulespath,
     'share-dir=s' => \$sharedir,
     'verbose|v+' => \$verbose,

+    'j|parallel=i' => \$wanted_parallel,

     );
 
 sub define_mode ($$) {
     );
 
 sub define_mode ($$) {

diff --git a/update-xfonts-traditional.8 b/update-xfonts-traditional.8
index 4787ee002e0ea38bbaa18c8004a80fd12f168e13..54c5cae5eebabbcdf0374c6e9123743b33e94de6 100644 (file)
--- a/update-xfonts-traditional.8
+++ b/update-xfonts-traditional.8
@@ -55,6 +55,11 @@ Use
 .I share-dir
 instead of
 .BR "/usr[/local]/share/xfonts-traditional" .
 .I share-dir
 instead of
 .BR "/usr[/local]/share/xfonts-traditional" .

+.TP
+\fB--parallel\fP|\fB-j\fP \fIcpus\fP
+Use up to
+.I cpus
+cores for processing fonts.  The default is all the cpus available.

 .SH FILES
 .TP
 .B xfonts-traditional.done
 .SH FILES
 .TP
 .B xfonts-traditional.done