X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=blobdiff_plain;f=i18n-diff-auditor;h=e2c010142d334bc42b439bf8b9c590e7a9c95802;hb=1b5fa179018c14bb6bc8cd9149dee0a4ea7085bc;hp=8fd623a5a67f84036be5a8e5b115bd9d94dc42fe;hpb=8f47fccc111486af01b6f115c8a770eb54217956;p=dgit.git

diff --git a/i18n-diff-auditor b/i18n-diff-auditor
index 8fd623a5..e2c01014 100755
--- a/i18n-diff-auditor
+++ b/i18n-diff-auditor
@@ -4,15 +4,14 @@ use Carp;
 use Data::Dumper;
 use Getopt::Long;
 
-open DEBUG, ">/dev/null" or die $!;
-
-GetOptions("debug|D" => sub { open DEBUG, ">&2" or die $!; }
+our $debug = 0;
+GetOptions("debug|D+" => \$debug
 	   );
 
 our @debug;
 sub debug ($$) {
     my ($i,$s) = @_;
-    push @{ $debug[$i] }, $s;
+    push @{ $debug[$i] }, $s if $debug;
 }
 
 my @d = <>;
@@ -108,11 +107,22 @@ our ($ihunkhead, $ihunkend);
 our ($ichunkstart, $ichunkend);
 our ($before, $after);
 
-sub is_string ($) { $_[0]{T} =~ m/heredoc|string/; };
+sub is_string ($) { $_[0]{T} =~ m/heredoc|string/; }
+sub is_trans ($) { grep { $_[0]{E} eq $_ } qw(__ f_ i_); }
+
+sub qp ($) {
+    my ($p) = @_;
+    $p =~ s{\\}{\\\\}g;
+    $p =~ s{\'}{\\'}g;
+    $p =~ s{\n}{\\n}g;
+    $p =~ s{\t}{\\t}g;
+    return "'$p'";
+};
 
 sub semiparse ($) {
     ($_) = @_;
     my @o;
+    #my $in = $_;
     # entries contain
     #   T     type
     #   E     exact input text (does not contain here doc contents)
@@ -128,25 +138,31 @@ sub semiparse ($) {
 	    $q ||= '"';
 	    push @o, { T => 'heredoc', Q => $q, Delim => $d,
 		       E => $&, P => "<<$q$d$q" };
-	    s{^
-                (             .* \n    )
-                ( (?: (?! $d) .* \n )* )
-                          $d     \n
-              }{ $1 }x or die "missing end of here doc $d\n";
-	    $o[$#o]{V} = $2;
+	    if (s{^
+		    (                 .* \n     )
+		    ( (?: (?! $d \n ) .* \n )*? )
+			      $d         \n
+	         }{ $1 }xe) {
+		$o[$#o]{V} = $2;
+            } else {
+		m{^.*\n} or confess;
+		$_ = $&;
+		$o[$#o]{V} = $';
+		$o[$#o]{Invented} = 1;
+	    }
 	} elsif (s{^ (["'])( (?: [^\\'"]
                                | \\ [^"']
                                | (?! \1 ) [^"]
                               )*
                        ) \1 }{}x) {
 	    my ($q,$v) = ($1,$2);
-	    push @o, { T => 'string', E => $&, P => "$q-string",
+	    push @o, { T => 'string', E => $&, P => "$q$q",
 		       Q => $q, V => $v};
 	} elsif (s{^$perlop_re|^\;}{}) {
 	    push @o, { T => 'op', E => $&, P => $& };
-	} elsif (s/[[{(]//) {
+	} elsif (s/^[[{(]//) {
 	    push @o, { T => 'bra', E => $&, P => $& };
-	} elsif (s/[]})]//) {
+	} elsif (s/^[]})]//) {
 	    push @o, { T => 'ket', E => $&, P => $& };
 	} elsif (s/^( [\$\@\%] )( \{ )//x) {
 	    push @o, { T => 'deref', E => $1, P => $1 },
@@ -160,12 +176,14 @@ sub semiparse ($) {
 	    die "cannot tokenise \`$&'";
 	}
     }
-    for (my $i=@o-2; $i>0; --$i) {
+    for (my $i=0; $i+2 < @o; $i++) {
 	next unless $o[$i+1]{E} eq '.';
 	my @inputs = @o[$i, $i+2];
+	#print STDERR Dumper(\@inputs);
 	next if grep { !is_string($_) } @inputs;
 	my $q = $inputs[0]{Q};
 	next if grep { $_->{Q} ne $q } @inputs;
+	next if grep { $_->{Invented} } @inputs;
 	my $new = { T => 'joinedstrings',
 		    E => (join '.', map { $_->{E} } @inputs),
 		    P => (join '.', map { $_->{P} } @inputs),
@@ -173,36 +191,68 @@ sub semiparse ($) {
 		    Q => $q,
 		  };
 	@o = (@o[0..$i-1], $new, @o[$i+3..$#o]);
-	print STDERR Dumper(\@o);
+	$i--; # counteracts $i++
     }
     debug $ichunkstart, "semiparsed: ".join ' ', map { $_->{P} } @o;
+    # debug $ichunkstart, "semiparsed V: ".join ' ', map { defined $_->{V} ? ">$_->{V}<" : '-' } @o;
     return @o;
 }	    
 
+our @analysed_x;
+our @analysed_y;
+
 sub analyse_chunk_core () {
     die "plain deletion\n" unless defined $after;
     die "plain insertion\n" unless defined $before;
     my @xs = semiparse $before;
     my @ys = semiparse $after;
+    @analysed_x = @analysed_y = ();
     my $next_something = sub {
-	my ($ary,$var,$what) = @_;
+	my ($ary,$anal,$var,$what) = @_;
 	die "ran out of $what\n" unless @$ary;
-	$$var = shift @$ary;
+	my $r = shift @$ary;
+	push @$anal, $r->{P};
+	$$var = $r;
     };
     my ($x,$y);
-    my $next_x = sub { $next_something->(\@xs, \$x, 'before'); };
-    my $next_y = sub { $next_something->(\@ys, \$y, 'after' ); };
+    my $next_x = sub { $next_something->(\@xs, \@analysed_x, \$x, 'before'); };
+    my $next_y = sub { $next_something->(\@ys, \@analysed_y, \$y, 'after' ); };
+    our @y_expect_suffix = ();
     for (;;) {
+	while (my $e = shift @y_expect_suffix) {
+	    $next_y->();
+	    $y->{E} eq $e
+		or die "suffix mismatch, expected $e got $y->{E}\n";
+	}
 	last unless @xs or @ys;
 	$next_x->();
 	$next_y->();
 	next if $x->{E} eq $y->{E};
+	next if $x->{E} eq 'sprintf' and $y->{E} eq 'f_';
+	next if $x->{E} eq 'die'     and $y->{E} eq 'confess';
+	next if $x->{E} eq 'die'     and $y->{E} eq 'fail';
+	if ($y->{E} eq '+'
+	    and @ys >= 3
+	    and $ys[0]{E} eq '('
+	    and is_trans($ys[1])) {
+	    $next_y->(); # (
+	    $next_y->(); # __ f_ i_
+	    @y_expect_suffix = ')';
+	} elsif ($y->{E} eq '('
+	    and @ys > 2
+	    and is_trans($ys[0])
+	    and @analysed_y
+	    and (grep { $_ eq $analysed_y[-1] } (qw( => [ { ? : . ),
+						 '(', ',') )) {
+	    $next_y->(); # __ f_ i_
+	    @y_expect_suffix = ')';
+	}
 	my $string_changed;
 	my $ye = $y->{E};
-	if ($ye eq '__' or $ye eq 'f_') {
+	if (is_trans($y)) {
 	    $next_y->();
 	    die "__ on non-string $y->{P}\n"     unless is_string($y);
-	    die "__ on was non-string $y->{P}\n" unless is_string($x);
+	    die "__ on was non-string $x->{P}\n" unless is_string($x);
 	    if ($y->{Q} ne "'") {
 		die "var subst in new string\n"
 		    if $y->{V} =~ m{(?<!\\) [\$\@]};
@@ -219,6 +269,10 @@ sub analyse_chunk_core () {
 	    die $string_changed if length $string_changed;
 	    next;
 	}
+	if ($ye eq 'i_') {
+	    die $string_changed if length $string_changed;
+	    next;
+	}
 	if ($ye eq 'f_') {
 	    my $fmt = $y->{V};
 	    die "no percent in _f string\n" unless $fmt =~ m{\%};
@@ -229,17 +283,20 @@ sub analyse_chunk_core () {
 		my ($lit, $what) = @_;
 		my $xl = substr($xs, 0, length($lit));
 		if ($xl ne $lit) {
-		    debug $ichunkstart, "not exactly x: $xl";
-		    debug $ichunkstart, "not exactly y: $lit";
+		    debug $ichunkstart, "not exactly x: ..".qp($xs);
+		    debug $ichunkstart, "not exactly y:   ".qp($lit);
 		    my $next = @ys ? $ys[0]{P} : '(end)';
 		    die "string contents mismatch near $what before $next\n";
 		}
 		$xs = substr($xs, length($lit));
 	    };
 	    for (;;) {
+		#print STDERR Dumper($fmt, $xs, \@xs, @ys);
 		if ($fmt !~ m{\%[^\%]}) {
 		    $exactly->($fmt, '(tail)');
 		    $fmt = '';
+		    die "text deleted from end of string: ".qp($xs)."\n"
+			if length $xs;
 		    last;
 		}
 		$exactly->($`, '(literal)');
@@ -248,6 +305,15 @@ sub analyse_chunk_core () {
 		elsif ($& ne '%s') { die "unhandled %-subst $&\n"; }
 		$next_y->();
 		die "expected comma, got $y->{P}\n" unless $y->{E} eq ',';
+		if (!length $fmt and
+		    !length $xs and
+		    @xs and
+		    $xs[0]{E} eq '.') {
+		    # X has   "<earlier>" .                <something>
+		    # Y has   "<earlier>%s" [other args] , <something>
+		    $next_x->(); # eat the '.'
+		    next;
+		}
 		if ($xs =~ m{^\@}) {
 		    $next_y->();
 		    die "\@... => not string" unless is_string($y);
@@ -262,14 +328,22 @@ sub analyse_chunk_core () {
 		    }
 		    $next_y->();
 		    if (!$bras and
-			(grep { $y->{E} eq $_ } qw( or xor and not ; : )
+			(grep { $y->{E} eq $_ } qw( or xor and not ; :
+						    if unless while when )
+			 or $y->{E} eq ','
 			 or $y->{T} eq 'ket'
 			)) {
+			# lookahead shows close of containing scope
+			# or lower precedence operator
 			unshift @ys, $y;
+			pop @analysed_y;
 			last;
 		    }
-		    $xs =~ s{^\s+}{};
-		    #debug $ichunkstart, "TOKEN $y->{P}\n";
+		    $xs =~ s{^\s+}{} if $bras;
+		    if (is_string($y) and $y->{Q} eq '"') {
+			$exactly->($y->{V}, $y->{P});
+			next;
+		    }
 		    $exactly->($y->{E}, $y->{P});
 		    if ($y->{T} eq 'bra' or $y->{E} eq '?') {
 			$bras++;
@@ -291,6 +365,8 @@ sub analyse_chunk () {
 	return unless length $@;
 	if ($@ =~ m{^missing end of here doc (\S+)\n}) {
 	    # fudge this
+	    # (this never happens now, but in the future we might
+	    # want this code again eg to try adding to the chunk)
 	    $before .= "\n$1\n";
 	    $after .= "\n$1\n";
 	    next;
@@ -360,6 +436,8 @@ for ($ifilehead = 0; l_ok $ifilehead; $ifilehead++) {
 		    $ichunkend = $i;
 		    eval { analyse_chunk(); 1; };
 		    if (length $@) {
+			debug $ichunkstart, "done x: @analysed_x";
+			debug $ichunkstart, "done y: @analysed_y";
 			push @report, { M => $@,
 					S => $ichunkstart,
 					E => $ichunkend };