+sub semiparse ($) {
+ ($_) = @_;
+ my @o;
+ for (;;) {
+ s{^\s+}{};
+ if (s{^[\$\@\%]?[_0-9a-zA-Z]+}{}) {
+ push @o, { T => 'ident', L => $& };
+ } elsif (s{^\<\<('?)([A-Z_]+)\1}{}) {
+ my ($q,$d) = ($1,$2);
+ push @o, { T => 'heredoc', Q => $q, Delim => $d };
+ s{^
+ ( .* \n )
+ ( (?: (?! $d) .* \n )* )
+ }{ $1 } or die "missing end of here doc $d\n";
+ $o[$#o]{V} = $2;
+ } elsif (s{^ (["'])( (?: [^\\] | \\ \1 )* )}{}x) {
+ my ($q,$v) = ($1,$2);
+ push @o, { T => 'string', Q => $q, V => $v };
+ } elsif (s{^$perlop_re}{}) {
+ push @o, { T => 'op', L => $& };
+ } elsif (s/[[{(]//) {
+ push @o, { T => 'bra', L => $& };
+ } elsif (s/[]})]//) {
+ push @o, { T => 'ket', L => $& };
+ } elsif (!length) {
+ last;
+ } else {
+ m{^.{0,10}};
+ die "cannot tokenise \`$&'";
+ }
+ }
+ return @o;
+}
+
+sub analyse_chunk () {
+ die "plain deletion\n" unless defined $after;
+ die "plain insertion\n" unless defined $before;
+ my @before = semiparse $before;
+ my @after = semiparse $after;
+ print Dumper($ichunkstart, $ichunkend, \@before, \@after);
+ flush STDOUT;
+}
+