From: Jeff Ober <jober@ziprecruiter.com>
Date: Wed, 15 Jan 2020 17:09:02 +0000 (-0500)
Subject: Another round of performance improvements with a focus on regexp performance
X-Git-Tag: nailing-cargo/1.0.0~234^2~38
X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=commitdiff_plain;h=08b2c2c77784950ba10f8b15ec0ab657f1a1b7c3;p=nailing-cargo.git

Another round of performance improvements with a focus on regexp performance
---

diff --git a/lib/TOML/Tiny/Grammar.pm b/lib/TOML/Tiny/Grammar.pm
index 739f677..fa2ba29 100644
--- a/lib/TOML/Tiny/Grammar.pm
+++ b/lib/TOML/Tiny/Grammar.pm
@@ -27,7 +27,7 @@ our $TOML = qr{
     | (?&InlineTable)
   )
 
-  (?<NLSeq> (?> \x0D? \x0A))
+  (?<NLSeq> \x0D? \x0A)
   (?<NL> (?&NLSeq) | (?&Comment))
 
   (?<WSChar> \x20 | \x09)       # (space, tab)
@@ -115,10 +115,10 @@ our $TOML = qr{
   #-----------------------------------------------------------------------------
   # Key
   #-----------------------------------------------------------------------------
-  (?<BareKey>   [-_a-zA-Z0-9]+)
-  (?<QuotedKey> (?&BasicString) | (?&StringLiteral))
-  (?<SimpleKey> (?&BareKey) | (?&QuotedKey))
-  (?<DottedKey> (?&SimpleKey) (?: \x2E (?&SimpleKey) )+)
+  (?<BareKey>   (?> [-_a-zA-Z0-9]+ ))
+  (?<QuotedKey> (?> (?&BasicString) | (?&StringLiteral)))
+  (?<SimpleKey> (?> (?&BareKey) | (?&QuotedKey)))
+  (?<DottedKey> (?> (?&SimpleKey) (?: \x2E (?&SimpleKey) )+))
   (?<Key>       (?&BareKey) | (?&QuotedKey) | (?&DottedKey))
 
   #-----------------------------------------------------------------------------
@@ -136,29 +136,39 @@ our $TOML = qr{
   (?<BinChar>       [01])
 
   (?<Zero> [-+]? 0)
-  (?<Dec> (?&Zero) | (?: [-+]? (?&DecFirstChar) (?: (?&DecChar) | (?: _ (?&DecChar) ))*))
-  (?<Hex> 0x (?&HexChar) (?: (?&HexChar) | (?: [_] (?&HexChar) ))*)
-  (?<Oct> 0o (?&OctChar) (?: (?&OctChar) | (?: [_] (?&OctChar) ))*)
-  (?<Bin> 0b (?&BinChar) (?: (?&BinChar) | (?: [_] (?&BinChar) ))*)
+  (?<Hex> 0x (?&HexChar) (?> _? (?&HexChar) )*)
+  (?<Oct> 0o (?&OctChar) (?> _? (?&OctChar) )*)
+  (?<Bin> 0b (?&BinChar) (?> _? (?&BinChar) )*)
+  (?<Dec>
+      (?&Zero)
+    | (?> [-+]? (?&DecFirstChar) (?> _?  (?&DecChar) )* )
+  )
 
-  (?<Integer> (?&Hex) | (?&Oct) | (?&Bin) | (?&Dec))
+  (?<Integer>
+    (?>
+        (?&Hex)
+      | (?&Oct)
+      | (?&Bin)
+      | (?&Dec)
+    )
+  )
 
   #-----------------------------------------------------------------------------
   # Float
   #-----------------------------------------------------------------------------
   (?<Exponent>      [eE] (?&Dec))
-  (?<SpecialFloat>  [-+]? (?:inf) | (?:nan))
+  (?<SpecialFloat>  [-+]? (?> (?:inf) | (?:nan)))
   (?<Fraction>      [.] (?&Dec) )
 
   (?<Float>
-      (?:
-        (?&Dec)
+    (?>
+      (?&Dec)
 
-        (?:
-            (?: (?&Fraction) (?&Exponent)? )
-          | (?&Exponent)
-        )
+      (?>
+          (?> (?&Fraction) (?&Exponent)? )
+        | (?&Exponent)
       )
+    )
     | (?&SpecialFloat)
   )
 
@@ -167,31 +177,32 @@ our $TOML = qr{
   #-----------------------------------------------------------------------------
   (?<EscapeChar>
     \x5C                        # leading \
-    (?:
+    (?>
         [\x5C"btnfr]            # escapes: \\ \" \b \t \n \f \r
-      | (?: u [_0-9a-fA-F]{4} ) # unicode (4 bytes)
-      | (?: U [_0-9a-fA-F]{8} ) # unicode (8 bytes)
+      | (?> u [_0-9a-fA-F]{4} ) # unicode (4 bytes)
+      | (?> U [_0-9a-fA-F]{8} ) # unicode (8 bytes)
     )
   )
 
   (?<StringLiteral>
-    (?: ' [^']* ')            # single quoted string (no escaped chars allowed)
+    (?> ' [^']* ')            # single quoted string (no escaped chars allowed)
   )
 
   (?<MultiLineStringLiteral>
-    (?m)
-    (?s)
-    '''                       # opening triple-quote
-    .*?
-    '''                       # closing triple-quote
-    (?-s)
-    (?-m)
+    (?>
+      '''                     # opening triple-quote
+      (?>
+          [^']
+        | '{1,2}
+      )*?
+      '''                     # closing triple-quote
+    )
   )
 
   (?<BasicString>
-    (?:
+    (?>
       "                       # opening quote
-      (?:                     # escape sequences or any char except " or \
+      (?>                     # escape sequences or any char except " or \
           [^"\\]
         | (?&EscapeChar)
       )*
@@ -200,24 +211,20 @@ our $TOML = qr{
   )
 
   (?<MultiLineString>
-    (?m)
-    (?s)
     """                       # opening triple-quote
-    (?:
+    (?>
         [^"\\]
       | "{1,2}                # 1-2 quotation marks
       | (?&EscapeChar)        # escape
-      | (?: \\ $)
+      | (?: \\ (?&NLSeq))     # backslash-terminated line
     )*?
     """                       # closing triple-quote
-    (?-s)
-    (?-m)
   )
 
   (?<String>
-      (?&MultiLineString)
+      (?&MultiLineString)     # multi-line first or first two chars match empty basic string
     | (?&BasicString)
-    | (?&MultiLineStringLiteral)
+    | (?&MultiLineStringLiteral) 
     | (?&StringLiteral)
   )
 
@@ -225,28 +232,11 @@ our $TOML = qr{
   # Dates (RFC 3339)
   #   1985-04-12T23:20:50.52Z
   #-----------------------------------------------------------------------------
-  (?<Date> \d{4}-\d{2}-\d{2})
-
-  (?<Offset>
-      (?: [-+] \d{2}:\d{2} )
-    | [Z]
-  )
-
-  (?<SimpleTime>
-    \d{2}:\d{2}:\d{2}
-    (?: [.] \d+ )?
-  )
-
-  (?<Time>
-    (?&SimpleTime)
-    (?&Offset)?
-  )
-
-  (?<DateTime>
-      (?: (?&Date) [T ] (?&Time) )
-    | (?&Date)
-    | (?&Time)
-  )
+  (?<Date>        \d{4}-\d{2}-\d{2} )
+  (?<Offset>      (?: [-+] \d{2}:\d{2} ) | Z )
+  (?<SimpleTime>  \d{2}:\d{2}:\d{2} (?: \. \d+ )? )
+  (?<Time>        (?&SimpleTime) (?&Offset)? )
+  (?<DateTime>    (?> (?&Date) (?> [T ] (?&Time) )? ) | (?&Time) )
 )
 
 }x;
diff --git a/lib/TOML/Tiny/Tokenizer.pm b/lib/TOML/Tiny/Tokenizer.pm
index 286bcce..ea71113 100644
--- a/lib/TOML/Tiny/Tokenizer.pm
+++ b/lib/TOML/Tiny/Tokenizer.pm
@@ -26,13 +26,8 @@ sub new {
 sub next_token {
   my $self = shift;
 
-  if (!defined($self->{source})) {
-    return;
-  }
-
-  if ($self->is_exhausted) {
-    return;
-  }
+  return unless defined $self->{source}
+      && $self->{position} < $self->{last_position};
 
   if (!@{ $self->{tokens} }) {
     my $root = $self->_make_token('table', []);
@@ -46,29 +41,29 @@ sub next_token {
 
   my $token;
 
-  while (!defined($token) && !$self->is_exhausted) {
+  state $key = qr/(?&Key) $TOML/x;
+
+  while ($self->{position} < $self->{last_position} && !$token) {
     for ($self->{source}) {
-      when (/\G (?&NL) $TOML/xgc) {
-        ++$self->{line};
-        $token = $self->_make_token('EOL');
+      when (/\G [\x20 \x09]+/xgc) {
+        ;
       }
 
-      when (/\G (?&WSChar)+ $TOML/xgc) {
+      when (/\G \x23 .*/xgc) {
         ;
       }
 
-      when (/\G ((?&Key)) (?&WS) (?= =) $TOML/xgc) {
-         $token = $self->_make_token('key', $1);
+      when (/\G \x0D? \x0A/xgc) {
+        ++$self->{line};
+        $token = $self->_make_token('EOL');
       }
 
-      when (/\G \[ (?&WS) ((?&Key)) (?&WS) \] (?&WS) (?=(?&NL) | $)$TOML/xgc) {
-        my $key = $self->tokenize_key($1);
-        $token = $self->_make_token('table', $key);
+      when (/\G \[ [\x20 \x09]* ($key) [\x20 \x09]* \] [\x20 \x09]* (?= (:? \x23 .* )? (?: \x0D? \x0A) | $ )/xgc) {
+        $token = $self->_make_token('table', $self->tokenize_key($1));
       }
 
-      when (/\G \[\[ (?&WS) ((?&Key)) (?&WS) \]\] (?&WS) (?=(?&NL) | $) $TOML/xgc) {
-        my $key = $self->tokenize_key($1);
-        $token = $self->_make_token('array_table', $key);
+      when (/\G \[\[ [\x20 \x09]* ($key) [\x20 \x09]* \]\] [\x20 \x09]* (?= (:? \x23 .* )? (?: \x0D? \x0A) | $ )/xgc) {
+        $token = $self->_make_token('array_table', $self->tokenize_key($1));
       }
 
       when (/\G \[ /xgc) {
@@ -87,36 +82,40 @@ sub next_token {
         $token = $self->_make_token('inline_table_close', $1);
       }
 
-      when (/\G ((?&Boolean)) $TOML/xgc) {
-        $token = $self->_make_token('bool', $1);
+      when (/\G = /xgc) {
+        $token = $self->_make_token('assign', $1);
       }
 
-      when (/\G ((?&DateTime)) $TOML/xgc) {
-        $token = $self->_make_token('datetime', $1);
+      when (/\G , /xgc) {
+        $token = $self->_make_token('comma', $1);
       }
 
-      when (/\G ((?&Float)) $TOML/xgc) {
-        $token = $self->_make_token('float', $1);
+      when (/\G ($key) [\x20 \x09]* (?= =)/xgc) {
+         $token = $self->_make_token('key', $1);
       }
 
-      when (/\G ((?&Integer)) $TOML/xgc) {
-        $token = $self->_make_token('integer', $1);
+      when (/\G ((?&Boolean)) $TOML/xgc) {
+        $token = $self->_make_token('bool', $1);
       }
 
       when (/\G ((?&String)) $TOML/xgc) {
         $token = $self->_make_token('string', $1);
       }
 
-      when (/\G = /xgc) {
-        $token = $self->_make_token('assign', $1);
+      when (/\G ((?&DateTime)) $TOML/xgc) {
+        $token = $self->_make_token('datetime', $1);
       }
 
-      when (/\G , /xgc) {
-        $token = $self->_make_token('comma', $1);
+      when (/\G ((?&Float)) $TOML/xgc) {
+        $token = $self->_make_token('float', $1);
+      }
+
+      when (/\G ((?&Integer)) $TOML/xgc) {
+        $token = $self->_make_token('integer', $1);
       }
 
       default{
-        my $substr = substr($self->{source}, $self->{position} - 20, 40) // 'undef';
+        my $substr = substr($self->{source}, $self->{position}, 30) // 'undef';
         die "toml syntax error on line $self->{line}\n\t--> $substr\n";
       }
     }
@@ -130,8 +129,7 @@ sub next_token {
 
 sub push_token {
   my $self = shift;
-  my $token = shift // return;
-  push @{$self->{tokens}}, $token;
+  @_ && push @{$self->{tokens}}, @_;
 }
 
 sub pop_token {
@@ -141,15 +139,12 @@ sub pop_token {
 
 sub _make_token {
   my ($self, $type, $value) = @_;
-
-  my $token = {
+  return {
     type  => $type,
     line  => $self->{line},
     pos   => $self->{position},
     value => $self->can("tokenize_$type") ?  $self->can("tokenize_$type")->($self, $value) : $value,
   };
-
-  return $token;
 }
 
 sub current_line {
@@ -159,10 +154,6 @@ sub current_line {
   substr $rest, 0, $stop;
 }
 
-sub is_exhausted {
-  return $_[0]->{position} >= $_[0]->{last_position};
-}
-
 sub update_position {
   my $self = shift;
   $self->{position} = pos($self->{source}) // 0;
@@ -194,18 +185,13 @@ sub tokenize_key {
 }
 
 sub tokenize_float {
-  my $self = shift;
-  my $toml = shift;
-  $toml =~ s/_//g;
-  $toml;
+  $_[1] =~ tr/_//d;
+  $_[1];
 }
 
 sub tokenize_integer {
-  my $self = shift;
-  my $toml = shift;
-  $toml =~ s/_//g;
-  $toml =~ s/^[+]//;
-  return $toml;
+  $_[1] =~ tr/_+//d;
+  $_[1];
 }
 
 sub tokenize_string {
@@ -219,7 +205,7 @@ sub tokenize_string {
     $str = substr $toml, 3, length($toml) - 6;
     my @newlines = $str =~ /(\x0D?\x0A)/g;
     $self->{line} += scalar @newlines;
-    $str =~ s/^(?&WS) (?&NL) $TOML//x; # trim leading whitespace
+    $str =~ s/^[\x20 \x09]* (?&NL) $TOML//x; # trim leading whitespace
     $str =~ s/\\(?&NL)\s* $TOML//xgs;  # trim newlines from lines ending in backslash
   } else {
     $str = substr($toml, 1, length($toml) - 2);
diff --git a/t/tokens/string.t b/t/tokens/string.t
index 4606562..d8c5698 100644
--- a/t/tokens/string.t
+++ b/t/tokens/string.t
@@ -10,6 +10,18 @@ sub test_simple_matches {
   }
 }
 
+subtest 'string group' => sub{
+  use Regexp::Debugger;
+  my $re = qr{ ((?&String)) $TOML }x;
+
+  test_simple_matches($re,
+    [q{"A"}, q{"A"}, 'basic string'],
+    [q{'A'}, q{'A'}, 'string literal'],
+    [q{"""A"""}, q{"""A"""}, 'multi-line string'],
+    [q{'''A'''}, q{'''A'''}, 'multi-line string literal'],
+  );
+};
+
 subtest 'escaped characters' => sub{
   my $re = qr{
     ((?&EscapeChar))
@@ -64,44 +76,22 @@ subtest 'multi-line strings' => sub{
   }x;
 
   test_simple_matches($re,
-    [
-      qq{"""\nabc"""},
-      qq{"""\nabc"""},
-      'simple',
-    ],
-
-    [
-      qq{"""a\n"b"\nc"""},
-      qq{"""a\n"b"\nc"""},
-      'individual quotes within ml string',
-    ],
-
-    [
-      qq{"""foo"""bar"""},
-      qq{"""foo"""},
-      'invalid: triple-quotes appear within ml string',
-    ],
+    [ qq{"""\nabc"""}, qq{"""\nabc"""}, 'simple' ],
+    [ qq{""" " """}, q{""" " """}, 'containing 1 quote' ],
+    [ qq{""" "" """}, q{""" "" """}, 'containing 2 quotes' ],
+    [ qq{"""a\n"b"\nc"""}, qq{"""a\n"b"\nc"""}, 'individual quotes within ml string' ],
+    [ qq{"""foo"""bar"""}, qq{"""foo"""}, 'invalid: triple-quotes appear within ml string' ],
   );
 };
 
 subtest 'multi-line string literals' => sub{
-  my $re = qr{
-    ((?&MultiLineStringLiteral))
-    $TOML
-  }x;
+  my $re = qr{ ((?&MultiLineStringLiteral)) $TOML }x;
 
   test_simple_matches($re,
-    [
-      qq{'''\nabc'''},
-      qq{'''\nabc'''},
-      'simple',
-    ],
-
-    [
-      qq{'''foo'''bar'''},
-      qq{'''foo'''},
-      'invalid: triple-quotes appear within ml string',
-    ],
+    [ qq{'''\nabc'''}, qq{'''\nabc'''}, 'simple' ],
+    [ qq{''' ' '''}, q{''' ' '''}, 'containing 1 single tick' ],
+    [ qq{''' '' '''}, q{''' '' '''}, 'containing 2 single ticks' ],
+    [ qq{'''foo'''bar'''}, qq{'''foo'''}, 'invalid: triple-quotes appear within ml string' ],
   );
 };