From: Jeff Ober <jober@ziprecruiter.com>
Date: Tue, 21 Jan 2020 21:52:56 +0000 (-0500)
Subject: Performance improvements in next_token
X-Git-Tag: nailing-cargo/1.0.0~234^2~2
X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=commitdiff_plain;h=1ddfddef90c031136e7207a480981f2b6123c757;p=nailing-cargo.git

Performance improvements in next_token
---

diff --git a/Changes b/Changes
index 6fa2090..00a2887 100644
--- a/Changes
+++ b/Changes
@@ -1,4 +1,5 @@
 {{$NEXT}}
+-Performance improvements in TOML::Tiny::Tokenizer->next_token
 
 0.05 2020-01-21
 -POD fixes for TOML::Tiny::Grammar
diff --git a/README.pod b/README.pod
index 8032567..a42a7ed 100644
--- a/README.pod
+++ b/README.pod
@@ -8,7 +8,7 @@ TOML::Tiny - a minimal, pure perl TOML parser and serializer
 
 =head1 VERSION
 
-version 0.05
+version 0.06
 
 =head1 SYNOPSIS
 
diff --git a/lib/TOML/Tiny/Tokenizer.pm b/lib/TOML/Tiny/Tokenizer.pm
index 4704d8f..e2deb91 100644
--- a/lib/TOML/Tiny/Tokenizer.pm
+++ b/lib/TOML/Tiny/Tokenizer.pm
@@ -121,11 +121,14 @@ sub next_token {
     }
 
     if ($type) {
+      state $tokenizers = {};
+      my $tokenize = $tokenizers->{$type} //= $self->can("tokenize_$type") || 0;
+
       $token = $self->{last_token} = {
         line  => $self->{line},
         pos   => $self->{pos},
         type  => $type,
-        value => $self->can("tokenize_$type") ? $self->can("tokenize_$type")->($self, $value) : $value,
+        value => $tokenize ? $tokenize->($self, $value) : $value,
       };
     }
 
@@ -158,11 +161,7 @@ sub error {
 sub tokenize_key {
   my $self = shift;
   my $toml = shift;
-  my @keys;
-
-  while ($toml =~ s/^ ($SimpleKey) [.]?//x) {
-    push @keys, $1;
-  }
+  my @keys = $toml =~ /($SimpleKey)\.?/g;
 
   for (@keys) {
     s/^["']//;
@@ -185,8 +184,9 @@ sub tokenize_integer {
 sub tokenize_string {
   my $self = shift;
   my $toml = shift;
-  my $ml   = $toml =~ /^(?:''')|(?:""")/;
-  my $lit  = $toml =~ /^'/;
+  my $ml   = index($toml, q{'''}) == 0
+          || index($toml, q{"""}) == 0;
+  my $lit  = index($toml, q{'}) == 0;
   my $str  = '';
 
   if ($ml) {
@@ -203,7 +203,7 @@ sub tokenize_string {
     $str = $self->unescape_str($str);
   }
 
-  return ''.$str;
+  return $str;
 }
 
 sub unescape_chars {