Performance improvements in next_token

diff --git a/Changes b/Changes
index 6fa20902882b172b25858bc69d5d31283a664693..00a2887be4fd3e2f3573f83730854294fdc96e4f 100644 (file)
--- a/Changes
+++ b/Changes
@@ -1,4 +1,5 @@
 {{$NEXT}}
+-Performance improvements in TOML::Tiny::Tokenizer->next_token
 
 0.05 2020-01-21
 -POD fixes for TOML::Tiny::Grammar

diff --git a/README.pod b/README.pod
index 8032567191165c49da607110fb6ecb352a19d3b9..a42a7edb2453c68e17e06a8bc85b8aaa7735296c 100644 (file)
--- a/README.pod
+++ b/README.pod
@@ -8,7 +8,7 @@ TOML::Tiny - a minimal, pure perl TOML parser and serializer
 
 =head1 VERSION
 
-version 0.05
+version 0.06
 
 =head1 SYNOPSIS
 

diff --git a/lib/TOML/Tiny/Tokenizer.pm b/lib/TOML/Tiny/Tokenizer.pm
index 4704d8f401aeaebea41387535ff11f3f0853e649..e2deb91d6ae213328149d0200a071746735178e1 100644 (file)
--- a/lib/TOML/Tiny/Tokenizer.pm
+++ b/lib/TOML/Tiny/Tokenizer.pm
@@ -121,11 +121,14 @@ sub next_token {
     }
 
     if ($type) {
+      state $tokenizers = {};
+      my $tokenize = $tokenizers->{$type} //= $self->can("tokenize_$type") || 0;
+
       $token = $self->{last_token} = {
         line  => $self->{line},
         pos   => $self->{pos},
         type  => $type,
-        value => $self->can("tokenize_$type") ? $self->can("tokenize_$type")->($self, $value) : $value,
+        value => $tokenize ? $tokenize->($self, $value) : $value,
       };
     }
 
@@ -158,11 +161,7 @@ sub error {
 sub tokenize_key {
   my $self = shift;
   my $toml = shift;
-  my @keys;
-
-  while ($toml =~ s/^ ($SimpleKey) [.]?//x) {
-    push @keys, $1;
-  }
+  my @keys = $toml =~ /($SimpleKey)\.?/g;
 
   for (@keys) {
     s/^["']//;
@@ -185,8 +184,9 @@ sub tokenize_integer {
 sub tokenize_string {
   my $self = shift;
   my $toml = shift;
-  my $ml   = $toml =~ /^(?:''')|(?:""")/;
-  my $lit  = $toml =~ /^'/;
+  my $ml   = index($toml, q{'''}) == 0
+          || index($toml, q{"""}) == 0;
+  my $lit  = index($toml, q{'}) == 0;
   my $str  = '';
 
   if ($ml) {
@@ -203,7 +203,7 @@ sub tokenize_string {
     $str = $self->unescape_str($str);
   }
 
-  return ''.$str;
+  return $str;
 }
 
 sub unescape_chars {