Float optimizations

diff --git a/lib/TOML/Tiny/Grammar.pm b/lib/TOML/Tiny/Grammar.pm
index fa2ba293c7fcb9efa37d51b8e30039a9f16a7091..73c58ec59e383564407af02fab841538920e8002 100644 (file)
--- a/lib/TOML/Tiny/Grammar.pm
+++ b/lib/TOML/Tiny/Grammar.pm
@@ -158,7 +158,7 @@ our $TOML = qr{
   #-----------------------------------------------------------------------------
   (?<Exponent>      [eE] (?&Dec))
   (?<SpecialFloat>  [-+]? (?> (?:inf) | (?:nan)))
-  (?<Fraction>      [.] (?&Dec) )
+  (?<Fraction>      [.] (?&DecChar) (?> _? (?&DecChar) )* )
 
   (?<Float>
     (?>

diff --git a/lib/TOML/Tiny/Parser.pm b/lib/TOML/Tiny/Parser.pm
index 4d92eeaedf3dd3031caec3b76a139ad9949e6272..ab0733eb5cc5d1a3dff910bf71d4b8cf76e72b33 100644 (file)
--- a/lib/TOML/Tiny/Parser.pm
+++ b/lib/TOML/Tiny/Parser.pm
@@ -59,10 +59,10 @@ sub parse {
 sub parse_error {
   my ($self, $token, $msg) = @_;
   my $line = $token ? $token->{line} : 'EOF';
-  if ($self->{annotated}) {
+  if ($self->{annotated} || $ENV{TOML_TINY_DEBUG}) {
     my $root = Dumper($self->{root});
     my $tok  = Dumper($token);
-    my $src  = substr $self->{tokenizer}{source}, $self->{tokenizer}{position} - 20, 40;
+    my $src  = substr $self->{tokenizer}{source}, $self->{tokenizer}{position}, 30;
 
     confess qq{
 toml parse error at line $line:

diff --git a/lib/TOML/Tiny/Tokenizer.pm b/lib/TOML/Tiny/Tokenizer.pm
index ea71113267cd045035612a2f04093285e27eef40..51f78bd86d2af3a4b6ea616a0b5ddb3ab524a998 100644 (file)
--- a/lib/TOML/Tiny/Tokenizer.pm
+++ b/lib/TOML/Tiny/Tokenizer.pm
@@ -16,13 +16,23 @@ sub new {
     source        => $param{source},
     last_position => length $param{source},
     position      => 0,
-    line          => 0,
+    line          => 1,
     tokens        => [],
   }, $class;
 
   return $self;
 }
 
+sub prev_token_type {
+  my $self = shift;
+
+  if (@{$self->{tokens}}) {
+    return $self->{tokens}[-1]{type} // 'EOL';
+  }
+
+  return 'EOL';
+}
+
 sub next_token {
   my $self = shift;
 
@@ -44,26 +54,28 @@ sub next_token {
   state $key = qr/(?&Key) $TOML/x;
 
   while ($self->{position} < $self->{last_position} && !$token) {
+    my $prev = $self->prev_token_type;
+    my $newline = !!($prev eq 'EOL' || $prev eq 'table' || $prev eq 'array_table');
+
     for ($self->{source}) {
-      when (/\G [\x20 \x09]+/xgc) {
-        ;
-      }
+      /\G[\x20\x09]+/gc;    # ignore whitespace
+      /\G\x23.*/gc && next; # ignore comments
 
-      when (/\G \x23 .*/xgc) {
-        ;
-      }
+      last when /\G $/xgc;
 
       when (/\G \x0D? \x0A/xgc) {
         ++$self->{line};
         $token = $self->_make_token('EOL');
       }
 
-      when (/\G \[ [\x20 \x09]* ($key) [\x20 \x09]* \] [\x20 \x09]* (?= (:? \x23 .* )? (?: \x0D? \x0A) | $ )/xgc) {
-        $token = $self->_make_token('table', $self->tokenize_key($1));
-      }
+      if ($newline) {
+        when (/\G \[ [\x20 \x09]* ($key) [\x20 \x09]* \] [\x20 \x09]* (?= (:? \x23 .* )? (?: \x0D? \x0A) | $ )/xgc) {
+          $token = $self->_make_token('table', $self->tokenize_key($1));
+        }
 
-      when (/\G \[\[ [\x20 \x09]* ($key) [\x20 \x09]* \]\] [\x20 \x09]* (?= (:? \x23 .* )? (?: \x0D? \x0A) | $ )/xgc) {
-        $token = $self->_make_token('array_table', $self->tokenize_key($1));
+        when (/\G \[\[ [\x20 \x09]* ($key) [\x20 \x09]* \]\] [\x20 \x09]* (?= (:? \x23 .* )? (?: \x0D? \x0A) | $ )/xgc) {
+          $token = $self->_make_token('array_table', $self->tokenize_key($1));
+        }
       }
 
       when (/\G \[ /xgc) {
@@ -91,7 +103,7 @@ sub next_token {
       }
 
       when (/\G ($key) [\x20 \x09]* (?= =)/xgc) {
-         $token = $self->_make_token('key', $1);
+        $token = $self->_make_token('key', $1);
       }
 
       when (/\G ((?&Boolean)) $TOML/xgc) {

diff --git a/t/tokens/float.t b/t/tokens/float.t
index c80768819a28c9210835fa6d23905bf7af557b4f..79e4ed0891b4d595e7d7d85d1d6950cc849617f2 100644 (file)
--- a/t/tokens/float.t
+++ b/t/tokens/float.t
@@ -4,6 +4,7 @@ use TOML::Tiny::Grammar;
 my $re = qr{ ((?&Float)) $TOML }x;
 
 my @valid = qw(
+  0.01
   +1.0
   3.1415
   -0.01