From: Jeff Ober <jober@ziprecruiter.com>
Date: Fri, 17 Jan 2020 19:02:24 +0000 (-0500)
Subject: Tokenizer no longer keeps copy of every token
X-Git-Tag: nailing-cargo/1.0.0~234^2~16
X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=commitdiff_plain;h=436ad07f1cce8d202c601ca0d0609e01e42fba4b;p=nailing-cargo.git

Tokenizer no longer keeps copy of every token
---

diff --git a/lib/TOML/Tiny/Tokenizer.pm b/lib/TOML/Tiny/Tokenizer.pm
index f40c228..4704d8f 100644
--- a/lib/TOML/Tiny/Tokenizer.pm
+++ b/lib/TOML/Tiny/Tokenizer.pm
@@ -17,32 +17,20 @@ sub new {
     last_position => length $param{source},
     position      => 0,
     line          => 1,
-    tokens        => [],
+    last_token    => undef,
   }, $class;
 
   return $self;
 }
 
-sub prev_token_type {
-  my $self = shift;
-
-  if (@{$self->{tokens}}) {
-    return $self->{tokens}[-1]{type} // 'EOL';
-  }
-
-  return 'EOL';
-}
-
 sub next_token {
   my $self = shift;
 
   return unless defined $self->{source}
       && $self->{position} < $self->{last_position};
 
-  if (!@{ $self->{tokens} }) {
-    my $root = {type => 'table', pos => 0, line => 1, value => []};
-    $self->push_token($root);
-    return $root;
+  if (!$self->{last_token}) {
+    return $self->{last_token} = {type => 'table', pos => 0, line => 1, value => []};
   }
 
   # Update the regex engine's position marker in case some other regex
@@ -70,7 +58,7 @@ sub next_token {
 
   # More complex matches with regexps
   while ($self->{position} < $self->{last_position} && !defined($type)) {
-    my $prev = $self->prev_token_type;
+    my $prev = $self->{last_token} ? $self->{last_token}{type} : 'EOL';
     my $newline = !!($prev eq 'EOL' || $prev eq 'table' || $prev eq 'array_table');
 
     for ($self->{source}) {
@@ -133,14 +121,12 @@ sub next_token {
     }
 
     if ($type) {
-      $token = {
+      $token = $self->{last_token} = {
         line  => $self->{line},
         pos   => $self->{pos},
         type  => $type,
         value => $self->can("tokenize_$type") ? $self->can("tokenize_$type")->($self, $value) : $value,
       };
-
-      $self->push_token($token);
     }
 
     $self->update_position;
@@ -149,16 +135,6 @@ sub next_token {
   return $token;
 }
 
-sub push_token {
-  my $self = shift;
-  @_ && push @{$self->{tokens}}, @_;
-}
-
-sub pop_token {
-  my $self = shift;
-  pop @{$self->{tokens}};
-}
-
 sub current_line {
   my $self = shift;
   my $rest = substr $self->{source}, $self->{position};