Introduce Int, Str and Token types for terminal symbols so that these token can store location information

lib/lrama/grammar.rb

@@ -66,7 +66,7 @@ class Grammar
     #   @required: bool
     #   @union: Union
     #   @precedences: Array[Precedence]
-    #   @start_nterm: Lrama::Lexer::Token?
+    #   @start_nterm: Lrama::Lexer::Token::Base?
 
     extend Forwardable
 
@@ -85,11 +85,11 @@ class Grammar
     attr_accessor :lex_param #: String
     attr_accessor :parse_param #: String
     attr_accessor :initial_action #: Grammar::Code::InitialActionCode
-    attr_accessor :after_shift #: Lexer::Token
-    attr_accessor :before_reduce #: Lexer::Token
-    attr_accessor :after_reduce #: Lexer::Token
-    attr_accessor :after_shift_error_token #: Lexer::Token
-    attr_accessor :after_pop_stack #: Lexer::Token
+    attr_accessor :after_shift #: Lexer::Token::Base
+    attr_accessor :before_reduce #: Lexer::Token::Base
+    attr_accessor :after_reduce #: Lexer::Token::Base
+    attr_accessor :after_shift_error_token #: Lexer::Token::Base
+    attr_accessor :after_pop_stack #: Lexer::Token::Base
     attr_accessor :symbols_resolver #: Symbols::Resolver
     attr_accessor :types #: Array[Type]
     attr_accessor :rules #: Array[Rule]
@@ -141,7 +141,7 @@ def create_rule_builder(rule_counter, midrule_action_counter)
       RuleBuilder.new(rule_counter, midrule_action_counter, @parameterized_resolver)
     end
 
-    # @rbs (id: Lexer::Token, code: Lexer::Token::UserCode) -> Array[PercentCode]
+    # @rbs (id: Lexer::Token::Base, code: Lexer::Token::UserCode) -> Array[PercentCode]
     def add_percent_code(id:, code:)
       @percent_codes << PercentCode.new(id.s_value, code.s_value)
     end
@@ -161,7 +161,7 @@ def add_error_token(ident_or_tags:, token_code:, lineno:)
       @error_tokens << ErrorToken.new(ident_or_tags: ident_or_tags, token_code: token_code, lineno: lineno)
     end
 
-    # @rbs (id: Lexer::Token, tag: Lexer::Token::Tag) -> Array[Type]
+    # @rbs (id: Lexer::Token::Base, tag: Lexer::Token::Tag) -> Array[Type]
     def add_type(id:, tag:)
       @types << Type.new(id: id, tag: tag)
     end
@@ -186,7 +186,7 @@ def add_precedence(sym, precedence, s_value, lineno)
       set_precedence(sym, Precedence.new(s_value: s_value, type: :precedence, precedence: precedence, lineno: lineno))
     end
 
-    # @rbs (Lrama::Lexer::Token id) -> Lrama::Lexer::Token
+    # @rbs (Lrama::Lexer::Token::Base id) -> Lrama::Lexer::Token::Base
     def set_start_nterm(id)
       # When multiple `%start` directives are defined, Bison does not generate an error,
       # whereas Lrama does generate an error.
@@ -195,7 +195,7 @@ def set_start_nterm(id)
       if @start_nterm.nil?
         @start_nterm = id
       else
-        start = @start_nterm #: Lrama::Lexer::Token
+        start = @start_nterm #: Lrama::Lexer::Token::Base
         raise "Start non-terminal is already set to #{start.s_value} (line: #{start.first_line}). Cannot set to #{id.s_value} (line: #{id.first_line})."
       end
     end
@@ -459,12 +459,12 @@ def normalize_rules
     # Add $accept rule to the top of rules
     def add_accept_rule
       if @start_nterm
-        start = @start_nterm #: Lrama::Lexer::Token
+        start = @start_nterm #: Lrama::Lexer::Token::Base
         @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [start, @eof_symbol.id], token_code: nil, lineno: start.line)
       else
         rule_builder = @rule_builders.first # : RuleBuilder
         lineno = rule_builder ? rule_builder.line : 0
-        lhs = rule_builder.lhs # : Lexer::Token
+        lhs = rule_builder.lhs # : Lexer::Token::Base
         @rules << Rule.new(id: @rule_counter.increment, _lhs: @accept_symbol.id, _rhs: [lhs, @eof_symbol.id], token_code: nil, lineno: lineno)
       end
     end
@@ -477,7 +477,7 @@ def collect_symbols
         case s
         when Lrama::Lexer::Token::Char
           add_term(id: s)
-        when Lrama::Lexer::Token
+        when Lrama::Lexer::Token::Base
           # skip
         else
           raise "Unknown class: #{s}"

lib/lrama/grammar/binding.rb

@@ -4,16 +4,16 @@
 module Lrama
   class Grammar
     class Binding
-      # @rbs @actual_args: Array[Lexer::Token]
-      # @rbs @param_to_arg: Hash[String, Lexer::Token]
+      # @rbs @actual_args: Array[Lexer::Token::Base]
+      # @rbs @param_to_arg: Hash[String, Lexer::Token::Base]
 
-      # @rbs (Array[Lexer::Token] params, Array[Lexer::Token] actual_args) -> void
+      # @rbs (Array[Lexer::Token::Base] params, Array[Lexer::Token::Base] actual_args) -> void
       def initialize(params, actual_args)
         @actual_args = actual_args
         @param_to_arg = build_param_to_arg(params, @actual_args)
       end
 
-      # @rbs (Lexer::Token sym) -> Lexer::Token
+      # @rbs (Lexer::Token::Base sym) -> Lexer::Token::Base
       def resolve_symbol(sym)
         return create_instantiate_rule(sym) if sym.is_a?(Lexer::Token::InstantiateRule)
         find_arg_for_param(sym)
@@ -36,12 +36,12 @@ def create_instantiate_rule(sym)
         )
       end
 
-      # @rbs (Array[Lexer::Token]) -> Array[Lexer::Token]
+      # @rbs (Array[Lexer::Token::Base]) -> Array[Lexer::Token::Base]
       def resolve_args(args)
         args.map { |arg| resolve_symbol(arg) }
       end
 
-      # @rbs (Lexer::Token sym) -> Lexer::Token
+      # @rbs (Lexer::Token::Base sym) -> Lexer::Token::Base
       def find_arg_for_param(sym)
         if (arg = @param_to_arg[sym.s_value]&.dup)
           arg.alias_name = sym.alias_name
@@ -51,7 +51,7 @@ def find_arg_for_param(sym)
         end
       end
 
-      # @rbs (Array[Lexer::Token] params, Array[Lexer::Token] actual_args) -> Hash[String, Lexer::Token?]
+      # @rbs (Array[Lexer::Token::Base] params, Array[Lexer::Token::Base] actual_args) -> Hash[String, Lexer::Token::Base?]
       def build_param_to_arg(params, actual_args)
         params.zip(actual_args).map do |param, arg|
           [param.s_value, arg]

lib/lrama/grammar/parameterized/resolver.rb

@@ -6,7 +6,7 @@ class Grammar
     class Parameterized
       class Resolver
         attr_accessor :rules #: Array[Rule]
-        attr_accessor :created_lhs_list #: Array[Lexer::Token]
+        attr_accessor :created_lhs_list #: Array[Lexer::Token::Base]
 
         # @rbs () -> void
         def initialize
@@ -24,12 +24,12 @@ def find_rule(token)
           select_rules(@rules, token).last
         end
 
-        # @rbs (Lexer::Token token) -> Rule?
+        # @rbs (Lexer::Token::Base token) -> Rule?
         def find_inline(token)
           @rules.reverse.find { |rule| rule.name == token.s_value && rule.inline? }
         end
 
-        # @rbs (String lhs_s_value) -> Lexer::Token?
+        # @rbs (String lhs_s_value) -> Lexer::Token::Base?
         def created_lhs(lhs_s_value)
           @created_lhs_list.reverse.find { |created_lhs| created_lhs.s_value == lhs_s_value }
         end

lib/lrama/grammar/parameterized/rhs.rb

@@ -5,7 +5,7 @@ module Lrama
   class Grammar
     class Parameterized
       class Rhs
-        attr_accessor :symbols #: Array[Lexer::Token]
+        attr_accessor :symbols #: Array[Lexer::Token::Base]
         attr_accessor :user_code #: Lexer::Token::UserCode?
         attr_accessor :precedence_sym #: Grammar::Symbol?
 

lib/lrama/grammar/parameterized/rule.rb

@@ -6,12 +6,12 @@ class Grammar
     class Parameterized
       class Rule
         attr_reader :name #: String
-        attr_reader :parameters #: Array[Lexer::Token]
+        attr_reader :parameters #: Array[Lexer::Token::Base]
         attr_reader :rhs #: Array[Rhs]
         attr_reader :required_parameters_count #: Integer
         attr_reader :tag #: Lexer::Token::Tag?
 
-        # @rbs (String name, Array[Lexer::Token] parameters, Array[Rhs] rhs, tag: Lexer::Token::Tag?, is_inline: bool) -> void
+        # @rbs (String name, Array[Lexer::Token::Base] parameters, Array[Rhs] rhs, tag: Lexer::Token::Tag?, is_inline: bool) -> void
         def initialize(name, parameters, rhs, tag: nil, is_inline: false)
           @name = name
           @parameters = parameters

lib/lrama/grammar/reference.rb

@@ -14,11 +14,11 @@ class Reference < Struct.new(:type, :name, :number, :index, :ex_tag, :first_colu
       #   attr_accessor name: String
       #   attr_accessor number: Integer
       #   attr_accessor index: Integer
-      #   attr_accessor ex_tag: Lexer::Token?
+      #   attr_accessor ex_tag: Lexer::Token::Base?
       #   attr_accessor first_column: Integer
       #   attr_accessor last_column: Integer
       #
-      #   def initialize: (type: ::Symbol, ?name: String, ?number: Integer, ?index: Integer, ?ex_tag: Lexer::Token?, first_column: Integer, last_column: Integer) -> void
+      #   def initialize: (type: ::Symbol, ?name: String, ?number: Integer, ?index: Integer, ?ex_tag: Lexer::Token::Base?, first_column: Integer, last_column: Integer) -> void
 
       # @rbs () -> (String|Integer)
       def value

lib/lrama/grammar/rule.rb

@@ -13,10 +13,10 @@ class Rule < Struct.new(:id, :_lhs, :lhs, :lhs_tag, :_rhs, :rhs, :token_code, :p
       #   end
       #
       #   attr_accessor id: Integer
-      #   attr_accessor _lhs: Lexer::Token
+      #   attr_accessor _lhs: Lexer::Token::Base
       #   attr_accessor lhs: Grammar::Symbol
       #   attr_accessor lhs_tag: Lexer::Token::Tag?
-      #   attr_accessor _rhs: Array[Lexer::Token]
+      #   attr_accessor _rhs: Array[Lexer::Token::Base]
       #   attr_accessor rhs: Array[Grammar::Symbol]
       #   attr_accessor token_code: Lexer::Token::UserCode?
       #   attr_accessor position_in_original_rule_rhs: Integer
@@ -25,7 +25,7 @@ class Rule < Struct.new(:id, :_lhs, :lhs, :lhs_tag, :_rhs, :rhs, :token_code, :p
       #   attr_accessor lineno: Integer?
       #
       #   def initialize: (
-      #     ?id: Integer, ?_lhs: Lexer::Token?, ?lhs: Lexer::Token, ?lhs_tag: Lexer::Token::Tag?, ?_rhs: Array[Lexer::Token], ?rhs: Array[Grammar::Symbol],
+      #     ?id: Integer, ?_lhs: Lexer::Token::Base?, ?lhs: Lexer::Token::Base, ?lhs_tag: Lexer::Token::Tag?, ?_rhs: Array[Lexer::Token::Base], ?rhs: Array[Grammar::Symbol],
       #     ?token_code: Lexer::Token::UserCode?, ?position_in_original_rule_rhs: Integer?, ?nullable: bool,
       #     ?precedence_sym: Grammar::Symbol?, ?lineno: Integer?
       #   ) -> void

lib/lrama/grammar/rule_builder.rb

@@ -16,15 +16,15 @@ class RuleBuilder
       #   @rule_builders_for_derived_rules: Array[RuleBuilder]
       #   @parameterized_rules: Array[Rule]
       #   @midrule_action_rules: Array[Rule]
-      #   @replaced_rhs: Array[Lexer::Token]?
+      #   @replaced_rhs: Array[Lexer::Token::Base]?
 
-      attr_accessor :lhs #: Lexer::Token?
+      attr_accessor :lhs #: Lexer::Token::Base?
       attr_accessor :line #: Integer?
       attr_reader :rule_counter #: Counter
       attr_reader :midrule_action_counter #: Counter
       attr_reader :parameterized_resolver #: Grammar::Parameterized::Resolver
       attr_reader :lhs_tag #: Lexer::Token::Tag?
-      attr_reader :rhs #: Array[Lexer::Token]
+      attr_reader :rhs #: Array[Lexer::Token::Base]
       attr_reader :user_code #: Lexer::Token::UserCode?
       attr_reader :precedence_sym #: Grammar::Symbol?
 
@@ -49,7 +49,7 @@ def initialize(rule_counter, midrule_action_counter, parameterized_resolver, pos
         @midrule_action_rules = []
       end
 
-      # @rbs (Lexer::Token rhs) -> void
+      # @rbs (Lexer::Token::Base rhs) -> void
       def add_rhs(rhs)
         @line ||= rhs.line
 
@@ -111,7 +111,7 @@ def preprocess_references
 
       # @rbs () -> void
       def build_rules
-        tokens = @replaced_rhs #: Array[Lexer::Token]
+        tokens = @replaced_rhs #: Array[Lexer::Token::Base]
         return if tokens.any? { |t| @parameterized_resolver.find_inline(t) }
 
         rule = Rule.new(
@@ -137,7 +137,7 @@ def build_rules
       def process_rhs
         return if @replaced_rhs
 
-        replaced_rhs = [] #: Array[Lexer::Token]
+        replaced_rhs = [] #: Array[Lexer::Token::Base]
 
         rhs.each_with_index do |token, i|
           case token
@@ -219,7 +219,7 @@ def numberize_references
                 ref.name = '$'
               else
                 candidates = ([lhs] + rhs).each_with_index.select do |token, _i|
-                  # @type var token: Lexer::Token
+                  # @type var token: Lexer::Token::Base
                   token.referred_by?(ref_name)
                 end
 

lib/lrama/grammar/symbol.rb

@@ -9,7 +9,7 @@
 module Lrama
   class Grammar
     class Symbol
-      attr_accessor :id #: Lexer::Token
+      attr_accessor :id #: Lexer::Token::Base
       attr_accessor :alias_name #: String?
       attr_reader :number #: Integer
       attr_accessor :number_bitmap #: Bitmap::bitmap
@@ -28,7 +28,7 @@ class Symbol
       attr_writer :undef_symbol #: bool
       attr_writer :accept_symbol #: bool
 
-      # @rbs (id: Lexer::Token, term: bool, ?alias_name: String?, ?number: Integer?, ?tag: Lexer::Token?,
+      # @rbs (id: Lexer::Token::Base, term: bool, ?alias_name: String?, ?number: Integer?, ?tag: Lexer::Token::Tag?,
       #      ?token_id: Integer?, ?nullable: bool?, ?precedence: Precedence?, ?printer: Printer?) -> void
       def initialize(id:, term:, alias_name: nil, number: nil, tag: nil, token_id: nil, nullable: nil, precedence: nil, printer: nil, destructor: nil)
         @id = id

lib/lrama/grammar/symbols/resolver.rb

@@ -11,11 +11,11 @@ class Resolver
         #     def symbols: () -> Array[Grammar::Symbol]
         #     def nterms: () -> Array[Grammar::Symbol]
         #     def terms: () -> Array[Grammar::Symbol]
-        #     def add_nterm: (id: Lexer::Token, ?alias_name: String?, ?tag: Lexer::Token::Tag?) -> Grammar::Symbol
-        #     def add_term: (id: Lexer::Token, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?, ?replace: bool) -> Grammar::Symbol
+        #     def add_nterm: (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?) -> Grammar::Symbol
+        #     def add_term: (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?, ?replace: bool) -> Grammar::Symbol
         #     def find_symbol_by_number!: (Integer number) -> Grammar::Symbol
-        #     def find_symbol_by_id!: (Lexer::Token id) -> Grammar::Symbol
-        #     def token_to_symbol: (Lexer::Token token) -> Grammar::Symbol
+        #     def find_symbol_by_id!: (Lexer::Token::Base id) -> Grammar::Symbol
+        #     def token_to_symbol: (Lexer::Token::Base token) -> Grammar::Symbol
         #     def find_symbol_by_s_value!: (::String s_value) -> Grammar::Symbol
         #     def fill_nterm_type: (Array[Grammar::Type] types) -> void
         #     def fill_symbol_number: () -> void
@@ -48,7 +48,7 @@ def sort_by_number!
           symbols.sort_by!(&:number)
         end
 
-        # @rbs (id: Lexer::Token, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?, ?replace: bool) -> Grammar::Symbol
+        # @rbs (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?, ?token_id: Integer?, ?replace: bool) -> Grammar::Symbol
         def add_term(id:, alias_name: nil, tag: nil, token_id: nil, replace: false)
           if token_id && (sym = find_symbol_by_token_id(token_id))
             if replace
@@ -73,7 +73,7 @@ def add_term(id:, alias_name: nil, tag: nil, token_id: nil, replace: false)
           term
         end
 
-        # @rbs (id: Lexer::Token, ?alias_name: String?, ?tag: Lexer::Token::Tag?) -> Grammar::Symbol
+        # @rbs (id: Lexer::Token::Base, ?alias_name: String?, ?tag: Lexer::Token::Tag?) -> Grammar::Symbol
         def add_nterm(id:, alias_name: nil, tag: nil)
           if (sym = find_symbol_by_id(id))
             return sym
@@ -103,14 +103,14 @@ def find_symbol_by_s_value!(s_value)
           find_symbol_by_s_value(s_value) || (raise "Symbol not found. value: `#{s_value}`")
         end
 
-        # @rbs (Lexer::Token id) -> Grammar::Symbol?
+        # @rbs (Lexer::Token::Base id) -> Grammar::Symbol?
         def find_symbol_by_id(id)
           symbols.find do |s|
             s.id == id || s.alias_name == id.s_value
           end
         end
 
-        # @rbs (Lexer::Token id) -> Grammar::Symbol
+        # @rbs (Lexer::Token::Base id) -> Grammar::Symbol
         def find_symbol_by_id!(id)
           find_symbol_by_id(id) || (raise "Symbol not found. #{id}")
         end
@@ -203,10 +203,10 @@ def fill_error_token(error_tokens)
           end
         end
 
-        # @rbs (Lexer::Token token) -> Grammar::Symbol
+        # @rbs (Lexer::Token::Base token) -> Grammar::Symbol
         def token_to_symbol(token)
           case token
-          when Lrama::Lexer::Token
+          when Lrama::Lexer::Token::Base
             find_symbol_by_id!(token)
           else
             raise "Unknown class: #{token}"
@@ -222,7 +222,7 @@ def validate!
 
         private
 
-        # @rbs (Lexer::Token id) -> Grammar::Symbol
+        # @rbs (Lexer::Token::Base id) -> Grammar::Symbol
         def find_nterm_by_id!(id)
           @nterms.find do |s|
             s.id == id

lib/lrama/grammar/type.rb

@@ -9,13 +9,13 @@ class Type
       #       see: https://github.com/soutaro/rbs-inline/pull/149
       #
       # @rbs!
-      #   @id: Lexer::Token
+      #   @id: Lexer::Token::Base
       #   @tag: Lexer::Token::Tag
 
-      attr_reader :id #: Lexer::Token
+      attr_reader :id #: Lexer::Token::Base
       attr_reader :tag #: Lexer::Token::Tag
 
-      # @rbs (id: Lexer::Token, tag: Lexer::Token::Tag) -> void
+      # @rbs (id: Lexer::Token::Base, tag: Lexer::Token::Tag) -> void
       def initialize(id:, tag:)
         @id = id
         @tag = tag