class Gem::RequestSet::Lockfile::Tokenizer - Documentation for Ruby 4.0
Description: Documentation for the Gem::RequestSet::Lockfile::Tokenizer class
Keywords: ruby,class,Gem::RequestSet::Lockfile::Tokenizer

🔍

No recent searches

# class Gem::RequestSet::Lockfile::Tokenizer

### Constants

EOF
Token

### Public Class Methods

from_file (file)
Source

    # File lib/rubygems/request_set/lockfile/tokenizer.rb, line 10
    def self.from_file(file)
      new File.read(file), file
    end

new (input, filename = nil, line = 0, pos = 0)
Source

    # File lib/rubygems/request_set/lockfile/tokenizer.rb, line 14
    def initialize(input, filename = nil, line = 0, pos = 0)
      @line     = line
      @line_pos = pos
      @tokens   = []
      @filename = filename
      tokenize input
    end

### Public Instance Methods

empty? ()
Source

    # File lib/rubygems/request_set/lockfile/tokenizer.rb, line 42
    def empty?
      @tokens.empty?
    end

make_parser (set, platforms)
Source

    # File lib/rubygems/request_set/lockfile/tokenizer.rb, line 22
    def make_parser(set, platforms)
      Gem::RequestSet::Lockfile::Parser.new self, set, platforms, @filename
    end

next_token ()
Source

    # File lib/rubygems/request_set/lockfile/tokenizer.rb, line 50
    def next_token
      @tokens.shift
    end

Also aliased as: shift
peek ()
Source

    # File lib/rubygems/request_set/lockfile/tokenizer.rb, line 55
    def peek
      @tokens.first || EOF
    end

shift ()
Alias for: next_token
skip (type)
Source

    # File lib/rubygems/request_set/lockfile/tokenizer.rb, line 30
    def skip(type)
      @tokens.shift while !@tokens.empty? && peek.type == type
    end

to_a ()
Source

    # File lib/rubygems/request_set/lockfile/tokenizer.rb, line 26
    def to_a
      @tokens.map {|token| [token.type, token.value, token.column, token.line] }
    end

unshift (token)
Source

    # File lib/rubygems/request_set/lockfile/tokenizer.rb, line 46
    def unshift(token)
      @tokens.unshift token
    end

### Private Instance Methods

tokenize (input)
Source

    # File lib/rubygems/request_set/lockfile/tokenizer.rb, line 61
    def tokenize(input)
      require "strscan"
      s = StringScanner.new input

      until s.eos? do
        pos = s.pos

        pos = s.pos if leading_whitespace = s.scan(/ +/)

        if s.scan(/[<|=>]{7}/)
          message = "your #{@filename} contains merge conflict markers"
          column, line = token_pos pos

          raise Gem::RequestSet::Lockfile::ParseError.new message, column, line, @filename
        end

        @tokens <<
          if s.scan(/\r?\n/)

            token = Token.new(:newline, nil, *token_pos(pos))
            @line_pos = s.pos
            @line += 1
            token
          elsif s.scan(/[A-Z]+/)

            if leading_whitespace
              text = s.matched
              text += s.scan(/[^\s)]*/).to_s # in case of no match
              Token.new(:text, text, *token_pos(pos))
            else
              Token.new(:section, s.matched, *token_pos(pos))
            end
          elsif s.scan(/([a-z]+):\s/)

            s.pos -= 1 # rewind for possible newline
            Token.new(:entry, s[1], *token_pos(pos))
          elsif s.scan(/\(/)

            Token.new(:l_paren, nil, *token_pos(pos))
          elsif s.scan(/\)/)

            Token.new(:r_paren, nil, *token_pos(pos))
          elsif s.scan(/<=|>=|=|~>|<|>|!=/)

            Token.new(:requirement, s.matched, *token_pos(pos))
          elsif s.scan(/,/)

            Token.new(:comma, nil, *token_pos(pos))
          elsif s.scan(/!/)

            Token.new(:bang, nil, *token_pos(pos))
          elsif s.scan(/[^\s),!]*/)

            Token.new(:text, s.matched, *token_pos(pos))
          else
            raise "BUG: can't create token for: #{s.string[s.pos..-1].inspect}"
          end
      end

      @tokens
    end