class Ameba::Tokenizer

Overview

Represents Crystal syntax tokenizer based on Crystal::Lexer.

source = Ameba::Source.new code, path
tokenizer = Ameba::Tokenizer.new(source)
tokenizer.run do |token|
  puts token
end

ameba/tokenizer.cr

#run(&block : Crystal::Token -> _)
Runs the tokenizer and yields each token as a block argument.

def self.new(lexer : Crystal::Lexer) #

Instantiates Tokenizer using a lexer.

lexer = Crystal::Lexer.new(code)
Ameba::Tokenizer.new(lexer)

[View source]

def self.new(source) #

Instantiates Tokenizer using a source.

source = Ameba::Source.new code, path
Ameba::Tokenizer.new(source)

[View source]

def run(&block : Crystal::Token -> _) #

Runs the tokenizer and yields each token as a block argument.

Ameba::Tokenizer.new(source).run do |token|
  puts token
end

[View source]