class KDL::Tokenizer

• KDL::Tokenizer
• Reference
• Object

Defined in:

kdl/tokenizer.cr

Constant Summary

ALLOWED_IN_TYPE = [Context::Ident, Context::String, Context::Rawstring, Context::MultiLineComment, Context::Whitespace]

EQUALS = ['=', '﹦', '＝', '🟰']

FORBIDDEN = [*'\0'..'\b', *'\u000E'..'\u001F', '\u007F', *'\u200E'..'\u200F', *'\u202A'..'\u202E', *'\u2066'..'\u2069', '\uFEFF']

IDENTIFER_CHARS = /[^#{NON_IDENTIFIER_CHARS}\0-\x20]/

INITIAL_IDENTIFIER_CHARS = /[^#{NON_IDENTIFIER_CHARS}0-9\x0-\x20]/

NEWLINES = ['\n', '\u0085', '\f', '\u2028', '\u2029']

NON_IDENTIFIER_CHARS = Regex.escape("#{SYMBOLS.keys.join("")}()[]/\\\"# ")

NOT_ALLOWED_AFTER_TYPE = [Context::SingleLineComment]

SYMBOLS = {'{' => Token::Type::LBRACE, '}' => Token::Type::RBRACE, ';' => Token::Type::SEMICOLON}.merge(EQUALS.map do |e| {e, Token::Type::EQUALS} end.to_h)

WHITESPACE = ['\t', '\v', ' ', '\u00A0', '\u1680', '\u2000', '\u2001', '\u2002', '\u2003', '\u2004', '\u2005', '\u2006', '\u2007', '\u2008', '\u2009', '\u200A', '\u202F', '\u205F', '\u3000']

Constructors

• .new(str : String, start : Int32 = 0)

Instance Method Summary

• #[](i)
• #context : KDL::Tokenizer::Context?
• #done? : Bool
• #index : Int32
• #next_token
• #peek_token
• #peek_token_after_next
• #read_next_token
• #reset
• #tokens

Constructor Detail

Instance Method Detail