class KDL::Tokenizer

• KDL::Tokenizer
• Reference
• Object

Defined in:

kdl/tokenizer.cr

Constant Summary

FORBIDDEN = [*'\0'..'\b', *'\u000E'..'\u001F', '\u007F', *'\u200E'..'\u200F', *'\u202A'..'\u202E', *'\u2066'..'\u2069', '\uFEFF']

IDENTIFER_CHARS = /[^#{NON_IDENTIFIER_CHARS}]/

INITIAL_IDENTIFIER_CHARS = /[^#{NON_IDENTIFIER_CHARS}0-9]/

NEWLINES = ['\n', '\u0085', '\v', '\f', '\u2028', '\u2029']

NEWLINES_PATTERN = /(#{NEWLINES.map do |c| Regex.escape(c) end.join('|')}|\r\n?)/

NON_IDENTIFIER_CHARS = Regex.escape("#{SYMBOLS.keys.join("")}()[]/\\\"##{WHITESPACE.join}#{OTHER_NON_IDENTIFIER_CHARS.join}")

OTHER_NON_IDENTIFIER_CHARS = ('\0'..' ').to_a - WHITESPACE

SYMBOLS = {'{' => Token::Type::LBRACE, '}' => Token::Type::RBRACE, ';' => Token::Type::SEMICOLON, '=' => Token::Type::EQUALS}

UNESCAPE = /\\(?:[#{WHITESPACE.join}#{NEWLINES.join}\r]+|[^u])/

UNESCAPE_NON_WS = /\\(?:[^u])/

VERSION_PATTERN = /\A\/-[#{WHITESPACE.join}]*kdl-version[#{WHITESPACE.join}]+(\d+)[#{WHITESPACE.join}]*[#{NEWLINES.join}]/

WHITESPACE = ['\t', ' ', '\u00A0', '\u1680', '\u2000', '\u2001', '\u2002', '\u2003', '\u2004', '\u2005', '\u2006', '\u2007', '\u2008', '\u2009', '\u200A', '\u202F', '\u205F', '\u3000']

WS = "[#{Regex.escape(WHITESPACE.join)}]"

WS_PLUS = /\A#{WS}+\z/

WS_STAR = /\A#{WS}*\z/

Constructors

• .new(str : String, start : Int32 = 0)

Instance Method Summary

• #[](i)
• #column : Int32
• #column=(column : Int32)
• #comment : String
• #comment=(comment : String)
• #context : KDL::Tokenizer::Context?
• #done? : Bool
• #has_comment : Bool
• #has_comment=(has_comment : Bool)
• #index : Int32
• #line : Int32
• #line=(line : Int32)
• #next_token
• #peek_token
• #peek_token_after_next
• #read_next_token
• #reset
• #tokens
• #version_directive

Constructor Detail

Instance Method Detail