class Cadmium::Token
- Cadmium::Token
- Reference
- Object
Overview
An individual token — i.e. a word, punctuation symbol, whitespace, etc
Defined in:
cadmium/token.crConstructors
Instance Method Summary
-
#is_base_form? : Bool
ameba:disable Metrics/CyclomaticComplexity
- #is_oov : Bool
- #is_oov=(is_oov : Bool)
- #is_punctuation : Bool
- #is_punctuation=(is_punctuation : Bool)
- #is_start_sentence : Bool
- #is_start_sentence=(is_start_sentence : Bool)
- #is_stop_word : Bool
- #is_stop_word=(is_stop_word : Bool)
- #language : Symbol
- #language=(language : Symbol)
- #lemma : String | Nil
- #lemma=(lemma : String | Nil)
- #morphology : Symbol | String | Nil
- #morphology=(morphology : Symbol | String | Nil)
- #ner_tag : String | Nil
- #ner_tag=(ner_tag : String | Nil)
- #pos : String | Nil
- #pos=(pos : String | Nil)
- #size
- #token_vector : Apatite::Vector(Float64) | Nil
- #token_vector=(token_vector : Apatite::Vector(Float64) | Nil)
- #univ_pos : Symbol | Nil
- #univ_pos=(univ_pos : Symbol | Nil)
- #verbatim : String
- #verbatim=(verbatim : String)
Constructor Detail
def self.new(verbatim : String = "", pos : Nil | String = "", univ_pos : Symbol | Nil = :none, morphology : String | Symbol | Nil = :none, is_start_sentence : Bool = false, is_punctuation : Bool = false, language : Symbol = :en)
#