ferritin_plms::esm::tokenization::sequence_tokenizer

Trait EsmTokenizerBase

source 
pub trait EsmTokenizerBase {

Show 14 methods    // Required methods
    fn encode(&self) -> Result<()>;
    fn decode(&self) -> Result<()>;
    fn mask_token(&self) -> &str;
    fn mask_token_id(&self) -> u32;
    fn bos_token(&self) -> &str;
    fn bos_token_id(&self) -> u32;
    fn eos_token(&self) -> &str;
    fn eos_token_id(&self) -> u32;
    fn pad_token(&self) -> &str;
    fn pad_token_id(&self) -> u32;
    fn chain_break_token(&self) -> &str;
    fn chain_break_token_id(&self) -> u32;
    fn all_token_ids(&self) -> Vec<u32>;
    fn special_token_ids(&self) -> Vec<u32>;

}

Required Methods§

source

fn encode(&self) -> Result<()>

source

fn decode(&self) -> Result<()>

source

fn mask_token(&self) -> &str

source

fn mask_token_id(&self) -> u32

source

fn bos_token(&self) -> &str

source

fn bos_token_id(&self) -> u32

source

fn eos_token(&self) -> &str

source

fn eos_token_id(&self) -> u32

source

fn pad_token(&self) -> &str

source

fn pad_token_id(&self) -> u32

source

fn chain_break_token(&self) -> &str

source

fn chain_break_token_id(&self) -> u32

source

fn all_token_ids(&self) -> Vec<u32>

source

fn special_token_ids(&self) -> Vec<u32>

Implementors§