Struct ferritin_featurizers::ProteinTokenizer

pub struct ProteinTokenizer { /* private fields */ }

Implementations

impl ProteinTokenizer

pub fn new<P: AsRef<Path>>(tokenizer_path: P) -> Result<Self>

pub fn len(&self) -> usize

pub fn token_to_id(&self, token: &str) -> u32

pub fn id_to_token(&self, id: u32) -> String

pub fn encode( &self, tokens: &[String], max_length: Option<usize>, add_special_tokens: bool, random_truncate: bool, ) -> Result<Tensor>

pub fn decode( &self, token_ids: &[u32], skip_special_tokens: bool, ) -> Result<String>