tiger_lib/
special_tokens.rs

1use crate::helpers::TigerHashSet;
2use crate::token::{Token, TokenIdentity};
3
4#[derive(Debug, Clone)]
5pub struct SpecialTokens {
6    st: Option<TigerHashSet<TokenIdentity>>,
7}
8
9impl SpecialTokens {
10    pub fn none() -> Self {
11        SpecialTokens { st: None }
12    }
13
14    #[allow(dead_code)]
15    pub fn empty() -> Self {
16        SpecialTokens { st: Some(TigerHashSet::default()) }
17    }
18
19    #[allow(dead_code)]
20    pub fn iter(&self) -> impl Iterator<Item = &Token> {
21        self.st.iter().flat_map(|set| set.iter().map(TokenIdentity::inner))
22    }
23
24    #[allow(dead_code)]
25    pub fn into_iter(self) -> impl IntoIterator<Item = Token> {
26        self.st.into_iter().flat_map(|set| set.into_iter().map(TokenIdentity::into_inner))
27    }
28
29    pub fn insert<T: Into<Token>>(&mut self, token: T) {
30        if let Some(ref mut st) = self.st {
31            st.insert(TokenIdentity::new(token.into()));
32        }
33    }
34
35    pub fn merge(&mut self, other: &Self) {
36        for token in other.iter() {
37            self.insert(token);
38        }
39    }
40}
tiger_lib/special_tokens.rs

tiger_lib/
special_tokens.rs
