Skip to content

Pretokenizer and counters

Pretokenizer

ffbpe.pretokenizer.PreTokenizer 

PreTokenizer(special_tokens, eot_token=None, pat_str=None, unicode_bigrams=None, unicode_bigram_mixed_boundary='keep')
Source code in python/ffbpe/pretokenizer.py 
20
21
22
23
24
25
26
27
28
29
30
31
32
33
def __init__(
  self,
  special_tokens: Sequence[str],
  eot_token: str | None = None,
  pat_str: str | None = None,
  unicode_bigrams: Sequence[str] | None = None,
  unicode_bigram_mixed_boundary: UnicodeBigramMixedBoundary = "keep",
) -> None:
  self._special_tokens = list(special_tokens)
  self._eot_token = eot_token
  self._pat_str = pat_str
  self._unicode_bigrams = list(unicode_bigrams) if unicode_bigrams is not None else None
  self._unicode_bigram_mixed_boundary = unicode_bigram_mixed_boundary
  self._inner = _PreTokenizer(special_tokens, eot_token, pat_str, unicode_bigrams, unicode_bigram_mixed_boundary)

with_unicode_bigrams 

with_unicode_bigrams(bigrams)

Return a pretokenizer using a frozen Unicode bigram set.

Source code in python/ffbpe/pretokenizer.py 
35
36
37
38
39
40
41
42
43
def with_unicode_bigrams(self, bigrams: Sequence[str]) -> "PreTokenizer":
  """Return a pretokenizer using a frozen Unicode bigram set."""
  return PreTokenizer(
    self._special_tokens,
    self._eot_token,
    self._pat_str,
    bigrams,
    self._unicode_bigram_mixed_boundary,
  )

bigram_counter 

bigram_counter()

Create an empty mergeable Unicode bigram counter.

Source code in python/ffbpe/pretokenizer.py 
45
46
47
def bigram_counter(self) -> BigramCounter:
  """Create an empty mergeable Unicode bigram counter."""
  return self._inner.bigram_counter()

word_counter 

word_counter()

Create an empty mergeable word counter.

Source code in python/ffbpe/pretokenizer.py 
49
50
51
def word_counter(self) -> WordCounter:
  """Create an empty mergeable word counter."""
  return self._inner.word_counter()

load_word_counter 

load_word_counter(path)

Load an exact native word inventory saved by WordCounter.save.

Source code in python/ffbpe/pretokenizer.py 
53
54
55
def load_word_counter(self, path: str | PathLike) -> WordCounter:
  """Load an exact native word inventory saved by `WordCounter.save`."""
  return self._inner.load_word_counter(path)

get_words 

get_words(text)

Pretokenize text and return word frequencies.

Source code in python/ffbpe/pretokenizer.py 
57
58
59
def get_words(self, text: str) -> dict[str, int]:
  """Pretokenize text and return word frequencies."""
  return self._inner.get_words(text)

select_unicode_bigrams_from_file 

select_unicode_bigrams_from_file(path, *, chunk_size=1024 * 1024, boundary='auto', top_k=100000, min_freq=16)

Select Unicode bigrams and retain their effective frequency boundary.

Source code in python/ffbpe/pretokenizer.py 
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
def select_unicode_bigrams_from_file(
  self,
  path: str | PathLike,
  *,
  chunk_size: int = 1024 * 1024,
  boundary: BoundaryMode = "auto",
  top_k: int = 100_000,
  min_freq: int = 16,
) -> UnicodeBigramSelection:
  """Select Unicode bigrams and retain their effective frequency boundary."""
  return self._inner.select_unicode_bigrams_from_file(
    path,
    chunk_size=chunk_size,
    boundary=boundary,
    top_k=top_k,
    min_freq=min_freq,
  )

Unicode bigram counter

ffbpe._lib.BigramCounter

Unicode bigram selection

ffbpe._lib.UnicodeBigramSelection

Word counter

ffbpe._lib.WordCounter