Encoder

ffbpe.encoder.BpeEncoder

BpeEncoder(unit='byte', *, special_tokens=None, merges=None, vocab=None, pat_str=None, unicode_bigrams=None, unicode_bigram_mixed_boundary='keep', split_on_vocab_bigrams=True)

BPE encoder.

This is a thin Python wrapper around the Rust implementation.

Parameters:

Name	Type	Description	Default
unit	Unit	Primary segmentation unit. Unicode models may include UTF-8 byte fallback merges.	'byte'
special_tokens	Sequence[str] | None	Optional list of special tokens. When provided, they are treated as indivisible tokens.	None
merges	list[tuple[bytes, bytes]] | None	In-memory merge rules and vocabulary. If omitted, use :meth:load to load from files.	None
split_on_vocab_bigrams	bool	Whether encoding may partition PAT words using bigrams derived from the model vocabulary. Disable it for byte models when that optimization is slower for the workload.	True

Source code in python/ffbpe/encoder.py

def __init__(
    self,
    unit: "Unit" = "byte",
    *,
    special_tokens: Sequence[str] | None = None,
    merges: list[tuple[bytes, bytes]] | None = None,
    vocab: dict[bytes, int] | None = None,
    pat_str: str | None = None,
    unicode_bigrams: Sequence[str] | None = None,
    unicode_bigram_mixed_boundary: str = "keep",
    split_on_vocab_bigrams: bool = True,
) -> None:
  _validate_unit(unit)
  self.unit = unit
  file_format = _resolve_format(unit, None)
  self._encoder = BpeEncoderBase(
    format=file_format,
    unit=unit,
    merges_file=None,
    vocab_file=None,
    merges=merges,
    vocab=cast(dict[Sequence[int], int], vocab),
    special_tokens=special_tokens,
    pat_str=pat_str,
    unicode_bigrams=unicode_bigrams,
    unicode_bigram_mixed_boundary=unicode_bigram_mixed_boundary,
    split_on_vocab_bigrams=split_on_vocab_bigrams,
  )

load classmethod

load(name=None, *, unit='byte', format=None, special_tokens=None, input_dir=None, merges_file=None, vocab_file=None, pat_str=None, unicode_bigrams=None, unicode_bigram_mixed_boundary='keep', split_on_vocab_bigrams=True)

Load an encoder from vocab/merge files.

Parameters:

Name	Type	Description	Default
name	str | None	Optional model name used to derive default filenames: merges.{name}[{unit}].txt and vocab.{name}[{unit}].json.	None
unit	Unit	Primary segmentation unit ("byte" or "unicode").	'byte'
format	FileFormat | None	Override the format used to decode the files ("gpt2" or "unitoken"). If omitted, defaults to "gpt2" for byte units and "unitoken" for Unicode units.	None
special_tokens	Sequence[str] | None	Optional list of special tokens to configure the encoder.	None
input_dir	str | PathLike | None	Optional directory to resolve merges_file/vocab_file relative to.	None
merges_file	str | PathLike | None	Explicit filenames/paths for merges and vocab.	None
pat_str	str | None	Optional pretokenizer regex.	None
unicode_bigrams	Sequence[str] | None	Optional retained Unicode bigrams used to shape pretokenizer boundaries.	None
unicode_bigram_mixed_boundary	str	Mixed-boundary policy: "keep" or "split".	'keep'
split_on_vocab_bigrams	bool	Whether encoding may partition PAT words using model-vocabulary bigrams. Disable it for byte models when benchmarking shows no benefit.	True

Source code in python/ffbpe/encoder.py

@classmethod
def load(
  cls,
  name: str | None = None,
  *,
  unit: "Unit" = "byte",
  format: "FileFormat | None" = None,
  special_tokens: Sequence[str] | None = None,
  input_dir: str | PathLike | None = None,
  merges_file: str | PathLike | None = None,
  vocab_file: str | PathLike | None = None,
  pat_str: str | None = None,
  unicode_bigrams: Sequence[str] | None = None,
  unicode_bigram_mixed_boundary: str = "keep",
  split_on_vocab_bigrams: bool = True,
) -> "BpeEncoder":
  """Load an encoder from vocab/merge files.

  Parameters
  ----------
  name:
      Optional model name used to derive default filenames:
      `merges.{name}[{unit}].txt` and `vocab.{name}[{unit}].json`.
  unit:
      Primary segmentation unit (`"byte"` or `"unicode"`).
  format:
      Override the format used to decode the files (`"gpt2"` or
      `"unitoken"`). If omitted, defaults to `"gpt2"` for byte units and
      `"unitoken"` for Unicode units.
  special_tokens:
      Optional list of special tokens to configure the encoder.
  input_dir:
      Optional directory to resolve `merges_file`/`vocab_file` relative to.
  merges_file / vocab_file:
      Explicit filenames/paths for merges and vocab.
  pat_str:
      Optional pretokenizer regex.
  unicode_bigrams:
      Optional retained Unicode bigrams used to shape pretokenizer boundaries.
  unicode_bigram_mixed_boundary:
      Mixed-boundary policy: `"keep"` or `"split"`.
  split_on_vocab_bigrams:
      Whether encoding may partition PAT words using model-vocabulary bigrams.
      Disable it for byte models when benchmarking shows no benefit.
  """
  resolved_format = _resolve_format(unit, format)
  if name is not None:
    if merges_file is None:
      merges_file = f"merges.{name}[{unit}].txt"
    if vocab_file is None:
      vocab_file = f"vocab.{name}[{unit}].json"
  if input_dir is not None:
    if merges_file is not None:
      merges_file = Path(input_dir) / merges_file
    if vocab_file is not None:
      vocab_file = Path(input_dir) / vocab_file
  return cls._from_encoder(
    unit,
    BpeEncoderBase(
      format=resolved_format,
      unit=unit,
      merges_file=merges_file,
      vocab_file=vocab_file,
      merges=None,
      vocab=None,
      special_tokens=special_tokens,
      pat_str=pat_str,
      unicode_bigrams=unicode_bigrams,
      unicode_bigram_mixed_boundary=unicode_bigram_mixed_boundary,
      split_on_vocab_bigrams=split_on_vocab_bigrams,
    ),
  )

from_pretrained classmethod

from_pretrained(directory)

Load an encoder from a directory created by BpeModel.save_pretrained.

Source code in python/ffbpe/encoder.py

@classmethod
def from_pretrained(cls, directory: str | PathLike) -> "BpeEncoder":
  """Load an encoder from a directory created by `BpeModel.save_pretrained`."""
  input_dir = Path(directory)
  config = read_model_config(input_dir)
  return cls.load(
    unit=config["unit"],
    format=config["format"],
    special_tokens=config["special_tokens"],
    merges_file=input_dir / config["merges_file"],
    vocab_file=input_dir / config["vocab_file"],
    pat_str=config["pat_str"],
    unicode_bigrams=config["unicode_bigrams"],
    unicode_bigram_mixed_boundary=config["unicode_bigram_mixed_boundary"],
    split_on_vocab_bigrams=config["split_on_vocab_bigrams"],
  )

encode_word

encode_word(word)

Encode one already-pretokenized word without PAT or special-token handling.

Source code in python/ffbpe/encoder.py

def encode_word(self, /, word: str) -> list[int]:
  """Encode one already-pretokenized word without PAT or special-token handling."""
  return self._encoder.encode_word(word)

encode_words

encode_words(words)

Encode multiple already-pretokenized words into token ids.

Source code in python/ffbpe/encoder.py

def encode_words(self, /, words: Sequence[str]) -> list[list[int]]:
  """Encode multiple already-pretokenized words into token ids."""
  return self._encoder.encode_words(words)

encode

encode(text)

Encode text into a Python list of token ids.

Source code in python/ffbpe/encoder.py

def encode(self, /, text: str) -> list[int]:
  """Encode text into a Python list of token ids."""
  return self._encoder.encode(text)

encode_to_numpy

encode_to_numpy(text)

Encode text into a NumPy array of token ids.

Source code in python/ffbpe/encoder.py

def encode_to_numpy(self, /, text: str) -> IdxArray:
  """Encode text into a NumPy array of token ids."""
  return self._encoder.encode_to_numpy(text)

encode_file

encode_file(path, num_chunks=1024)

Encode a text file into a NumPy array of token ids.

Parameters:

Name	Type	Description	Default
path	str | PathLike	Path to a UTF-8 text file.	required
num_chunks	int	Number of chunks to split the file into. Chunk boundaries are aligned on the end-of-text token.	1024

Source code in python/ffbpe/encoder.py

def encode_file(self, /, path: str | PathLike, num_chunks: int = 1024) -> IdxArray:
  """Encode a text file into a NumPy array of token ids.

  Parameters
  ----------
  path:
      Path to a UTF-8 text file.
  num_chunks:
      Number of chunks to split the file into. Chunk boundaries are aligned on
      the end-of-text token.
  """
  return self._encoder.encode_file(path, num_chunks)

decode

decode(idxs)

Decode token ids back into a UTF-8 string.

Source code in python/ffbpe/encoder.py

def decode(self, /, idxs: Sequence[int] | IdxArray) -> str:
  """Decode token ids back into a UTF-8 string."""
  return self._encoder.decode(idxs)