Model

ffbpe.model.BpeModel

BpeModel(model)

An immutable BPE model produced by :meth:BpeTrainer.validate_model.

Source code in python/ffbpe/model.py

def __init__(self, model: BpeModelBase) -> None:
  self._model = model
  self._encoder_cache: BpeEncoder | None = None

unit property

unit

Primary segmentation unit used by this model.

vocab property

vocab

Return a snapshot of the validated token-to-id vocabulary.

last_merge_freq property

last_merge_freq

Frequency of the final pair merge, if the model contains one.

special_tokens property

special_tokens

Reserved special tokens in vocabulary order.

encoder

encoder(*, pat_str=None, unicode_bigrams=None, unicode_bigram_mixed_boundary='keep', split_on_vocab_bigrams=True)

Build an encoder directly from this model.

Source code in python/ffbpe/model.py

def encoder(
  self,
  *,
  pat_str: str | None = None,
  unicode_bigrams: Sequence[str] | None = None,
  unicode_bigram_mixed_boundary: str = "keep",
  split_on_vocab_bigrams: bool = True,
) -> "BpeEncoder":
  """Build an encoder directly from this model."""
  from .encoder import BpeEncoder
  use_cache = (
    pat_str is None
    and unicode_bigrams is None
    and unicode_bigram_mixed_boundary == "keep"
    and split_on_vocab_bigrams
  )
  if use_cache and self._encoder_cache is not None:
    return self._encoder_cache
  encoder = BpeEncoder._from_encoder(
    self.unit,
    self._model.encoder(
      pat_str=pat_str,
      unicode_bigrams=unicode_bigrams,
      unicode_bigram_mixed_boundary=unicode_bigram_mixed_boundary,
      split_on_vocab_bigrams=split_on_vocab_bigrams,
    ),
  )
  if use_cache:
    self._encoder_cache = encoder
  return encoder

encode

encode(text)

Encode text with the model's default pretokenizer.

Source code in python/ffbpe/model.py

def encode(self, text: str) -> list[int]:
  """Encode text with the model's default pretokenizer."""
  return self.encoder().encode(text)

decode

decode(ids)

Decode token ids into text.

Source code in python/ffbpe/model.py

def decode(self, ids: Sequence[int]) -> str:
  """Decode token ids into text."""
  return self.encoder().decode(ids)

save_vocab_json

save_vocab_json(path, *, format=None)

Save the validated vocabulary to a JSON file.

Source code in python/ffbpe/model.py

def save_vocab_json(
  self,
  path: str | PathLike,
  *,
  format: FileFormat | None = None,
) -> None:
  """Save the validated vocabulary to a JSON file."""
  self._model.save_vocab(path, _resolve_format(self.unit, format))

save_merges_txt

save_merges_txt(path, *, format=None)

Save the validated merge list to a text file.

Source code in python/ffbpe/model.py

def save_merges_txt(
  self,
  path: str | PathLike,
  *,
  format: FileFormat | None = None,
) -> None:
  """Save the validated merge list to a text file."""
  self._model.save_merges_txt(path, _resolve_format(self.unit, format))

save

save(name, *, outdir='.', format=None)

Save vocab.{name}[{unit}].json and merges.{name}[{unit}].txt into outdir.

Source code in python/ffbpe/model.py

def save(self, name: str, *, outdir: str | PathLike = ".", format: FileFormat | None = None) -> None:
  """Save `vocab.{name}[{unit}].json` and `merges.{name}[{unit}].txt` into `outdir`."""
  vocab_path = Path(outdir) / f"vocab.{name}[{self.unit}].json"
  merges_path = Path(outdir) / f"merges.{name}[{self.unit}].txt"
  self.save_files(vocab_path, merges_path, format=format)

save_files

save_files(vocab_path, merges_path, *, format=None)

Save the validated vocabulary and merge list to explicit paths.

Source code in python/ffbpe/model.py

def save_files(
  self,
  vocab_path: str | PathLike,
  merges_path: str | PathLike,
  *,
  format: FileFormat | None = None,
) -> None:
  """Save the validated vocabulary and merge list to explicit paths."""
  resolved_format = _resolve_format(self.unit, format)
  self._model.save_vocab(vocab_path, resolved_format)
  self._model.save_merges_txt(merges_path, resolved_format)

save_pretrained

save_pretrained(directory, *, format=None, pat_str=None, unicode_bigrams=None, unicode_bigram_mixed_boundary='keep', split_on_vocab_bigrams=True)

Save a self-describing model directory loadable by BpeEncoder.from_pretrained.

Source code in python/ffbpe/model.py

def save_pretrained(
  self,
  directory: str | PathLike,
  *,
  format: FileFormat | None = None,
  pat_str: str | None = None,
  unicode_bigrams: Sequence[str] | None = None,
  unicode_bigram_mixed_boundary: str = "keep",
  split_on_vocab_bigrams: bool = True,
) -> None:
  """Save a self-describing model directory loadable by `BpeEncoder.from_pretrained`."""
  # Validate the complete encoding configuration before creating partial output.
  self.encoder(
    pat_str=pat_str,
    unicode_bigrams=unicode_bigrams,
    unicode_bigram_mixed_boundary=unicode_bigram_mixed_boundary,
    split_on_vocab_bigrams=split_on_vocab_bigrams,
  )
  output_dir = Path(directory)
  output_dir.mkdir(parents=True, exist_ok=True)
  resolved_format = _resolve_format(self.unit, format)
  vocab_file = "vocab.json"
  merges_file = "merges.txt"
  self.save_files(
    output_dir / vocab_file,
    output_dir / merges_file,
    format=resolved_format,
  )
  config: ModelConfig = {
    "version": MODEL_CONFIG_VERSION,
    "unit": self.unit,
    "format": resolved_format,
    "vocab_file": vocab_file,
    "merges_file": merges_file,
    "special_tokens": self.special_tokens,
    "pat_str": pat_str,
    "unicode_bigrams": list(unicode_bigrams) if unicode_bigrams is not None else None,
    "unicode_bigram_mixed_boundary": unicode_bigram_mixed_boundary,
    "split_on_vocab_bigrams": split_on_vocab_bigrams,
  }
  write_model_config(output_dir / MODEL_CONFIG_FILENAME, config)