AI-Red-Teaming-CSCD94/application-and-system-attacks/sponge-genetic-algo/Tokenizer.ipynb

In [ ]:

from transformers import AutoTokenizer
import json

model = 'openai-community/gpt2'

while 1:
	text = input("> ")

	tokens = AutoTokenizer.from_pretrained(model).tokenize(text)
	print(f"Number of Input Characters: {len(text)}")
	print(f"Number of Tokens: {len(tokens)}")
	print(json.dumps(tokens, indent=2))

>  hello im jeremy

Number of Input Characters: 15
Number of Tokens: 5
[
  "hello",
  "\u0120im",
  "\u0120j",
  "ere",
  "my"
]

/home/jeremy/.conda/envs/ai/lib/python3.11/site-packages/huggingface_hub/file_download.py:943: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.
  warnings.warn(

>  safhawepoiefj

Number of Input Characters: 13
Number of Tokens: 6
[
  "saf",
  "haw",
  "ep",
  "o",
  "ief",
  "j"
]

In [ ]: