alihan/exo
1
0
Fork 0
mirror of https://github.com/exo-explore/exo.git synced 2025-10-23 02:57:14 +03:00
Code Issues Packages Projects Releases Wiki Activity

TODO for why use_fast=False is giving inconsistent behaviour (no spaces decoding invididual tokens) for Mistral-Large-Instruct-2407-4bit

Browse Source
This commit is contained in:
Alex Cheema
2024-08-22 14:45:08 +01:00
parent e17e5f9a41
commit 710e5a31e7
1 changed files with 1 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
test/test_mistral_tokenizer.py
View File

@@ -39,6 +39,7 @@ models = [
import os
verbose = os.environ.get("VERBOSE", "0").lower() == "1"
for m in models:
# TODO: figure out why use_fast=False is giving inconsistent behaviour (no spaces decoding invididual tokens) for Mistral-Large-Instruct-2407-4bit
# test_tokenizer(m, AutoProcessor.from_pretrained(m, use_fast=False), verbose)
test_tokenizer(m, AutoProcessor.from_pretrained(m, use_fast=True), verbose)
test_tokenizer(m, AutoTokenizer.from_pretrained(m), verbose)