alihan/exo
1
0
Fork 0
mirror of https://github.com/exo-explore/exo.git synced 2025-10-23 02:57:14 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix tokenizers test with restructured models

Browse Source
This commit is contained in:
Alex Cheema
2024-11-13 17:06:11 +04:00
parent 65d2ae0287
commit 34f3c4a155
1 changed files with 7 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
test/test_tokenizers.py
View File

@@ -1,7 +1,7 @@
import os
import re
from transformers import AutoTokenizer, AutoProcessor
from exo.models import model_base_shards
from exo.models import model_cards
def test_tokenizer(name, tokenizer, verbose=False):
@@ -26,7 +26,12 @@ def test_tokenizer(name, tokenizer, verbose=False):
ignore = ["TriAiExperiments/SFR-Iterative-DPO-LLaMA-3-70B-R", "mlx-community/DeepSeek-Coder-V2-Lite-Instruct-4bit-mlx", "mlx-community/DeepSeek-V2.5-MLX-AQ4_1_64", "llava-hf/llava-1.5-7b-hf", "mlx-community/Qwen*", "dummy", "mlx-community/Meta-Llama-3.1-405B-Instruct-8bit"]
ignore_pattern = re.compile(r"^(" + "|".join(model.replace("*", ".*") for model in ignore) + r")")
models = [shard.model_id for shards in model_base_shards.values() for shard in shards.values() if not ignore_pattern.match(shard.model_id)]
models = []
for model_id in model_cards:
for engine_type, repo_id in model_cards[model_id].get("repo", {}).items():
if not ignore_pattern.match(repo_id):
models.append(repo_id)
models = list(set(models))
verbose = os.environ.get("VERBOSE", "0").lower() == "1"
for m in models: