alihan/exo
1
0
Fork 0
mirror of https://github.com/exo-explore/exo.git synced 2025-10-23 02:57:14 +03:00
Code Issues Packages Projects Releases Wiki Activity

undo diff

Browse Source
This commit is contained in:
Rory Clear
2024-11-27 16:42:19 +00:00
parent 1d1fa8c608
commit 3c81845ab7
1 changed files with 1 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
exo/inference/tinygrad/inference.py
View File

@@ -51,6 +51,7 @@ def build_transformer(model_path: Path, shard: Shard, model_size="8B", device=No
weights = load(str(model_path), shard) weights = load(str(model_path), shard)
weights = convert_from_huggingface(weights, model, MODEL_PARAMS[model_size]["args"]["n_heads"], MODEL_PARAMS[model_size]["args"]["n_kv_heads"]) weights = convert_from_huggingface(weights, model, MODEL_PARAMS[model_size]["args"]["n_heads"], MODEL_PARAMS[model_size]["args"]["n_kv_heads"])
weights = fix_bf16(weights) weights = fix_bf16(weights)
with Context(BEAM=0): with Context(BEAM=0):
# replace weights in model # replace weights in model
load_state_dict(model, weights, strict=False, consume=False) # consume=True load_state_dict(model, weights, strict=False, consume=False) # consume=True