alihan/llm-text-generation-inference
1
0
Fork 0
mirror of https://github.com/huggingface/text-generation-inference.git synced 2023-08-15 01:09:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix(server): fix bnb quantization for CausalLM models (#385)

Browse Source
This commit is contained in:
OlivierDehaene
2023-05-31 11:48:28 +02:00
committed by GitHub
parent 87dc034b59
commit 337afb2842
5 changed files with 15 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/text_generation_server/models/bloom.py
View File

@@ -245,6 +245,8 @@ class BLOOMSharded(BLOOM):
return linear
module.linear = replace_linear(state)
else:
tensor = tensor.to(device)
elif quantize == "gptq":
raise NotImplementedError("`gptq` is not implemented for now")
elif quantize is None:

2
server/text_generation_server/models/galactica.py
View File

@@ -364,6 +364,8 @@ class GalacticaSharded(Galactica):
return linear
module.linear = replace_linear(state)
else:
tensor = tensor.to(device)
elif quantize == "gptq":
raise NotImplementedError("`gptq` is not implemented for now")
elif quantize is None:

2
server/text_generation_server/models/gpt_neox.py
View File

@@ -210,6 +210,8 @@ class GPTNeoxSharded(CausalLM):
return linear
module.linear = replace_linear(state)
else:
tensor = tensor.to(device)
elif quantize == "gptq":
raise NotImplementedError("`gptq` is not implemented for now")
elif quantize is None:

9
server/text_generation_server/models/opt.py
View File

@@ -166,7 +166,7 @@ class OPTSharded(OPT):
tensor = tensor.contiguous().to(dtype)
if quantize:
if quantize == "bitsandbytes":
if not HAS_BITS_AND_BYTES:
raise ImportError(
"bitsandbytes is not available on your machine either because it is not installed "
@@ -216,9 +216,14 @@ class OPTSharded(OPT):
return linear
module.linear = replace_linear(state)
else:
tensor = tensor.to(device)
elif quantize == "gptq":
raise NotImplementedError("`gptq` is not implemented for now")
elif quantize is None:
tensor = tensor.to(device)
else:
raise ValueError(f"Unexpected quantize `{quantize}`")
module._parameters[param_name] = tensor
if name == "model.decoder.embed_tokens.weight":

3
server/text_generation_server/models/t5.py
View File

@@ -222,7 +222,8 @@ class T5Sharded(Seq2SeqLM):
return linear
module.linear = replace_linear(state)
else:
tensor = tensor.to(device)
elif quantize == "gptq" and not module_name.endswith("wo"):
raise NotImplementedError("`gptq` is not implemented for now")
elif quantize is None or module_name.endswith("wo"):