Feat: support stream_chat in hf (#131)

* Feat: support stream_chat in hf

* update docstring
This commit is contained in:
liujiangning30
2024-01-31 20:37:16 +08:00
committed by GitHub
parent f887b423fa
commit 190fb731be

View File

@@ -297,6 +297,23 @@ class HFTransformer(BaseModel):
response = [response]
yield ModelStatusCode.SERVER_ERR, response, None
def stream_chat(
self,
inputs: List[dict],
do_sample: bool = True,
**kwargs,
):
"""Return the chat completions in stream mode.
Args:
inputs (List[dict]): input messages to be completed.
do_sample (bool): do sampling if enabled
Returns:
the text/chat completion
"""
prompt = self.template_parser(inputs)
yield from self.stream_generate(prompt, do_sample, **kwargs)
class HFTransformerCasualLM(HFTransformer):