Merge pull request #178 from MadcowD/wguss/vllm_streamfix_0

vllm stream fix
This commit is contained in:
William Guss
2024-09-18 21:18:23 -07:00
committed by GitHub

View File

@@ -149,16 +149,16 @@ try:
response = [call_result.response]
else:
response = call_result.response
for chunk in response:
if hasattr(chunk, "usage") and chunk.usage:
metadata = chunk.to_dict()
if call_result.actual_streaming:
continue
for choice in chunk.choices:
choices_progress[choice.index].append(choice)
if choice.index == 0 and logger:
# print(choice, streaming)
logger(choice.delta.content if call_result.actual_streaming else