alihan/llama-cpp-python
1
0
Fork 0
mirror of https://github.com/abetlen/llama-cpp-python.git synced 2023-09-07 17:34:22 +03:00
Code Issues Packages Projects Releases Wiki Activity

Use server sent events function for streaming completion

Browse Source
This commit is contained in:
Andrei Betlen
2023-05-19 02:04:30 -04:00
parent f0ec6e615e
commit dc39cc0fa4
1 changed files with 8 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
llama_cpp/server/app.py
View File

@@ -259,8 +259,15 @@ def create_completion(
) )
) )
if request.stream: if request.stream:
async def server_sent_events(
chunks: Iterator[llama_cpp.CompletionChunk],
):
for chunk in chunks:
yield dict(data=json.dumps(chunk))
chunks: Iterator[llama_cpp.CompletionChunk] = completion_or_chunks # type: ignore chunks: Iterator[llama_cpp.CompletionChunk] = completion_or_chunks # type: ignore
return EventSourceResponse(dict(data=json.dumps(chunk)) for chunk in chunks) return EventSourceResponse(server_sent_events(chunks))
completion: llama_cpp.Completion = completion_or_chunks # type: ignore completion: llama_cpp.Completion = completion_or_chunks # type: ignore
return completion return completion