Better streaming

- Always stream the visible scenarios, if the modelProvider supports it
 - Never stream the invisible scenarios

Also actually runs our query tasks in a background worker, which we weren't quite doing before.
This commit is contained in:
Kyle Corbitt
2023-07-24 18:34:30 -07:00
parent d6b97b29f7
commit e1cbeccb90
25 changed files with 152 additions and 153 deletions

View File

@@ -37,7 +37,7 @@ const modelProvider: OpenaiChatModelProvider = {
return null;
},
inputSchema: inputSchema as JSONSchema4,
shouldStream: (input) => input.stream ?? false,
canStream: true,
getCompletion,
...frontendModelProvider,
};