Streaming works for normal text

src/server/utils/generateChannelId.ts

@@ -0,0 +1,5 @@
+// generate random channel id
+
+export const generateChannelId = () => {
+  return Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15);
+};

src/server/utils/getChatCompletion.ts

@@ -1,6 +1,10 @@
+/* eslint-disable @typescript-eslint/no-unsafe-call */
 import { isObject } from "lodash";
 import { type JSONSerializable } from "../types";
 import { Prisma } from "@prisma/client";
+import { streamChatCompletion } from "./openai";
+import { wsConnection } from "~/utils/wsConnection";
+import { type ChatCompletion, type CompletionCreateParams } from "openai/resources/chat";
 
 type CompletionResponse = {
   output: Prisma.InputJsonValue | typeof Prisma.JsonNull;
@@ -11,7 +15,8 @@ type CompletionResponse = {
 
 export async function getChatCompletion(
   payload: JSONSerializable,
-  apiKey: string
+  apiKey: string,
+  channelId?: string,
 ): Promise<CompletionResponse> {
   const start = Date.now();
   const response = await fetch("https://api.openai.com/v1/chat/completions", {
@@ -31,8 +36,21 @@ export async function getChatCompletion(
   };
 
   try {
-    resp.timeToComplete = Date.now() - start;
-    resp.output = await response.json();
+    if (channelId) {
+      const completion = streamChatCompletion(payload as unknown as CompletionCreateParams);
+      let finalOutput: ChatCompletion | null = null;
+      await (async () => {
+        for await (const partialCompletion of completion) {
+          finalOutput = partialCompletion
+          wsConnection.emit("message", { channel: channelId, payload: partialCompletion });
+        }
+      })().catch((err) => console.error(err));
+      resp.output = finalOutput as unknown as Prisma.InputJsonValue;
+      resp.timeToComplete = Date.now() - start;
+    } else {
+      resp.timeToComplete = Date.now() - start;
+      resp.output = await response.json();
+    }
 
     if (!response.ok) {
       // If it's an object, try to get the error message