Use javascript functions for prompt completions instead of templated json

src/server/utils/constructPrompt.test.ts

@@ -0,0 +1,17 @@
+import { test } from "vitest";
+import { constructPrompt } from "./constructPrompt";
+
+test.skip("constructPrompt", async () => {
+  const constructed = await constructPrompt(
+    {
+      constructFn: `prompt = { "fooz": "bar" }`,
+    },
+    {
+      variableValues: {
+        foo: "bar",
+      },
+    },
+  );
+
+  console.log(constructed);
+});

src/server/utils/constructPrompt.ts

@@ -0,0 +1,37 @@
+import { type PromptVariant, type TestScenario } from "@prisma/client";
+import ivm from "isolated-vm";
+import { type JSONSerializable } from "../types";
+
+const isolate = new ivm.Isolate({ memoryLimit: 128 });
+
+export async function constructPrompt(
+  variant: Pick<PromptVariant, "constructFn">,
+  testScenario: Pick<TestScenario, "variableValues">,
+): Promise<JSONSerializable> {
+  const scenario = testScenario.variableValues as JSONSerializable;
+
+  const code = `
+      const scenario = ${JSON.stringify(scenario, null, 2)};
+      let prompt
+
+      ${variant.constructFn}
+
+      global.prompt = prompt;
+      `;
+
+  console.log("code is", code);
+
+  const context = await isolate.createContext();
+
+  const jail = context.global;
+  await jail.set("global", jail.derefInto());
+
+  const script = await isolate.compileScript(code);
+
+  await script.run(context);
+  const promptReference = (await context.global.get("prompt")) as ivm.Reference;
+
+  const prompt = await promptReference.copy(); // Get the actual value from the isolate
+
+  return prompt as JSONSerializable;
+}

src/server/utils/evaluations.ts

@@ -13,7 +13,11 @@ export const reevaluateVariant = async (variantId: string) => {
   });
 
   const modelOutputs = await prisma.modelOutput.findMany({
-    where: { promptVariantId: variantId, statusCode: { notIn: [429] }, testScenario: { visible: true } },
+    where: {
+      promptVariantId: variantId,
+      statusCode: { notIn: [429] },
+      testScenario: { visible: true },
+    },
     include: { testScenario: true },
   });
 
@@ -96,4 +100,4 @@ export const reevaluateAll = async (experimentId: string) => {
   });
 
   await Promise.all(evaluations.map(reevaluateEvaluation));
-}
+};

src/server/utils/openai.ts

@@ -8,8 +8,6 @@ import {
   type CompletionCreateParams,
 } from "openai/resources/chat";
 
-// console.log("creating openai client");
-
 export const openai = new OpenAI({ apiKey: env.OPENAI_API_KEY });
 
 export const mergeStreamedChunks = (