Better scenario variable editing

Some users have gotten confused by the scenario variable editing interface. This change makes the interface easier to understand.

README.md

@@ -65,7 +65,14 @@ OpenPipe includes a tool to generate new test scenarios based on your existing p
 4. Clone this repository: `git clone https://github.com/openpipe/openpipe`
 5. Install the dependencies: `cd openpipe && pnpm install`
 6. Create a `.env` file (`cp .env.example .env`) and enter your `OPENAI_API_KEY`.
-7. Update `DATABASE_URL` if necessary to point to your Postgres instance and run `pnpm prisma db push` to create the database.
+7. Update `DATABASE_URL` if necessary to point to your Postgres instance and run `pnpm prisma migrate dev` to create the database.
 8. Create a [GitHub OAuth App](https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/creating-an-oauth-app) and update the `GITHUB_CLIENT_ID` and `GITHUB_CLIENT_SECRET` values. (Note: a PR to make auth optional when running locally would be a great contribution!)
 9. Start the app: `pnpm dev`.
 10. Navigate to [http://localhost:3000](http://localhost:3000)
+
+## Testing Locally
+
+1. Copy your `.env` file to `.env.test`.
+2. Update the `DATABASE_URL` to have a different database name than your development one
+3. Run `DATABASE_URL=[your new datatase url] pnpm prisma migrate dev --skip-seed --skip-generate`
+4. Run `pnpm test`

app/.gitignore

@@ -34,6 +34,7 @@ yarn-error.log*
 # do not commit any .env files to git, except for the .env.example file. https://create.t3.gg/en/usage/env-variables#using-environment-variables
 .env
 .env*.local
+.env.test
 
 # vercel
 .vercel

app/package.json

@@ -19,7 +19,7 @@
     "codegen": "tsx src/server/scripts/client-codegen.ts",
     "seed": "tsx prisma/seed.ts",
     "check": "concurrently 'pnpm lint' 'pnpm tsc' 'pnpm prettier . --check'",
-    "test": "pnpm vitest --no-threads"
+    "test": "pnpm vitest"
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.5.8",

app/prisma/migrations/20230810161437_add_uuid_extension/migration.sql

@@ -0,0 +1 @@
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";

app/src/components/OutputsTable/EditEvaluations.tsx

@@ -12,6 +12,7 @@ import {
   Select,
   FormHelperText,
   Code,
+  IconButton,
 } from "@chakra-ui/react";
 import { type Evaluation, EvalType } from "@prisma/client";
 import { useCallback, useState } from "react";
@@ -183,46 +184,37 @@ export default function EditEvaluations() {
                 <Text flex={1}>
                   {evaluation.evalType}: &quot;{evaluation.value}&quot;
                 </Text>
-                <Button
+
+                <IconButton
+                  aria-label="Edit"
                   variant="unstyled"
-                  color="gray.400"
-                  height="unset"
-                  width="unset"
                   minW="unset"
+                  color="gray.400"
                   onClick={() => setEditingId(evaluation.id)}
-                  _hover={{
+                  _hover={{ color: "gray.800", cursor: "pointer" }}
-                    color: "gray.800",
+                  icon={<Icon as={BsPencil} />}
-                    cursor: "pointer",
+                />
-                  }}
+                <IconButton
-                >
+                  aria-label="Delete"
-                  <Icon as={BsPencil} boxSize={4} />
-                </Button>
-                <Button
                   variant="unstyled"
-                  color="gray.400"
-                  height="unset"
-                  width="unset"
                   minW="unset"
+                  color="gray.400"
                   onClick={() => onDelete(evaluation.id)}
-                  _hover={{
+                  _hover={{ color: "gray.800", cursor: "pointer" }}
-                    color: "gray.800",
+                  icon={<Icon as={BsX} boxSize={6} />}
-                    cursor: "pointer",
+                />
-                  }}
-                >
-                  <Icon as={BsX} boxSize={6} />
-                </Button>
               </HStack>
             ),
           )}
           {editingId == null && (
             <Button
               onClick={() => setEditingId("new")}
-              alignSelf="flex-start"
+              alignSelf="end"
               size="sm"
               mt={4}
               colorScheme="blue"
             >
-              Add Evaluation
+              New Evaluation
             </Button>
           )}
           {editingId == "new" && (

app/src/components/OutputsTable/EditScenarioVars.tsx

@@ -1,49 +1,162 @@
-import { Text, Button, HStack, Heading, Icon, Input, Stack } from "@chakra-ui/react";
+import { Text, Button, HStack, Heading, Icon, IconButton, Stack, VStack } from "@chakra-ui/react";
-import { useState } from "react";
+import { type TemplateVariable } from "@prisma/client";
-import { BsCheck, BsX } from "react-icons/bs";
+import { useEffect, useState } from "react";
+import { BsPencil, BsX } from "react-icons/bs";
 import { api } from "~/utils/api";
-import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+import { useExperiment, useHandledAsyncCallback, useScenarioVars } from "~/utils/hooks";
+import { maybeReportError } from "~/utils/standardResponses";
+import { FloatingLabelInput } from "./FloatingLabelInput";
+
+export const ScenarioVar = ({
+  variable,
+  isEditing,
+  setIsEditing,
+}: {
+  variable: Pick<TemplateVariable, "id" | "label">;
+  isEditing: boolean;
+  setIsEditing: (isEditing: boolean) => void;
+}) => {
+  const utils = api.useContext();
+
+  const [label, setLabel] = useState(variable.label);
+
+  useEffect(() => {
+    setLabel(variable.label);
+  }, [variable.label]);
+
+  const renameVarMutation = api.scenarioVars.rename.useMutation();
+  const [onRename] = useHandledAsyncCallback(async () => {
+    const resp = await renameVarMutation.mutateAsync({ id: variable.id, label });
+    if (maybeReportError(resp)) return;
+
+    setIsEditing(false);
+    await utils.scenarioVars.list.invalidate();
+    await utils.scenarios.list.invalidate();
+  }, [label, variable.id]);
+
+  const deleteMutation = api.scenarioVars.delete.useMutation();
+  const [onDeleteVar] = useHandledAsyncCallback(async () => {
+    await deleteMutation.mutateAsync({ id: variable.id });
+    await utils.scenarioVars.list.invalidate();
+  }, [variable.id]);
+
+  if (isEditing) {
+    return (
+      <HStack w="full">
+        <FloatingLabelInput
+          flex={1}
+          label="Renamed Variable"
+          value={label}
+          onChange={(e) => setLabel(e.target.value)}
+          onKeyDown={(e) => {
+            if (e.key === "Enter") {
+              e.preventDefault();
+              onRename();
+            }
+            // If the user types a space, replace it with an underscore
+            if (e.key === " ") {
+              e.preventDefault();
+              setLabel((label) => label && `${label}_`);
+            }
+          }}
+        />
+        <Button size="sm" onClick={() => setIsEditing(false)}>
+          Cancel
+        </Button>
+        <Button size="sm" colorScheme="blue" onClick={onRename}>
+          Save
+        </Button>
+      </HStack>
+    );
+  } else {
+    return (
+      <HStack w="full" borderTopWidth={1} borderColor="gray.200">
+        <Text flex={1}>{variable.label}</Text>
+        <IconButton
+          aria-label="Edit"
+          variant="unstyled"
+          minW="unset"
+          color="gray.400"
+          onClick={() => setIsEditing(true)}
+          _hover={{ color: "gray.800", cursor: "pointer" }}
+          icon={<Icon as={BsPencil} />}
+        />
+        <IconButton
+          aria-label="Delete"
+          variant="unstyled"
+          minW="unset"
+          color="gray.400"
+          onClick={onDeleteVar}
+          _hover={{ color: "gray.800", cursor: "pointer" }}
+          icon={<Icon as={BsX} boxSize={6} />}
+        />
+      </HStack>
+    );
+  }
+};
 
 export default function EditScenarioVars() {
   const experiment = useExperiment();
-  const vars =
+  const vars = useScenarioVars();
-    api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" }).data ?? [];
+
+  const [currentlyEditingId, setCurrentlyEditingId] = useState<string | null>(null);
 
   const [newVariable, setNewVariable] = useState<string>("");
-  const newVarIsValid = newVariable.length > 0 && !vars.map((v) => v.label).includes(newVariable);
+  const newVarIsValid = newVariable?.length ?? 0 > 0;
 
   const utils = api.useContext();
-  const addVarMutation = api.templateVars.create.useMutation();
+  const addVarMutation = api.scenarioVars.create.useMutation();
   const [onAddVar] = useHandledAsyncCallback(async () => {
     if (!experiment.data?.id) return;
-    if (!newVarIsValid) return;
+    if (!newVariable) return;
-    await addVarMutation.mutateAsync({
+    const resp = await addVarMutation.mutateAsync({
       experimentId: experiment.data.id,
       label: newVariable,
     });
-    await utils.templateVars.list.invalidate();
+    if (maybeReportError(resp)) return;
+
+    await utils.scenarioVars.list.invalidate();
     setNewVariable("");
   }, [addVarMutation, experiment.data?.id, newVarIsValid, newVariable]);
 
-  const deleteMutation = api.templateVars.delete.useMutation();
-  const [onDeleteVar] = useHandledAsyncCallback(async (id: string) => {
-    await deleteMutation.mutateAsync({ id });
-    await utils.templateVars.list.invalidate();
-  }, []);
-
   return (
     <Stack>
       <Heading size="sm">Scenario Variables</Heading>
-      <Stack spacing={2}>
+      <VStack spacing={4}>
         <Text fontSize="sm">
           Scenario variables can be used in your prompt variants as well as evaluations.
         </Text>
-        <HStack spacing={0}>
+        <VStack spacing={0} w="full">
-          <Input
+          {vars.data?.map((variable) => (
-            placeholder="Add Scenario Variable"
+            <ScenarioVar
+              variable={variable}
+              key={variable.id}
+              isEditing={currentlyEditingId === variable.id}
+              setIsEditing={(isEditing) => {
+                if (isEditing) {
+                  setCurrentlyEditingId(variable.id);
+                } else {
+                  setCurrentlyEditingId(null);
+                }
+              }}
+            />
+          ))}
+        </VStack>
+        {currentlyEditingId !== "new" && (
+          <Button
+            colorScheme="blue"
             size="sm"
-            borderTopRadius={0}
+            onClick={() => setCurrentlyEditingId("new")}
-            borderRightRadius={0}
+            alignSelf="end"
+          >
+            New Variable
+          </Button>
+        )}
+        {currentlyEditingId === "new" && (
+          <HStack w="full">
+            <FloatingLabelInput
+              flex={1}
+              label="New Variable"
               value={newVariable}
               onChange={(e) => setNewVariable(e.target.value)}
               onKeyDown={(e) => {
@@ -54,50 +167,19 @@ export default function EditScenarioVars() {
                 // If the user types a space, replace it with an underscore
                 if (e.key === " ") {
                   e.preventDefault();
-                setNewVariable((v) => v + "_");
+                  setNewVariable((v) => v && `${v}_`);
                 }
               }}
             />
-          <Button
+            <Button size="sm" onClick={() => setCurrentlyEditingId(null)}>
-            size="xs"
+              Cancel
-            height="100%"
+            </Button>
-            borderLeftRadius={0}
+            <Button size="sm" colorScheme="blue" onClick={onAddVar}>
-            isDisabled={!newVarIsValid}
+              Save
-            onClick={onAddVar}
-          >
-            <Icon as={BsCheck} boxSize={8} />
             </Button>
           </HStack>
-
+        )}
-        <HStack spacing={2} py={4} wrap="wrap">
+      </VStack>
-          {vars.map((variable) => (
-            <HStack
-              key={variable.id}
-              spacing={0}
-              bgColor="blue.100"
-              color="blue.600"
-              pl={2}
-              pr={0}
-              fontWeight="bold"
-            >
-              <Text fontSize="sm" flex={1}>
-                {variable.label}
-              </Text>
-              <Button
-                size="xs"
-                variant="ghost"
-                colorScheme="blue"
-                p="unset"
-                minW="unset"
-                px="unset"
-                onClick={() => onDeleteVar(variable.id)}
-              >
-                <Icon as={BsX} boxSize={6} color="blue.800" />
-              </Button>
-            </HStack>
-          ))}
-        </HStack>
-      </Stack>
     </Stack>
   );
 }

app/src/components/OutputsTable/OutputCell/OutputCell.tsx

@@ -1,7 +1,7 @@
 import { api } from "~/utils/api";
 import { type PromptVariant, type Scenario } from "../types";
 import { type StackProps, Text, VStack } from "@chakra-ui/react";
-import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+import { useScenarioVars, useHandledAsyncCallback } from "~/utils/hooks";
 import SyntaxHighlighter from "react-syntax-highlighter";
 import { docco } from "react-syntax-highlighter/dist/cjs/styles/hljs";
 import stringify from "json-stringify-pretty-compact";
@@ -23,10 +23,7 @@ export default function OutputCell({
   variant: PromptVariant;
 }): ReactElement | null {
   const utils = api.useContext();
-  const experiment = useExperiment();
+  const vars = useScenarioVars().data;
-  const vars = api.templateVars.list.useQuery({
-    experimentId: experiment.data?.id ?? "",
-  }).data;
 
   const scenarioVariables = scenario.variableValues as Record<string, string>;
   const templateHasVariables =

app/src/components/OutputsTable/ScenarioEditor.tsx

@@ -1,7 +1,7 @@
 import { isEqual } from "lodash-es";
 import { useEffect, useState, type DragEvent } from "react";
 import { api } from "~/utils/api";
-import { useExperiment, useExperimentAccess, useHandledAsyncCallback } from "~/utils/hooks";
+import { useExperimentAccess, useHandledAsyncCallback, useScenarioVars } from "~/utils/hooks";
 import { type Scenario } from "./types";
 
 import {
@@ -41,8 +41,7 @@ export default function ScenarioEditor({
     if (savedValues) setValues(savedValues);
   }, [savedValues]);
 
-  const experiment = useExperiment();
+  const vars = useScenarioVars();
-  const vars = api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
 
   const variableLabels = vars.data?.map((v) => v.label) ?? [];
 

app/src/components/OutputsTable/ScenarioEditorModal.tsx

@@ -58,7 +58,7 @@ export const ScenarioEditorModal = ({
     await utils.scenarios.list.invalidate();
   }, [mutation, values]);
 
-  const vars = api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
+  const vars = api.scenarioVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
   const variableLabels = vars.data?.map((v) => v.label) ?? [];
 
   return (

app/src/components/OutputsTable/ScenariosHeader.tsx

@@ -72,7 +72,7 @@ export const ScenariosHeader = () => {
               Autogenerate Scenario
             </MenuItem>
             <MenuItem icon={<BsPencil />} onClick={openDrawer}>
-              Edit Vars
+              Add or Remove Variables
             </MenuItem>
           </MenuList>
         </Menu>

app/src/server/api/root.router.ts

@@ -3,7 +3,7 @@ import { createTRPCRouter } from "~/server/api/trpc";
 import { experimentsRouter } from "./routers/experiments.router";
 import { scenariosRouter } from "./routers/scenarios.router";
 import { scenarioVariantCellsRouter } from "./routers/scenarioVariantCells.router";
-import { templateVarsRouter } from "./routers/templateVariables.router";
+import { scenarioVarsRouter } from "./routers/scenarioVariables.router";
 import { evaluationsRouter } from "./routers/evaluations.router";
 import { worldChampsRouter } from "./routers/worldChamps.router";
 import { datasetsRouter } from "./routers/datasets.router";
@@ -22,7 +22,7 @@ export const appRouter = createTRPCRouter({
   experiments: experimentsRouter,
   scenarios: scenariosRouter,
   scenarioVariantCells: scenarioVariantCellsRouter,
-  templateVars: templateVarsRouter,
+  scenarioVars: scenarioVarsRouter,
   evaluations: evaluationsRouter,
   worldChamps: worldChampsRouter,
   datasets: datasetsRouter,

app/src/server/api/routers/promptVariants.router.ts

@@ -3,7 +3,7 @@ import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/
 import { prisma } from "~/server/db";
 import { Prisma } from "@prisma/client";
 import { generateNewCell } from "~/server/utils/generateNewCell";
-import userError from "~/server/utils/error";
+import { error, success } from "~/utils/standardResponses";
 import { recordExperimentUpdated } from "~/server/utils/recordExperimentUpdated";
 import { reorderPromptVariants } from "~/server/utils/reorderPromptVariants";
 import { type PromptVariant } from "@prisma/client";
@@ -315,7 +315,7 @@ export const promptVariantsRouter = createTRPCRouter({
       const constructedPrompt = await parsePromptConstructor(existing.promptConstructor);
 
       if ("error" in constructedPrompt) {
-        return userError(constructedPrompt.error);
+        return error(constructedPrompt.error);
       }
 
       const model = input.newModel
@@ -353,7 +353,7 @@ export const promptVariantsRouter = createTRPCRouter({
       const parsedPrompt = await parsePromptConstructor(input.promptConstructor);
 
       if ("error" in parsedPrompt) {
-        return userError(parsedPrompt.error);
+        return error(parsedPrompt.error);
       }
 
       // Create a duplicate with only the config changed
@@ -398,7 +398,7 @@ export const promptVariantsRouter = createTRPCRouter({
         });
       }
 
-      return { status: "ok" } as const;
+      return success();
     }),
 
   reorder: protectedProcedure

app/src/server/api/routers/scenarioVariables.router.ts

@@ -0,0 +1,143 @@
+import { type TemplateVariable } from "@prisma/client";
+import { sql } from "kysely";
+import { z } from "zod";
+import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/api/trpc";
+import { kysely, prisma } from "~/server/db";
+import { error, success } from "~/utils/standardResponses";
+import { requireCanModifyExperiment, requireCanViewExperiment } from "~/utils/accessControl";
+
+export const scenarioVarsRouter = createTRPCRouter({
+  create: protectedProcedure
+    .input(z.object({ experimentId: z.string(), label: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      await requireCanModifyExperiment(input.experimentId, ctx);
+
+      // Make sure there isn't an existing variable with the same name
+      const existingVariable = await prisma.templateVariable.findFirst({
+        where: {
+          experimentId: input.experimentId,
+          label: input.label,
+        },
+      });
+      if (existingVariable) {
+        return error(`A variable named ${input.label} already exists.`);
+      }
+
+      await prisma.templateVariable.create({
+        data: {
+          experimentId: input.experimentId,
+          label: input.label,
+        },
+      });
+
+      return success();
+    }),
+
+  rename: protectedProcedure
+    .input(z.object({ id: z.string(), label: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      const templateVariable = await prisma.templateVariable.findUniqueOrThrow({
+        where: { id: input.id },
+      });
+      await requireCanModifyExperiment(templateVariable.experimentId, ctx);
+
+      // Make sure there isn't an existing variable with the same name
+      const existingVariable = await prisma.templateVariable.findFirst({
+        where: {
+          experimentId: templateVariable.experimentId,
+          label: input.label,
+        },
+      });
+      if (existingVariable) {
+        return error(`A variable named ${input.label} already exists.`);
+      }
+
+      await renameTemplateVariable(templateVariable, input.label);
+      return success();
+    }),
+
+  delete: protectedProcedure
+    .input(z.object({ id: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      const { experimentId } = await prisma.templateVariable.findUniqueOrThrow({
+        where: { id: input.id },
+      });
+
+      await requireCanModifyExperiment(experimentId, ctx);
+
+      await prisma.templateVariable.delete({ where: { id: input.id } });
+    }),
+
+  list: publicProcedure
+    .input(z.object({ experimentId: z.string() }))
+    .query(async ({ input, ctx }) => {
+      await requireCanViewExperiment(input.experimentId, ctx);
+      return await prisma.templateVariable.findMany({
+        where: {
+          experimentId: input.experimentId,
+        },
+        orderBy: {
+          createdAt: "asc",
+        },
+        select: {
+          id: true,
+          label: true,
+        },
+      });
+    }),
+});
+
+export const renameTemplateVariable = async (
+  templateVariable: TemplateVariable,
+  newLabel: string,
+) => {
+  const { experimentId } = templateVariable;
+
+  await kysely.transaction().execute(async (trx) => {
+    await trx
+      .updateTable("TemplateVariable")
+      .set({
+        label: newLabel,
+      })
+      .where("id", "=", templateVariable.id)
+      .execute();
+
+    await sql`
+      CREATE TEMP TABLE "TempTestScenario" AS
+      SELECT *
+      FROM "TestScenario"
+      WHERE "experimentId" = ${experimentId}
+
+      -- Only copy the rows that actually have a value for the variable, no reason to churn the rest and simplifies the update.
+      AND "variableValues"->${templateVariable.label} IS NOT NULL
+      `.execute(trx);
+
+    await sql`
+      UPDATE "TempTestScenario"
+      SET "variableValues" = jsonb_set(
+        "variableValues",
+        ${`{${newLabel}}`},
+        "variableValues"->${templateVariable.label}
+      ) - ${templateVariable.label},
+      "updatedAt" = NOW(),
+      "id" = uuid_generate_v4()
+    `.execute(trx);
+
+    // Print the contents of the temp table
+    const results = await sql`SELECT * FROM "TempTestScenario"`.execute(trx);
+    console.log(results.rows);
+
+    await trx
+      .updateTable("TestScenario")
+      .set({
+        visible: false,
+      })
+      .where("experimentId", "=", experimentId)
+      .execute();
+
+    await sql`
+      INSERT INTO "TestScenario" (id, "variableValues", "uiId", visible, "sortIndex", "experimentId", "createdAt", "updatedAt")
+      SELECT * FROM "TempTestScenario";
+    `.execute(trx);
+  });
+};

app/src/server/api/routers/templateVariables.router.test.ts

@@ -0,0 +1,110 @@
+import { expect, it } from "vitest";
+import { prisma } from "~/server/db";
+import { renameTemplateVariable } from "./scenarioVariables.router";
+
+const createExperiment = async () => {
+  return await prisma.experiment.create({
+    data: {
+      label: "Test Experiment",
+      project: {
+        create: {},
+      },
+    },
+  });
+};
+
+const createTemplateVar = async (experimentId: string, label: string) => {
+  return await prisma.templateVariable.create({
+    data: {
+      experimentId,
+      label,
+    },
+  });
+};
+
+it("renames templateVariables", async () => {
+  // Create experiments concurrently
+  const [exp1, exp2] = await Promise.all([createExperiment(), createExperiment()]);
+
+  // Create template variables concurrently
+  const [exp1Var, exp2Var1, exp2Var2] = await Promise.all([
+    createTemplateVar(exp1.id, "input1"),
+    createTemplateVar(exp2.id, "input1"),
+    createTemplateVar(exp2.id, "input2"),
+  ]);
+
+  // Create test scenarios concurrently
+  const [exp1Scenario, exp2Scenario, exp2HiddenScenario] = await Promise.all([
+    prisma.testScenario.create({
+      data: {
+        experimentId: exp1.id,
+        visible: true,
+        variableValues: { input1: "test" },
+      },
+    }),
+    prisma.testScenario.create({
+      data: {
+        experimentId: exp2.id,
+        visible: true,
+        variableValues: { input1: "test1", otherInput: "otherTest" },
+      },
+    }),
+    prisma.testScenario.create({
+      data: {
+        experimentId: exp2.id,
+        visible: false,
+        variableValues: { otherInput: "otherTest2" },
+      },
+    }),
+  ]);
+
+  await renameTemplateVariable(exp2Var1, "input1-renamed");
+
+  expect(await prisma.templateVariable.findUnique({ where: { id: exp2Var1.id } })).toMatchObject({
+    label: "input1-renamed",
+  });
+
+  // It shouldn't mess with unrelated experiments
+  expect(await prisma.testScenario.findUnique({ where: { id: exp1Scenario.id } })).toMatchObject({
+    visible: true,
+    variableValues: { input1: "test" },
+  });
+
+  // Make sure there are a total of 4 scenarios for exp2
+  expect(
+    await prisma.testScenario.count({
+      where: {
+        experimentId: exp2.id,
+      },
+    }),
+  ).toBe(3);
+
+  // It shouldn't mess with the existing scenarios, except to hide them
+  expect(await prisma.testScenario.findUnique({ where: { id: exp2Scenario.id } })).toMatchObject({
+    visible: false,
+    variableValues: { input1: "test1", otherInput: "otherTest" },
+  });
+
+  // It should create a new scenario with the new variable name
+  const newScenario1 = await prisma.testScenario.findFirst({
+    where: {
+      experimentId: exp2.id,
+      variableValues: { equals: { "input1-renamed": "test1", otherInput: "otherTest" } },
+    },
+  });
+
+  expect(newScenario1).toMatchObject({
+    visible: true,
+  });
+
+  const newScenario2 = await prisma.testScenario.findFirst({
+    where: {
+      experimentId: exp2.id,
+      variableValues: { equals: { otherInput: "otherTest2" } },
+    },
+  });
+
+  expect(newScenario2).toMatchObject({
+    visible: false,
+  });
+});

app/src/server/api/routers/templateVariables.router.ts

@@ -1,49 +0,0 @@
-import { z } from "zod";
-import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/api/trpc";
-import { prisma } from "~/server/db";
-import { requireCanModifyExperiment, requireCanViewExperiment } from "~/utils/accessControl";
-
-export const templateVarsRouter = createTRPCRouter({
-  create: protectedProcedure
-    .input(z.object({ experimentId: z.string(), label: z.string() }))
-    .mutation(async ({ input, ctx }) => {
-      await requireCanModifyExperiment(input.experimentId, ctx);
-
-      await prisma.templateVariable.create({
-        data: {
-          experimentId: input.experimentId,
-          label: input.label,
-        },
-      });
-    }),
-
-  delete: protectedProcedure
-    .input(z.object({ id: z.string() }))
-    .mutation(async ({ input, ctx }) => {
-      const { experimentId } = await prisma.templateVariable.findUniqueOrThrow({
-        where: { id: input.id },
-      });
-
-      await requireCanModifyExperiment(experimentId, ctx);
-
-      await prisma.templateVariable.delete({ where: { id: input.id } });
-    }),
-
-  list: publicProcedure
-    .input(z.object({ experimentId: z.string() }))
-    .query(async ({ input, ctx }) => {
-      await requireCanViewExperiment(input.experimentId, ctx);
-      return await prisma.templateVariable.findMany({
-        where: {
-          experimentId: input.experimentId,
-        },
-        orderBy: {
-          createdAt: "asc",
-        },
-        select: {
-          id: true,
-          label: true,
-        },
-      });
-    }),
-});

app/src/server/scripts/test-queries.ts

@@ -1,63 +0,0 @@
-import dayjs from "dayjs";
-import { prisma } from "../db";
-
-const projectId = "1234";
-
-// Find all calls in the last 24 hours
-const responses = await prisma.loggedCall.findMany({
-  where: {
-    projectId: projectId,
-    startTime: {
-      gt: dayjs()
-        .subtract(24 * 3600)
-        .toDate(),
-    },
-  },
-  include: {
-    modelResponse: true,
-  },
-  orderBy: {
-    startTime: "desc",
-  },
-});
-
-// Find all calls in the last 24 hours with promptId 'hello-world'
-const helloWorld = await prisma.loggedCall.findMany({
-  where: {
-    projectId: projectId,
-    startTime: {
-      gt: dayjs()
-        .subtract(24 * 3600)
-        .toDate(),
-    },
-    tags: {
-      some: {
-        name: "promptId",
-        value: "hello-world",
-      },
-    },
-  },
-  include: {
-    modelResponse: true,
-  },
-  orderBy: {
-    startTime: "desc",
-  },
-});
-
-// Total spent on OpenAI in the last month
-const totalSpent = await prisma.loggedCallModelResponse.aggregate({
-  _sum: {
-    totalCost: true,
-  },
-  where: {
-    originalLoggedCall: {
-      projectId: projectId,
-    },
-    startTime: {
-      gt: dayjs()
-        .subtract(30 * 24 * 3600)
-        .toDate(),
-    },
-  },
-});

app/src/server/utils/error.ts

@@ -1,6 +0,0 @@
-export default function userError(message: string): { status: "error"; message: string } {
-  return {
-    status: "error",
-    message,
-  };
-}

app/src/state/persist.ts

@@ -1,5 +1,5 @@
-import { PersistOptions } from "zustand/middleware/persist";
+import { type PersistOptions } from "zustand/middleware/persist";
-import { State } from "./store";
+import { type State } from "./store";
 
 export const stateToPersist = {
   selectedProjectId: null as string | null,

app/src/state/sharedVariantEditor.slice.ts

@@ -8,9 +8,9 @@ export const editorBackground = "#fafafa";
 export type SharedVariantEditorSlice = {
   monaco: null | ReturnType<typeof loader.__getMonacoInstance>;
   loadMonaco: () => Promise<void>;
-  scenarios: RouterOutputs["scenarios"]["list"]["scenarios"];
+  scenarioVars: RouterOutputs["scenarioVars"]["list"];
   updateScenariosModel: () => void;
-  setScenarios: (scenarios: RouterOutputs["scenarios"]["list"]["scenarios"]) => void;
+  setScenarioVars: (scenarioVars: RouterOutputs["scenarioVars"]["list"]) => void;
 };
 
 export const createVariantEditorSlice: SliceCreator<SharedVariantEditorSlice> = (set, get) => ({
@@ -60,10 +60,10 @@ export const createVariantEditorSlice: SliceCreator<SharedVariantEditorSlice> =
     });
     get().sharedVariantEditor.updateScenariosModel();
   },
-  scenarios: [],
+  scenarioVars: [],
-  setScenarios: (scenarios) => {
+  setScenarioVars: (scenarios) => {
     set((state) => {
-      state.sharedVariantEditor.scenarios = scenarios;
+      state.sharedVariantEditor.scenarioVars = scenarios;
     });
 
     get().sharedVariantEditor.updateScenariosModel();
@@ -74,16 +74,15 @@ export const createVariantEditorSlice: SliceCreator<SharedVariantEditorSlice> =
     if (!monaco) return;
 
     const modelContents = `    
-    const scenarios = ${JSON.stringify(
+    declare var scenario: {
-      get().sharedVariantEditor.scenarios.map((s) => s.variableValues),
+      ${get()
-      null,
+        .sharedVariantEditor.scenarioVars.map((s) => `${s.label}: string;`)
-      2,
+        .join("\n")}
-    )} as const;
+    };
-    
-    type Scenario = typeof scenarios[number];
-    declare var scenario: Scenario | { [key: string]: string };
     `;
 
+    console.log(modelContents);
+
     const scenariosModel = monaco.editor.getModel(monaco.Uri.parse("file:///scenarios.ts"));
 
     if (scenariosModel) {

app/src/state/store.ts

@@ -7,7 +7,7 @@ import {
   createVariantEditorSlice,
 } from "./sharedVariantEditor.slice";
 import { type APIClient } from "~/utils/api";
-import { persistOptions, stateToPersist } from "./persist";
+import { persistOptions, type stateToPersist } from "./persist";
 
 export type State = {
   drawerOpen: boolean;

app/src/state/sync.tsx

@@ -1,16 +1,16 @@
 import { useEffect } from "react";
 import { api } from "~/utils/api";
-import { useScenarios } from "~/utils/hooks";
+import { useScenarioVars } from "~/utils/hooks";
 import { useAppStore } from "./store";
 
 export function useSyncVariantEditor() {
-  const scenarios = useScenarios();
+  const scenarioVars = useScenarioVars();
 
   useEffect(() => {
-    if (scenarios.data) {
+    if (scenarioVars.data) {
-      useAppStore.getState().sharedVariantEditor.setScenarios(scenarios.data.scenarios);
+      useAppStore.getState().sharedVariantEditor.setScenarioVars(scenarioVars.data);
     }
-  }, [scenarios.data]);
+  }, [scenarioVars.data]);
 }
 
 export function SyncAppStore() {

app/src/tests/helpers/loadEnv.ts

@@ -0,0 +1,5 @@
+import { configDotenv } from "dotenv";
+
+configDotenv({
+  path: ".env.test",
+});

app/src/tests/helpers/setup.ts

@@ -0,0 +1,13 @@
+import "./loadEnv";
+import { sql } from "kysely";
+import { beforeEach } from "vitest";
+import { kysely } from "~/server/db";
+
+// Reset all Prisma data
+const resetDb = async () => {
+  await sql`truncate "Experiment" cascade;`.execute(kysely);
+};
+
+beforeEach(async () => {
+  await resetDb();
+});

app/src/theme/ChakraThemeProvider.tsx

@@ -1,4 +1,9 @@
-import { extendTheme, defineStyleConfig, ChakraProvider } from "@chakra-ui/react";
+import {
+  extendTheme,
+  defineStyleConfig,
+  ChakraProvider,
+  createStandaloneToast,
+} from "@chakra-ui/react";
 import "@fontsource/inconsolata";
 import { modalAnatomy } from "@chakra-ui/anatomy";
 import { createMultiStyleConfigHelpers } from "@chakra-ui/styled-system";
@@ -63,6 +68,15 @@ const theme = extendTheme({
   },
 });
 
+const { ToastContainer, toast } = createStandaloneToast(theme);
+
+export { toast };
+
 export const ChakraThemeProvider = ({ children }: { children: JSX.Element }) => {
-  return <ChakraProvider theme={theme}>{children}</ChakraProvider>;
+  return (
+    <ChakraProvider theme={theme}>
+      <ToastContainer />
+      {children}
+    </ChakraProvider>
+  );
 };

app/src/utils/hooks.ts

@@ -157,3 +157,12 @@ export const useSelectedProject = () => {
     { enabled: !!selectedProjectId },
   );
 };
+
+export const useScenarioVars = () => {
+  const experiment = useExperiment();
+
+  return api.scenarioVars.list.useQuery(
+    { experimentId: experiment.data?.id ?? "" },
+    { enabled: experiment.data?.id != null },
+  );
+};

app/src/utils/standardResponses.ts

@@ -0,0 +1,31 @@
+import { toast } from "~/theme/ChakraThemeProvider";
+
+export function error(message: string): { status: "error"; message: string } {
+  return {
+    status: "error",
+    message,
+  };
+}
+export function success<T>(payload: T): { status: "success"; payload: T };
+export function success(payload?: undefined): { status: "success"; payload: undefined };
+export function success<T>(payload?: T) {
+  return { status: "success", payload };
+}
+
+type SuccessType<T> = ReturnType<typeof success<T>>;
+type ErrorType = ReturnType<typeof error>;
+
+// Used client-side to report generic errors
+export function maybeReportError<T>(response: SuccessType<T> | ErrorType): response is ErrorType {
+  if (response.status === "error") {
+    toast({
+      description: response.message,
+      status: "error",
+      duration: 5000,
+      isClosable: true,
+    });
+    return true;
+  }
+
+  return false;
+}

app/vitest.config.ts

@@ -4,6 +4,10 @@ import { configDefaults, defineConfig, type UserConfig } from "vitest/config";
 const config = defineConfig({
   test: {
     ...configDefaults, // Extending Vitest's default options
+    setupFiles: ["./src/tests/helpers/setup.ts"],
+
+    // Unfortunately using threads seems to cause issues with isolated-vm
+    threads: false,
   },
   plugins: [tsconfigPaths()],
 }) as UserConfig;