diff --git a/README.md b/README.md index cf5cccc..1541c13 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,14 @@ OpenPipe includes a tool to generate new test scenarios based on your existing p 4. Clone this repository: `git clone https://github.com/openpipe/openpipe` 5. Install the dependencies: `cd openpipe && pnpm install` 6. Create a `.env` file (`cp .env.example .env`) and enter your `OPENAI_API_KEY`. -7. Update `DATABASE_URL` if necessary to point to your Postgres instance and run `pnpm prisma db push` to create the database. +7. Update `DATABASE_URL` if necessary to point to your Postgres instance and run `pnpm prisma migrate dev` to create the database. 8. Create a [GitHub OAuth App](https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/creating-an-oauth-app) and update the `GITHUB_CLIENT_ID` and `GITHUB_CLIENT_SECRET` values. (Note: a PR to make auth optional when running locally would be a great contribution!) 9. Start the app: `pnpm dev`. 10. Navigate to [http://localhost:3000](http://localhost:3000) + +## Testing Locally + +1. Copy your `.env` file to `.env.test`. +2. Update the `DATABASE_URL` to have a different database name than your development one +3. Run `DATABASE_URL=[your new datatase url] pnpm prisma migrate dev --skip-seed --skip-generate` +4. Run `pnpm test` \ No newline at end of file diff --git a/app/.gitignore b/app/.gitignore index 317d27a..354334f 100644 --- a/app/.gitignore +++ b/app/.gitignore @@ -34,6 +34,7 @@ yarn-error.log* # do not commit any .env files to git, except for the .env.example file. https://create.t3.gg/en/usage/env-variables#using-environment-variables .env .env*.local +.env.test # vercel .vercel diff --git a/app/package.json b/app/package.json index e7c5818..65a965a 100644 --- a/app/package.json +++ b/app/package.json @@ -19,7 +19,7 @@ "codegen": "tsx src/server/scripts/client-codegen.ts", "seed": "tsx prisma/seed.ts", "check": "concurrently 'pnpm lint' 'pnpm tsc' 'pnpm prettier . --check'", - "test": "pnpm vitest --no-threads" + "test": "pnpm vitest" }, "dependencies": { "@anthropic-ai/sdk": "^0.5.8", diff --git a/app/prisma/migrations/20230810161437_add_uuid_extension/migration.sql b/app/prisma/migrations/20230810161437_add_uuid_extension/migration.sql new file mode 100644 index 0000000..d159cc5 --- /dev/null +++ b/app/prisma/migrations/20230810161437_add_uuid_extension/migration.sql @@ -0,0 +1 @@ +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; diff --git a/app/src/components/OutputsTable/EditEvaluations.tsx b/app/src/components/OutputsTable/EditEvaluations.tsx index 0028391..1375cf8 100644 --- a/app/src/components/OutputsTable/EditEvaluations.tsx +++ b/app/src/components/OutputsTable/EditEvaluations.tsx @@ -12,6 +12,7 @@ import { Select, FormHelperText, Code, + IconButton, } from "@chakra-ui/react"; import { type Evaluation, EvalType } from "@prisma/client"; import { useCallback, useState } from "react"; @@ -183,46 +184,37 @@ export default function EditEvaluations() { {evaluation.evalType}: "{evaluation.value}" - - + _hover={{ color: "gray.800", cursor: "pointer" }} + icon={} + /> ), )} {editingId == null && ( )} {editingId == "new" && ( diff --git a/app/src/components/OutputsTable/EditScenarioVars.tsx b/app/src/components/OutputsTable/EditScenarioVars.tsx index 97a359c..9854bfa 100644 --- a/app/src/components/OutputsTable/EditScenarioVars.tsx +++ b/app/src/components/OutputsTable/EditScenarioVars.tsx @@ -1,103 +1,185 @@ -import { Text, Button, HStack, Heading, Icon, Input, Stack } from "@chakra-ui/react"; -import { useState } from "react"; -import { BsCheck, BsX } from "react-icons/bs"; +import { Text, Button, HStack, Heading, Icon, IconButton, Stack, VStack } from "@chakra-ui/react"; +import { type TemplateVariable } from "@prisma/client"; +import { useEffect, useState } from "react"; +import { BsPencil, BsX } from "react-icons/bs"; import { api } from "~/utils/api"; -import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks"; +import { useExperiment, useHandledAsyncCallback, useScenarioVars } from "~/utils/hooks"; +import { maybeReportError } from "~/utils/standardResponses"; +import { FloatingLabelInput } from "./FloatingLabelInput"; + +export const ScenarioVar = ({ + variable, + isEditing, + setIsEditing, +}: { + variable: Pick; + isEditing: boolean; + setIsEditing: (isEditing: boolean) => void; +}) => { + const utils = api.useContext(); + + const [label, setLabel] = useState(variable.label); + + useEffect(() => { + setLabel(variable.label); + }, [variable.label]); + + const renameVarMutation = api.scenarioVars.rename.useMutation(); + const [onRename] = useHandledAsyncCallback(async () => { + const resp = await renameVarMutation.mutateAsync({ id: variable.id, label }); + if (maybeReportError(resp)) return; + + setIsEditing(false); + await utils.scenarioVars.list.invalidate(); + await utils.scenarios.list.invalidate(); + }, [label, variable.id]); + + const deleteMutation = api.scenarioVars.delete.useMutation(); + const [onDeleteVar] = useHandledAsyncCallback(async () => { + await deleteMutation.mutateAsync({ id: variable.id }); + await utils.scenarioVars.list.invalidate(); + }, [variable.id]); + + if (isEditing) { + return ( + + setLabel(e.target.value)} + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault(); + onRename(); + } + // If the user types a space, replace it with an underscore + if (e.key === " ") { + e.preventDefault(); + setLabel((label) => label && `${label}_`); + } + }} + /> + + + + ); + } else { + return ( + + {variable.label} + setIsEditing(true)} + _hover={{ color: "gray.800", cursor: "pointer" }} + icon={} + /> + } + /> + + ); + } +}; export default function EditScenarioVars() { const experiment = useExperiment(); - const vars = - api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" }).data ?? []; + const vars = useScenarioVars(); + + const [currentlyEditingId, setCurrentlyEditingId] = useState(null); const [newVariable, setNewVariable] = useState(""); - const newVarIsValid = newVariable.length > 0 && !vars.map((v) => v.label).includes(newVariable); + const newVarIsValid = newVariable?.length ?? 0 > 0; const utils = api.useContext(); - const addVarMutation = api.templateVars.create.useMutation(); + const addVarMutation = api.scenarioVars.create.useMutation(); const [onAddVar] = useHandledAsyncCallback(async () => { if (!experiment.data?.id) return; - if (!newVarIsValid) return; - await addVarMutation.mutateAsync({ + if (!newVariable) return; + const resp = await addVarMutation.mutateAsync({ experimentId: experiment.data.id, label: newVariable, }); - await utils.templateVars.list.invalidate(); + if (maybeReportError(resp)) return; + + await utils.scenarioVars.list.invalidate(); setNewVariable(""); }, [addVarMutation, experiment.data?.id, newVarIsValid, newVariable]); - const deleteMutation = api.templateVars.delete.useMutation(); - const [onDeleteVar] = useHandledAsyncCallback(async (id: string) => { - await deleteMutation.mutateAsync({ id }); - await utils.templateVars.list.invalidate(); - }, []); - return ( Scenario Variables - + Scenario variables can be used in your prompt variants as well as evaluations. - - setNewVariable(e.target.value)} - onKeyDown={(e) => { - if (e.key === "Enter") { - e.preventDefault(); - onAddVar(); - } - // If the user types a space, replace it with an underscore - if (e.key === " ") { - e.preventDefault(); - setNewVariable((v) => v + "_"); - } - }} - /> - - - - - {vars.map((variable) => ( - + {vars.data?.map((variable) => ( + - - {variable.label} - - - + isEditing={currentlyEditingId === variable.id} + setIsEditing={(isEditing) => { + if (isEditing) { + setCurrentlyEditingId(variable.id); + } else { + setCurrentlyEditingId(null); + } + }} + /> ))} - - + + {currentlyEditingId !== "new" && ( + + )} + {currentlyEditingId === "new" && ( + + setNewVariable(e.target.value)} + onKeyDown={(e) => { + if (e.key === "Enter") { + e.preventDefault(); + onAddVar(); + } + // If the user types a space, replace it with an underscore + if (e.key === " ") { + e.preventDefault(); + setNewVariable((v) => v && `${v}_`); + } + }} + /> + + + + )} + ); } diff --git a/app/src/components/OutputsTable/OutputCell/OutputCell.tsx b/app/src/components/OutputsTable/OutputCell/OutputCell.tsx index b5a1955..4eafbb7 100644 --- a/app/src/components/OutputsTable/OutputCell/OutputCell.tsx +++ b/app/src/components/OutputsTable/OutputCell/OutputCell.tsx @@ -1,7 +1,7 @@ import { api } from "~/utils/api"; import { type PromptVariant, type Scenario } from "../types"; import { type StackProps, Text, VStack } from "@chakra-ui/react"; -import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks"; +import { useScenarioVars, useHandledAsyncCallback } from "~/utils/hooks"; import SyntaxHighlighter from "react-syntax-highlighter"; import { docco } from "react-syntax-highlighter/dist/cjs/styles/hljs"; import stringify from "json-stringify-pretty-compact"; @@ -23,10 +23,7 @@ export default function OutputCell({ variant: PromptVariant; }): ReactElement | null { const utils = api.useContext(); - const experiment = useExperiment(); - const vars = api.templateVars.list.useQuery({ - experimentId: experiment.data?.id ?? "", - }).data; + const vars = useScenarioVars().data; const scenarioVariables = scenario.variableValues as Record; const templateHasVariables = diff --git a/app/src/components/OutputsTable/ScenarioEditor.tsx b/app/src/components/OutputsTable/ScenarioEditor.tsx index 5329118..51770aa 100644 --- a/app/src/components/OutputsTable/ScenarioEditor.tsx +++ b/app/src/components/OutputsTable/ScenarioEditor.tsx @@ -1,7 +1,7 @@ import { isEqual } from "lodash-es"; import { useEffect, useState, type DragEvent } from "react"; import { api } from "~/utils/api"; -import { useExperiment, useExperimentAccess, useHandledAsyncCallback } from "~/utils/hooks"; +import { useExperimentAccess, useHandledAsyncCallback, useScenarioVars } from "~/utils/hooks"; import { type Scenario } from "./types"; import { @@ -41,8 +41,7 @@ export default function ScenarioEditor({ if (savedValues) setValues(savedValues); }, [savedValues]); - const experiment = useExperiment(); - const vars = api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" }); + const vars = useScenarioVars(); const variableLabels = vars.data?.map((v) => v.label) ?? []; diff --git a/app/src/components/OutputsTable/ScenarioEditorModal.tsx b/app/src/components/OutputsTable/ScenarioEditorModal.tsx index dec1bd7..897e16f 100644 --- a/app/src/components/OutputsTable/ScenarioEditorModal.tsx +++ b/app/src/components/OutputsTable/ScenarioEditorModal.tsx @@ -58,7 +58,7 @@ export const ScenarioEditorModal = ({ await utils.scenarios.list.invalidate(); }, [mutation, values]); - const vars = api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" }); + const vars = api.scenarioVars.list.useQuery({ experimentId: experiment.data?.id ?? "" }); const variableLabels = vars.data?.map((v) => v.label) ?? []; return ( diff --git a/app/src/components/OutputsTable/ScenariosHeader.tsx b/app/src/components/OutputsTable/ScenariosHeader.tsx index 6dacf74..a785f50 100644 --- a/app/src/components/OutputsTable/ScenariosHeader.tsx +++ b/app/src/components/OutputsTable/ScenariosHeader.tsx @@ -72,7 +72,7 @@ export const ScenariosHeader = () => { Autogenerate Scenario } onClick={openDrawer}> - Edit Vars + Add or Remove Variables diff --git a/app/src/server/api/root.router.ts b/app/src/server/api/root.router.ts index 283e834..df7184a 100644 --- a/app/src/server/api/root.router.ts +++ b/app/src/server/api/root.router.ts @@ -3,7 +3,7 @@ import { createTRPCRouter } from "~/server/api/trpc"; import { experimentsRouter } from "./routers/experiments.router"; import { scenariosRouter } from "./routers/scenarios.router"; import { scenarioVariantCellsRouter } from "./routers/scenarioVariantCells.router"; -import { templateVarsRouter } from "./routers/templateVariables.router"; +import { scenarioVarsRouter } from "./routers/scenarioVariables.router"; import { evaluationsRouter } from "./routers/evaluations.router"; import { worldChampsRouter } from "./routers/worldChamps.router"; import { datasetsRouter } from "./routers/datasets.router"; @@ -22,7 +22,7 @@ export const appRouter = createTRPCRouter({ experiments: experimentsRouter, scenarios: scenariosRouter, scenarioVariantCells: scenarioVariantCellsRouter, - templateVars: templateVarsRouter, + scenarioVars: scenarioVarsRouter, evaluations: evaluationsRouter, worldChamps: worldChampsRouter, datasets: datasetsRouter, diff --git a/app/src/server/api/routers/promptVariants.router.ts b/app/src/server/api/routers/promptVariants.router.ts index 9a85560..f19bbd0 100644 --- a/app/src/server/api/routers/promptVariants.router.ts +++ b/app/src/server/api/routers/promptVariants.router.ts @@ -3,7 +3,7 @@ import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/ import { prisma } from "~/server/db"; import { Prisma } from "@prisma/client"; import { generateNewCell } from "~/server/utils/generateNewCell"; -import userError from "~/server/utils/error"; +import { error, success } from "~/utils/standardResponses"; import { recordExperimentUpdated } from "~/server/utils/recordExperimentUpdated"; import { reorderPromptVariants } from "~/server/utils/reorderPromptVariants"; import { type PromptVariant } from "@prisma/client"; @@ -315,7 +315,7 @@ export const promptVariantsRouter = createTRPCRouter({ const constructedPrompt = await parsePromptConstructor(existing.promptConstructor); if ("error" in constructedPrompt) { - return userError(constructedPrompt.error); + return error(constructedPrompt.error); } const model = input.newModel @@ -353,7 +353,7 @@ export const promptVariantsRouter = createTRPCRouter({ const parsedPrompt = await parsePromptConstructor(input.promptConstructor); if ("error" in parsedPrompt) { - return userError(parsedPrompt.error); + return error(parsedPrompt.error); } // Create a duplicate with only the config changed @@ -398,7 +398,7 @@ export const promptVariantsRouter = createTRPCRouter({ }); } - return { status: "ok" } as const; + return success(); }), reorder: protectedProcedure diff --git a/app/src/server/api/routers/scenarioVariables.router.ts b/app/src/server/api/routers/scenarioVariables.router.ts new file mode 100644 index 0000000..9c1f678 --- /dev/null +++ b/app/src/server/api/routers/scenarioVariables.router.ts @@ -0,0 +1,143 @@ +import { type TemplateVariable } from "@prisma/client"; +import { sql } from "kysely"; +import { z } from "zod"; +import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/api/trpc"; +import { kysely, prisma } from "~/server/db"; +import { error, success } from "~/utils/standardResponses"; +import { requireCanModifyExperiment, requireCanViewExperiment } from "~/utils/accessControl"; + +export const scenarioVarsRouter = createTRPCRouter({ + create: protectedProcedure + .input(z.object({ experimentId: z.string(), label: z.string() })) + .mutation(async ({ input, ctx }) => { + await requireCanModifyExperiment(input.experimentId, ctx); + + // Make sure there isn't an existing variable with the same name + const existingVariable = await prisma.templateVariable.findFirst({ + where: { + experimentId: input.experimentId, + label: input.label, + }, + }); + if (existingVariable) { + return error(`A variable named ${input.label} already exists.`); + } + + await prisma.templateVariable.create({ + data: { + experimentId: input.experimentId, + label: input.label, + }, + }); + + return success(); + }), + + rename: protectedProcedure + .input(z.object({ id: z.string(), label: z.string() })) + .mutation(async ({ input, ctx }) => { + const templateVariable = await prisma.templateVariable.findUniqueOrThrow({ + where: { id: input.id }, + }); + await requireCanModifyExperiment(templateVariable.experimentId, ctx); + + // Make sure there isn't an existing variable with the same name + const existingVariable = await prisma.templateVariable.findFirst({ + where: { + experimentId: templateVariable.experimentId, + label: input.label, + }, + }); + if (existingVariable) { + return error(`A variable named ${input.label} already exists.`); + } + + await renameTemplateVariable(templateVariable, input.label); + return success(); + }), + + delete: protectedProcedure + .input(z.object({ id: z.string() })) + .mutation(async ({ input, ctx }) => { + const { experimentId } = await prisma.templateVariable.findUniqueOrThrow({ + where: { id: input.id }, + }); + + await requireCanModifyExperiment(experimentId, ctx); + + await prisma.templateVariable.delete({ where: { id: input.id } }); + }), + + list: publicProcedure + .input(z.object({ experimentId: z.string() })) + .query(async ({ input, ctx }) => { + await requireCanViewExperiment(input.experimentId, ctx); + return await prisma.templateVariable.findMany({ + where: { + experimentId: input.experimentId, + }, + orderBy: { + createdAt: "asc", + }, + select: { + id: true, + label: true, + }, + }); + }), +}); + +export const renameTemplateVariable = async ( + templateVariable: TemplateVariable, + newLabel: string, +) => { + const { experimentId } = templateVariable; + + await kysely.transaction().execute(async (trx) => { + await trx + .updateTable("TemplateVariable") + .set({ + label: newLabel, + }) + .where("id", "=", templateVariable.id) + .execute(); + + await sql` + CREATE TEMP TABLE "TempTestScenario" AS + SELECT * + FROM "TestScenario" + WHERE "experimentId" = ${experimentId} + + -- Only copy the rows that actually have a value for the variable, no reason to churn the rest and simplifies the update. + AND "variableValues"->${templateVariable.label} IS NOT NULL + `.execute(trx); + + await sql` + UPDATE "TempTestScenario" + SET "variableValues" = jsonb_set( + "variableValues", + ${`{${newLabel}}`}, + "variableValues"->${templateVariable.label} + ) - ${templateVariable.label}, + "updatedAt" = NOW(), + "id" = uuid_generate_v4() + `.execute(trx); + + // Print the contents of the temp table + const results = await sql`SELECT * FROM "TempTestScenario"`.execute(trx); + console.log(results.rows); + + await trx + .updateTable("TestScenario") + .set({ + visible: false, + }) + .where("experimentId", "=", experimentId) + .execute(); + + await sql` + INSERT INTO "TestScenario" (id, "variableValues", "uiId", visible, "sortIndex", "experimentId", "createdAt", "updatedAt") + SELECT * FROM "TempTestScenario"; + `.execute(trx); + }); +}; diff --git a/app/src/server/api/routers/templateVariables.router.test.ts b/app/src/server/api/routers/templateVariables.router.test.ts new file mode 100644 index 0000000..68171d5 --- /dev/null +++ b/app/src/server/api/routers/templateVariables.router.test.ts @@ -0,0 +1,110 @@ +import { expect, it } from "vitest"; +import { prisma } from "~/server/db"; +import { renameTemplateVariable } from "./scenarioVariables.router"; + +const createExperiment = async () => { + return await prisma.experiment.create({ + data: { + label: "Test Experiment", + project: { + create: {}, + }, + }, + }); +}; + +const createTemplateVar = async (experimentId: string, label: string) => { + return await prisma.templateVariable.create({ + data: { + experimentId, + label, + }, + }); +}; + +it("renames templateVariables", async () => { + // Create experiments concurrently + const [exp1, exp2] = await Promise.all([createExperiment(), createExperiment()]); + + // Create template variables concurrently + const [exp1Var, exp2Var1, exp2Var2] = await Promise.all([ + createTemplateVar(exp1.id, "input1"), + createTemplateVar(exp2.id, "input1"), + createTemplateVar(exp2.id, "input2"), + ]); + + // Create test scenarios concurrently + const [exp1Scenario, exp2Scenario, exp2HiddenScenario] = await Promise.all([ + prisma.testScenario.create({ + data: { + experimentId: exp1.id, + visible: true, + variableValues: { input1: "test" }, + }, + }), + prisma.testScenario.create({ + data: { + experimentId: exp2.id, + visible: true, + variableValues: { input1: "test1", otherInput: "otherTest" }, + }, + }), + prisma.testScenario.create({ + data: { + experimentId: exp2.id, + visible: false, + variableValues: { otherInput: "otherTest2" }, + }, + }), + ]); + + await renameTemplateVariable(exp2Var1, "input1-renamed"); + + expect(await prisma.templateVariable.findUnique({ where: { id: exp2Var1.id } })).toMatchObject({ + label: "input1-renamed", + }); + + // It shouldn't mess with unrelated experiments + expect(await prisma.testScenario.findUnique({ where: { id: exp1Scenario.id } })).toMatchObject({ + visible: true, + variableValues: { input1: "test" }, + }); + + // Make sure there are a total of 4 scenarios for exp2 + expect( + await prisma.testScenario.count({ + where: { + experimentId: exp2.id, + }, + }), + ).toBe(3); + + // It shouldn't mess with the existing scenarios, except to hide them + expect(await prisma.testScenario.findUnique({ where: { id: exp2Scenario.id } })).toMatchObject({ + visible: false, + variableValues: { input1: "test1", otherInput: "otherTest" }, + }); + + // It should create a new scenario with the new variable name + const newScenario1 = await prisma.testScenario.findFirst({ + where: { + experimentId: exp2.id, + variableValues: { equals: { "input1-renamed": "test1", otherInput: "otherTest" } }, + }, + }); + + expect(newScenario1).toMatchObject({ + visible: true, + }); + + const newScenario2 = await prisma.testScenario.findFirst({ + where: { + experimentId: exp2.id, + variableValues: { equals: { otherInput: "otherTest2" } }, + }, + }); + + expect(newScenario2).toMatchObject({ + visible: false, + }); +}); diff --git a/app/src/server/api/routers/templateVariables.router.ts b/app/src/server/api/routers/templateVariables.router.ts deleted file mode 100644 index d62fec4..0000000 --- a/app/src/server/api/routers/templateVariables.router.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { z } from "zod"; -import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/api/trpc"; -import { prisma } from "~/server/db"; -import { requireCanModifyExperiment, requireCanViewExperiment } from "~/utils/accessControl"; - -export const templateVarsRouter = createTRPCRouter({ - create: protectedProcedure - .input(z.object({ experimentId: z.string(), label: z.string() })) - .mutation(async ({ input, ctx }) => { - await requireCanModifyExperiment(input.experimentId, ctx); - - await prisma.templateVariable.create({ - data: { - experimentId: input.experimentId, - label: input.label, - }, - }); - }), - - delete: protectedProcedure - .input(z.object({ id: z.string() })) - .mutation(async ({ input, ctx }) => { - const { experimentId } = await prisma.templateVariable.findUniqueOrThrow({ - where: { id: input.id }, - }); - - await requireCanModifyExperiment(experimentId, ctx); - - await prisma.templateVariable.delete({ where: { id: input.id } }); - }), - - list: publicProcedure - .input(z.object({ experimentId: z.string() })) - .query(async ({ input, ctx }) => { - await requireCanViewExperiment(input.experimentId, ctx); - return await prisma.templateVariable.findMany({ - where: { - experimentId: input.experimentId, - }, - orderBy: { - createdAt: "asc", - }, - select: { - id: true, - label: true, - }, - }); - }), -}); diff --git a/app/src/server/scripts/test-queries.ts b/app/src/server/scripts/test-queries.ts deleted file mode 100644 index 58caeeb..0000000 --- a/app/src/server/scripts/test-queries.ts +++ /dev/null @@ -1,63 +0,0 @@ -import dayjs from "dayjs"; -import { prisma } from "../db"; - -const projectId = "1234"; - -// Find all calls in the last 24 hours -const responses = await prisma.loggedCall.findMany({ - where: { - projectId: projectId, - startTime: { - gt: dayjs() - .subtract(24 * 3600) - .toDate(), - }, - }, - include: { - modelResponse: true, - }, - orderBy: { - startTime: "desc", - }, -}); - -// Find all calls in the last 24 hours with promptId 'hello-world' -const helloWorld = await prisma.loggedCall.findMany({ - where: { - projectId: projectId, - startTime: { - gt: dayjs() - .subtract(24 * 3600) - .toDate(), - }, - tags: { - some: { - name: "promptId", - value: "hello-world", - }, - }, - }, - include: { - modelResponse: true, - }, - orderBy: { - startTime: "desc", - }, -}); - -// Total spent on OpenAI in the last month -const totalSpent = await prisma.loggedCallModelResponse.aggregate({ - _sum: { - totalCost: true, - }, - where: { - originalLoggedCall: { - projectId: projectId, - }, - startTime: { - gt: dayjs() - .subtract(30 * 24 * 3600) - .toDate(), - }, - }, -}); diff --git a/app/src/server/utils/error.ts b/app/src/server/utils/error.ts deleted file mode 100644 index f284a8d..0000000 --- a/app/src/server/utils/error.ts +++ /dev/null @@ -1,6 +0,0 @@ -export default function userError(message: string): { status: "error"; message: string } { - return { - status: "error", - message, - }; -} diff --git a/app/src/state/persist.ts b/app/src/state/persist.ts index 4fc16ae..c7baa10 100644 --- a/app/src/state/persist.ts +++ b/app/src/state/persist.ts @@ -1,5 +1,5 @@ -import { PersistOptions } from "zustand/middleware/persist"; -import { State } from "./store"; +import { type PersistOptions } from "zustand/middleware/persist"; +import { type State } from "./store"; export const stateToPersist = { selectedProjectId: null as string | null, diff --git a/app/src/state/sharedVariantEditor.slice.ts b/app/src/state/sharedVariantEditor.slice.ts index 3c8030e..891b9a9 100644 --- a/app/src/state/sharedVariantEditor.slice.ts +++ b/app/src/state/sharedVariantEditor.slice.ts @@ -8,9 +8,9 @@ export const editorBackground = "#fafafa"; export type SharedVariantEditorSlice = { monaco: null | ReturnType; loadMonaco: () => Promise; - scenarios: RouterOutputs["scenarios"]["list"]["scenarios"]; + scenarioVars: RouterOutputs["scenarioVars"]["list"]; updateScenariosModel: () => void; - setScenarios: (scenarios: RouterOutputs["scenarios"]["list"]["scenarios"]) => void; + setScenarioVars: (scenarioVars: RouterOutputs["scenarioVars"]["list"]) => void; }; export const createVariantEditorSlice: SliceCreator = (set, get) => ({ @@ -60,10 +60,10 @@ export const createVariantEditorSlice: SliceCreator = }); get().sharedVariantEditor.updateScenariosModel(); }, - scenarios: [], - setScenarios: (scenarios) => { + scenarioVars: [], + setScenarioVars: (scenarios) => { set((state) => { - state.sharedVariantEditor.scenarios = scenarios; + state.sharedVariantEditor.scenarioVars = scenarios; }); get().sharedVariantEditor.updateScenariosModel(); @@ -73,17 +73,16 @@ export const createVariantEditorSlice: SliceCreator = const monaco = get().sharedVariantEditor.monaco; if (!monaco) return; - const modelContents = ` - const scenarios = ${JSON.stringify( - get().sharedVariantEditor.scenarios.map((s) => s.variableValues), - null, - 2, - )} as const; - - type Scenario = typeof scenarios[number]; - declare var scenario: Scenario | { [key: string]: string }; + const modelContents = ` + declare var scenario: { + ${get() + .sharedVariantEditor.scenarioVars.map((s) => `${s.label}: string;`) + .join("\n")} + }; `; + console.log(modelContents); + const scenariosModel = monaco.editor.getModel(monaco.Uri.parse("file:///scenarios.ts")); if (scenariosModel) { diff --git a/app/src/state/store.ts b/app/src/state/store.ts index bcdf5c4..ef0a590 100644 --- a/app/src/state/store.ts +++ b/app/src/state/store.ts @@ -7,7 +7,7 @@ import { createVariantEditorSlice, } from "./sharedVariantEditor.slice"; import { type APIClient } from "~/utils/api"; -import { persistOptions, stateToPersist } from "./persist"; +import { persistOptions, type stateToPersist } from "./persist"; export type State = { drawerOpen: boolean; diff --git a/app/src/state/sync.tsx b/app/src/state/sync.tsx index 050df75..3872bf9 100644 --- a/app/src/state/sync.tsx +++ b/app/src/state/sync.tsx @@ -1,16 +1,16 @@ import { useEffect } from "react"; import { api } from "~/utils/api"; -import { useScenarios } from "~/utils/hooks"; +import { useScenarioVars } from "~/utils/hooks"; import { useAppStore } from "./store"; export function useSyncVariantEditor() { - const scenarios = useScenarios(); + const scenarioVars = useScenarioVars(); useEffect(() => { - if (scenarios.data) { - useAppStore.getState().sharedVariantEditor.setScenarios(scenarios.data.scenarios); + if (scenarioVars.data) { + useAppStore.getState().sharedVariantEditor.setScenarioVars(scenarioVars.data); } - }, [scenarios.data]); + }, [scenarioVars.data]); } export function SyncAppStore() { diff --git a/app/src/tests/helpers/loadEnv.ts b/app/src/tests/helpers/loadEnv.ts new file mode 100644 index 0000000..9048754 --- /dev/null +++ b/app/src/tests/helpers/loadEnv.ts @@ -0,0 +1,5 @@ +import { configDotenv } from "dotenv"; + +configDotenv({ + path: ".env.test", +}); diff --git a/app/src/tests/helpers/setup.ts b/app/src/tests/helpers/setup.ts new file mode 100644 index 0000000..b6455da --- /dev/null +++ b/app/src/tests/helpers/setup.ts @@ -0,0 +1,13 @@ +import "./loadEnv"; +import { sql } from "kysely"; +import { beforeEach } from "vitest"; +import { kysely } from "~/server/db"; + +// Reset all Prisma data +const resetDb = async () => { + await sql`truncate "Experiment" cascade;`.execute(kysely); +}; + +beforeEach(async () => { + await resetDb(); +}); diff --git a/app/src/theme/ChakraThemeProvider.tsx b/app/src/theme/ChakraThemeProvider.tsx index a8b7ff7..5756476 100644 --- a/app/src/theme/ChakraThemeProvider.tsx +++ b/app/src/theme/ChakraThemeProvider.tsx @@ -1,4 +1,9 @@ -import { extendTheme, defineStyleConfig, ChakraProvider } from "@chakra-ui/react"; +import { + extendTheme, + defineStyleConfig, + ChakraProvider, + createStandaloneToast, +} from "@chakra-ui/react"; import "@fontsource/inconsolata"; import { modalAnatomy } from "@chakra-ui/anatomy"; import { createMultiStyleConfigHelpers } from "@chakra-ui/styled-system"; @@ -63,6 +68,15 @@ const theme = extendTheme({ }, }); +const { ToastContainer, toast } = createStandaloneToast(theme); + +export { toast }; + export const ChakraThemeProvider = ({ children }: { children: JSX.Element }) => { - return {children}; + return ( + + + {children} + + ); }; diff --git a/app/src/utils/hooks.ts b/app/src/utils/hooks.ts index d128fb8..78b6e44 100644 --- a/app/src/utils/hooks.ts +++ b/app/src/utils/hooks.ts @@ -157,3 +157,12 @@ export const useSelectedProject = () => { { enabled: !!selectedProjectId }, ); }; + +export const useScenarioVars = () => { + const experiment = useExperiment(); + + return api.scenarioVars.list.useQuery( + { experimentId: experiment.data?.id ?? "" }, + { enabled: experiment.data?.id != null }, + ); +}; diff --git a/app/src/utils/standardResponses.ts b/app/src/utils/standardResponses.ts new file mode 100644 index 0000000..e1897d6 --- /dev/null +++ b/app/src/utils/standardResponses.ts @@ -0,0 +1,31 @@ +import { toast } from "~/theme/ChakraThemeProvider"; + +export function error(message: string): { status: "error"; message: string } { + return { + status: "error", + message, + }; +} +export function success(payload: T): { status: "success"; payload: T }; +export function success(payload?: undefined): { status: "success"; payload: undefined }; +export function success(payload?: T) { + return { status: "success", payload }; +} + +type SuccessType = ReturnType>; +type ErrorType = ReturnType; + +// Used client-side to report generic errors +export function maybeReportError(response: SuccessType | ErrorType): response is ErrorType { + if (response.status === "error") { + toast({ + description: response.message, + status: "error", + duration: 5000, + isClosable: true, + }); + return true; + } + + return false; +} diff --git a/app/vitest.config.ts b/app/vitest.config.ts index c9f3325..2f7bde0 100644 --- a/app/vitest.config.ts +++ b/app/vitest.config.ts @@ -4,6 +4,10 @@ import { configDefaults, defineConfig, type UserConfig } from "vitest/config"; const config = defineConfig({ test: { ...configDefaults, // Extending Vitest's default options + setupFiles: ["./src/tests/helpers/setup.ts"], + + // Unfortunately using threads seems to cause issues with isolated-vm + threads: false, }, plugins: [tsconfigPaths()], }) as UserConfig;