alihan/OpenPipe-llm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Better scenario variable editing

Browse Source 
Some users have gotten confused by the scenario variable editing interface. This change makes the interface easier to understand.
This commit is contained in:
Kyle Corbitt
2023-08-10 12:08:17 -07:00
parent b8e0f392ab
commit 5ed7adadf9
27 changed files with 549 additions and 260 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
README.md
View File

@@ -65,7 +65,14 @@ OpenPipe includes a tool to generate new test scenarios based on your existing p
4. Clone this repository: `git clone https://github.com/openpipe/openpipe`
5. Install the dependencies: `cd openpipe && pnpm install`
6. Create a `.env` file (`cp .env.example .env`) and enter your `OPENAI_API_KEY`.
7. Update `DATABASE_URL` if necessary to point to your Postgres instance and run `pnpm prisma db push` to create the database.
7. Update `DATABASE_URL` if necessary to point to your Postgres instance and run `pnpm prisma migrate dev` to create the database.
8. Create a [GitHub OAuth App](https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/creating-an-oauth-app) and update the `GITHUB_CLIENT_ID` and `GITHUB_CLIENT_SECRET` values. (Note: a PR to make auth optional when running locally would be a great contribution!)
9. Start the app: `pnpm dev`.
10. Navigate to [http://localhost:3000](http://localhost:3000)
## Testing Locally
1. Copy your `.env` file to `.env.test`.
2. Update the `DATABASE_URL` to have a different database name than your development one
3. Run `DATABASE_URL=[your new datatase url] pnpm prisma migrate dev --skip-seed --skip-generate`
4. Run `pnpm test`

1
app/.gitignore vendored
View File

@@ -34,6 +34,7 @@ yarn-error.log*
# do not commit any .env files to git, except for the .env.example file. https://create.t3.gg/en/usage/env-variables#using-environment-variables
.env
.env*.local
.env.test
# vercel
.vercel

2
app/package.json
View File

@@ -19,7 +19,7 @@
"codegen": "tsx src/server/scripts/client-codegen.ts",
"seed": "tsx prisma/seed.ts",
"check": "concurrently 'pnpm lint' 'pnpm tsc' 'pnpm prettier . --check'",
"test": "pnpm vitest --no-threads"
"test": "pnpm vitest"
},
"dependencies": {
"@anthropic-ai/sdk": "^0.5.8",

1
app/prisma/migrations/20230810161437_add_uuid_extension/migration.sql Normal file
View File

@@ -0,0 +1 @@
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";

40
app/src/components/OutputsTable/EditEvaluations.tsx
View File

@@ -12,6 +12,7 @@ import {
Select,
FormHelperText,
Code,
IconButton,
} from "@chakra-ui/react";
import { type Evaluation, EvalType } from "@prisma/client";
import { useCallback, useState } from "react";
@@ -183,46 +184,37 @@ export default function EditEvaluations() {
<Text flex={1}>
{evaluation.evalType}: &quot;{evaluation.value}&quot;
</Text>
<Button
<IconButton
aria-label="Edit"
variant="unstyled"
color="gray.400"
height="unset"
width="unset"
minW="unset"
color="gray.400"
onClick={() => setEditingId(evaluation.id)}
_hover={{
color: "gray.800",
cursor: "pointer",
}}
>
<Icon as={BsPencil} boxSize={4} />
</Button>
<Button
_hover={{ color: "gray.800", cursor: "pointer" }}
icon={<Icon as={BsPencil} />}
/>
<IconButton
aria-label="Delete"
variant="unstyled"
color="gray.400"
height="unset"
width="unset"
minW="unset"
color="gray.400"
onClick={() => onDelete(evaluation.id)}
_hover={{
color: "gray.800",
cursor: "pointer",
}}
>
<Icon as={BsX} boxSize={6} />
</Button>
_hover={{ color: "gray.800", cursor: "pointer" }}
icon={<Icon as={BsX} boxSize={6} />}
/>
</HStack>
),
)}
{editingId == null && (
<Button
onClick={() => setEditingId("new")}
alignSelf="flex-start"
alignSelf="end"
size="sm"
mt={4}
colorScheme="blue"
>
Add Evaluation
New Evaluation
</Button>
)}
{editingId == "new" && (

234
app/src/components/OutputsTable/EditScenarioVars.tsx
View File

@@ -1,103 +1,185 @@
import { Text, Button, HStack, Heading, Icon, Input, Stack } from "@chakra-ui/react";
import { useState } from "react";
import { BsCheck, BsX } from "react-icons/bs";
import { Text, Button, HStack, Heading, Icon, IconButton, Stack, VStack } from "@chakra-ui/react";
import { type TemplateVariable } from "@prisma/client";
import { useEffect, useState } from "react";
import { BsPencil, BsX } from "react-icons/bs";
import { api } from "~/utils/api";
import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
import { useExperiment, useHandledAsyncCallback, useScenarioVars } from "~/utils/hooks";
import { maybeReportError } from "~/utils/standardResponses";
import { FloatingLabelInput } from "./FloatingLabelInput";
export const ScenarioVar = ({
variable,
isEditing,
setIsEditing,
}: {
variable: Pick<TemplateVariable, "id" | "label">;
isEditing: boolean;
setIsEditing: (isEditing: boolean) => void;
}) => {
const utils = api.useContext();
const [label, setLabel] = useState(variable.label);
useEffect(() => {
setLabel(variable.label);
}, [variable.label]);
const renameVarMutation = api.scenarioVars.rename.useMutation();
const [onRename] = useHandledAsyncCallback(async () => {
const resp = await renameVarMutation.mutateAsync({ id: variable.id, label });
if (maybeReportError(resp)) return;
setIsEditing(false);
await utils.scenarioVars.list.invalidate();
await utils.scenarios.list.invalidate();
}, [label, variable.id]);
const deleteMutation = api.scenarioVars.delete.useMutation();
const [onDeleteVar] = useHandledAsyncCallback(async () => {
await deleteMutation.mutateAsync({ id: variable.id });
await utils.scenarioVars.list.invalidate();
}, [variable.id]);
if (isEditing) {
return (
<HStack w="full">
<FloatingLabelInput
flex={1}
label="Renamed Variable"
value={label}
onChange={(e) => setLabel(e.target.value)}
onKeyDown={(e) => {
if (e.key === "Enter") {
e.preventDefault();
onRename();
}
// If the user types a space, replace it with an underscore
if (e.key === " ") {
e.preventDefault();
setLabel((label) => label && `${label}_`);
}
}}
/>
<Button size="sm" onClick={() => setIsEditing(false)}>
Cancel
</Button>
<Button size="sm" colorScheme="blue" onClick={onRename}>
Save
</Button>
</HStack>
);
} else {
return (
<HStack w="full" borderTopWidth={1} borderColor="gray.200">
<Text flex={1}>{variable.label}</Text>
<IconButton
aria-label="Edit"
variant="unstyled"
minW="unset"
color="gray.400"
onClick={() => setIsEditing(true)}
_hover={{ color: "gray.800", cursor: "pointer" }}
icon={<Icon as={BsPencil} />}
/>
<IconButton
aria-label="Delete"
variant="unstyled"
minW="unset"
color="gray.400"
onClick={onDeleteVar}
_hover={{ color: "gray.800", cursor: "pointer" }}
icon={<Icon as={BsX} boxSize={6} />}
/>
</HStack>
);
}
};
export default function EditScenarioVars() {
const experiment = useExperiment();
const vars =
api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" }).data ?? [];
const vars = useScenarioVars();
const [currentlyEditingId, setCurrentlyEditingId] = useState<string | null>(null);
const [newVariable, setNewVariable] = useState<string>("");
const newVarIsValid = newVariable.length > 0 && !vars.map((v) => v.label).includes(newVariable);
const newVarIsValid = newVariable?.length ?? 0 > 0;
const utils = api.useContext();
const addVarMutation = api.templateVars.create.useMutation();
const addVarMutation = api.scenarioVars.create.useMutation();
const [onAddVar] = useHandledAsyncCallback(async () => {
if (!experiment.data?.id) return;
if (!newVarIsValid) return;
await addVarMutation.mutateAsync({
if (!newVariable) return;
const resp = await addVarMutation.mutateAsync({
experimentId: experiment.data.id,
label: newVariable,
});
await utils.templateVars.list.invalidate();
if (maybeReportError(resp)) return;
await utils.scenarioVars.list.invalidate();
setNewVariable("");
}, [addVarMutation, experiment.data?.id, newVarIsValid, newVariable]);
const deleteMutation = api.templateVars.delete.useMutation();
const [onDeleteVar] = useHandledAsyncCallback(async (id: string) => {
await deleteMutation.mutateAsync({ id });
await utils.templateVars.list.invalidate();
}, []);
return (
<Stack>
<Heading size="sm">Scenario Variables</Heading>
<Stack spacing={2}>
<VStack spacing={4}>
<Text fontSize="sm">
Scenario variables can be used in your prompt variants as well as evaluations.
</Text>
<HStack spacing={0}>
<Input
placeholder="Add Scenario Variable"
size="sm"
borderTopRadius={0}
borderRightRadius={0}
value={newVariable}
onChange={(e) => setNewVariable(e.target.value)}
onKeyDown={(e) => {
if (e.key === "Enter") {
e.preventDefault();
onAddVar();
}
// If the user types a space, replace it with an underscore
if (e.key === " ") {
e.preventDefault();
setNewVariable((v) => v + "_");
}
}}
/>
<Button
size="xs"
height="100%"
borderLeftRadius={0}
isDisabled={!newVarIsValid}
onClick={onAddVar}
>
<Icon as={BsCheck} boxSize={8} />
</Button>
</HStack>
<HStack spacing={2} py={4} wrap="wrap">
{vars.map((variable) => (
<HStack
<VStack spacing={0} w="full">
{vars.data?.map((variable) => (
<ScenarioVar
variable={variable}
key={variable.id}
spacing={0}
bgColor="blue.100"
color="blue.600"
pl={2}
pr={0}
fontWeight="bold"
>
<Text fontSize="sm" flex={1}>
{variable.label}
</Text>
<Button
size="xs"
variant="ghost"
colorScheme="blue"
p="unset"
minW="unset"
px="unset"
onClick={() => onDeleteVar(variable.id)}
>
<Icon as={BsX} boxSize={6} color="blue.800" />
</Button>
</HStack>
isEditing={currentlyEditingId === variable.id}
setIsEditing={(isEditing) => {
if (isEditing) {
setCurrentlyEditingId(variable.id);
} else {
setCurrentlyEditingId(null);
}
}}
/>
))}
</HStack>
</Stack>
</VStack>
{currentlyEditingId !== "new" && (
<Button
colorScheme="blue"
size="sm"
onClick={() => setCurrentlyEditingId("new")}
alignSelf="end"
>
New Variable
</Button>
)}
{currentlyEditingId === "new" && (
<HStack w="full">
<FloatingLabelInput
flex={1}
label="New Variable"
value={newVariable}
onChange={(e) => setNewVariable(e.target.value)}
onKeyDown={(e) => {
if (e.key === "Enter") {
e.preventDefault();
onAddVar();
}
// If the user types a space, replace it with an underscore
if (e.key === " ") {
e.preventDefault();
setNewVariable((v) => v && `${v}_`);
}
}}
/>
<Button size="sm" onClick={() => setCurrentlyEditingId(null)}>
Cancel
</Button>
<Button size="sm" colorScheme="blue" onClick={onAddVar}>
Save
</Button>
</HStack>
)}
</VStack>
</Stack>
);
}

7
app/src/components/OutputsTable/OutputCell/OutputCell.tsx
View File

@@ -1,7 +1,7 @@
import { api } from "~/utils/api";
import { type PromptVariant, type Scenario } from "../types";
import { type StackProps, Text, VStack } from "@chakra-ui/react";
import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
import { useScenarioVars, useHandledAsyncCallback } from "~/utils/hooks";
import SyntaxHighlighter from "react-syntax-highlighter";
import { docco } from "react-syntax-highlighter/dist/cjs/styles/hljs";
import stringify from "json-stringify-pretty-compact";
@@ -23,10 +23,7 @@ export default function OutputCell({
variant: PromptVariant;
}): ReactElement | null {
const utils = api.useContext();
const experiment = useExperiment();
const vars = api.templateVars.list.useQuery({
experimentId: experiment.data?.id ?? "",
}).data;
const vars = useScenarioVars().data;
const scenarioVariables = scenario.variableValues as Record<string, string>;
const templateHasVariables =

5
app/src/components/OutputsTable/ScenarioEditor.tsx
View File

@@ -1,7 +1,7 @@
import { isEqual } from "lodash-es";
import { useEffect, useState, type DragEvent } from "react";
import { api } from "~/utils/api";
import { useExperiment, useExperimentAccess, useHandledAsyncCallback } from "~/utils/hooks";
import { useExperimentAccess, useHandledAsyncCallback, useScenarioVars } from "~/utils/hooks";
import { type Scenario } from "./types";
import {
@@ -41,8 +41,7 @@ export default function ScenarioEditor({
if (savedValues) setValues(savedValues);
}, [savedValues]);
const experiment = useExperiment();
const vars = api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
const vars = useScenarioVars();
const variableLabels = vars.data?.map((v) => v.label) ?? [];

2
app/src/components/OutputsTable/ScenarioEditorModal.tsx
View File

@@ -58,7 +58,7 @@ export const ScenarioEditorModal = ({
await utils.scenarios.list.invalidate();
}, [mutation, values]);
const vars = api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
const vars = api.scenarioVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
const variableLabels = vars.data?.map((v) => v.label) ?? [];
return (

2
app/src/components/OutputsTable/ScenariosHeader.tsx
View File

@@ -72,7 +72,7 @@ export const ScenariosHeader = () => {
Autogenerate Scenario
</MenuItem>
<MenuItem icon={<BsPencil />} onClick={openDrawer}>
Edit Vars
Add or Remove Variables
</MenuItem>
</MenuList>
</Menu>

4
app/src/server/api/root.router.ts
View File

@@ -3,7 +3,7 @@ import { createTRPCRouter } from "~/server/api/trpc";
import { experimentsRouter } from "./routers/experiments.router";
import { scenariosRouter } from "./routers/scenarios.router";
import { scenarioVariantCellsRouter } from "./routers/scenarioVariantCells.router";
import { templateVarsRouter } from "./routers/templateVariables.router";
import { scenarioVarsRouter } from "./routers/scenarioVariables.router";
import { evaluationsRouter } from "./routers/evaluations.router";
import { worldChampsRouter } from "./routers/worldChamps.router";
import { datasetsRouter } from "./routers/datasets.router";
@@ -22,7 +22,7 @@ export const appRouter = createTRPCRouter({
experiments: experimentsRouter,
scenarios: scenariosRouter,
scenarioVariantCells: scenarioVariantCellsRouter,
templateVars: templateVarsRouter,
scenarioVars: scenarioVarsRouter,
evaluations: evaluationsRouter,
worldChamps: worldChampsRouter,
datasets: datasetsRouter,

8
app/src/server/api/routers/promptVariants.router.ts
View File

@@ -3,7 +3,7 @@ import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/
import { prisma } from "~/server/db";
import { Prisma } from "@prisma/client";
import { generateNewCell } from "~/server/utils/generateNewCell";
import userError from "~/server/utils/error";
import { error, success } from "~/utils/standardResponses";
import { recordExperimentUpdated } from "~/server/utils/recordExperimentUpdated";
import { reorderPromptVariants } from "~/server/utils/reorderPromptVariants";
import { type PromptVariant } from "@prisma/client";
@@ -315,7 +315,7 @@ export const promptVariantsRouter = createTRPCRouter({
const constructedPrompt = await parsePromptConstructor(existing.promptConstructor);
if ("error" in constructedPrompt) {
return userError(constructedPrompt.error);
return error(constructedPrompt.error);
}
const model = input.newModel
@@ -353,7 +353,7 @@ export const promptVariantsRouter = createTRPCRouter({
const parsedPrompt = await parsePromptConstructor(input.promptConstructor);
if ("error" in parsedPrompt) {
return userError(parsedPrompt.error);
return error(parsedPrompt.error);
}
// Create a duplicate with only the config changed
@@ -398,7 +398,7 @@ export const promptVariantsRouter = createTRPCRouter({
});
}
return { status: "ok" } as const;
return success();
}),
reorder: protectedProcedure

143
app/src/server/api/routers/scenarioVariables.router.ts Normal file
View File

@@ -0,0 +1,143 @@
import { type TemplateVariable } from "@prisma/client";
import { sql } from "kysely";
import { z } from "zod";
import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/api/trpc";
import { kysely, prisma } from "~/server/db";
import { error, success } from "~/utils/standardResponses";
import { requireCanModifyExperiment, requireCanViewExperiment } from "~/utils/accessControl";
export const scenarioVarsRouter = createTRPCRouter({
create: protectedProcedure
.input(z.object({ experimentId: z.string(), label: z.string() }))
.mutation(async ({ input, ctx }) => {
await requireCanModifyExperiment(input.experimentId, ctx);
// Make sure there isn't an existing variable with the same name
const existingVariable = await prisma.templateVariable.findFirst({
where: {
experimentId: input.experimentId,
label: input.label,
},
});
if (existingVariable) {
return error(`A variable named ${input.label} already exists.`);
}
await prisma.templateVariable.create({
data: {
experimentId: input.experimentId,
label: input.label,
},
});
return success();
}),
rename: protectedProcedure
.input(z.object({ id: z.string(), label: z.string() }))
.mutation(async ({ input, ctx }) => {
const templateVariable = await prisma.templateVariable.findUniqueOrThrow({
where: { id: input.id },
});
await requireCanModifyExperiment(templateVariable.experimentId, ctx);
// Make sure there isn't an existing variable with the same name
const existingVariable = await prisma.templateVariable.findFirst({
where: {
experimentId: templateVariable.experimentId,
label: input.label,
},
});
if (existingVariable) {
return error(`A variable named ${input.label} already exists.`);
}
await renameTemplateVariable(templateVariable, input.label);
return success();
}),
delete: protectedProcedure
.input(z.object({ id: z.string() }))
.mutation(async ({ input, ctx }) => {
const { experimentId } = await prisma.templateVariable.findUniqueOrThrow({
where: { id: input.id },
});
await requireCanModifyExperiment(experimentId, ctx);
await prisma.templateVariable.delete({ where: { id: input.id } });
}),
list: publicProcedure
.input(z.object({ experimentId: z.string() }))
.query(async ({ input, ctx }) => {
await requireCanViewExperiment(input.experimentId, ctx);
return await prisma.templateVariable.findMany({
where: {
experimentId: input.experimentId,
},
orderBy: {
createdAt: "asc",
},
select: {
id: true,
label: true,
},
});
}),
});
export const renameTemplateVariable = async (
templateVariable: TemplateVariable,
newLabel: string,
) => {
const { experimentId } = templateVariable;
await kysely.transaction().execute(async (trx) => {
await trx
.updateTable("TemplateVariable")
.set({
label: newLabel,
})
.where("id", "=", templateVariable.id)
.execute();
await sql`
CREATE TEMP TABLE "TempTestScenario" AS
SELECT *
FROM "TestScenario"
WHERE "experimentId" = ${experimentId}
-- Only copy the rows that actually have a value for the variable, no reason to churn the rest and simplifies the update.
AND "variableValues"->${templateVariable.label} IS NOT NULL
`.execute(trx);
await sql`
UPDATE "TempTestScenario"
SET "variableValues" = jsonb_set(
"variableValues",
${`{${newLabel}}`},
"variableValues"->${templateVariable.label}
) - ${templateVariable.label},
"updatedAt" = NOW(),
"id" = uuid_generate_v4()
`.execute(trx);
// Print the contents of the temp table
const results = await sql`SELECT * FROM "TempTestScenario"`.execute(trx);
console.log(results.rows);
await trx
.updateTable("TestScenario")
.set({
visible: false,
})
.where("experimentId", "=", experimentId)
.execute();
await sql`
INSERT INTO "TestScenario" (id, "variableValues", "uiId", visible, "sortIndex", "experimentId", "createdAt", "updatedAt")
SELECT * FROM "TempTestScenario";
`.execute(trx);
});
};

110
app/src/server/api/routers/templateVariables.router.test.ts Normal file
View File

@@ -0,0 +1,110 @@
import { expect, it } from "vitest";
import { prisma } from "~/server/db";
import { renameTemplateVariable } from "./scenarioVariables.router";
const createExperiment = async () => {
return await prisma.experiment.create({
data: {
label: "Test Experiment",
project: {
create: {},
},
},
});
};
const createTemplateVar = async (experimentId: string, label: string) => {
return await prisma.templateVariable.create({
data: {
experimentId,
label,
},
});
};
it("renames templateVariables", async () => {
// Create experiments concurrently
const [exp1, exp2] = await Promise.all([createExperiment(), createExperiment()]);
// Create template variables concurrently
const [exp1Var, exp2Var1, exp2Var2] = await Promise.all([
createTemplateVar(exp1.id, "input1"),
createTemplateVar(exp2.id, "input1"),
createTemplateVar(exp2.id, "input2"),
]);
// Create test scenarios concurrently
const [exp1Scenario, exp2Scenario, exp2HiddenScenario] = await Promise.all([
prisma.testScenario.create({
data: {
experimentId: exp1.id,
visible: true,
variableValues: { input1: "test" },
},
}),
prisma.testScenario.create({
data: {
experimentId: exp2.id,
visible: true,
variableValues: { input1: "test1", otherInput: "otherTest" },
},
}),
prisma.testScenario.create({
data: {
experimentId: exp2.id,
visible: false,
variableValues: { otherInput: "otherTest2" },
},
}),
]);
await renameTemplateVariable(exp2Var1, "input1-renamed");
expect(await prisma.templateVariable.findUnique({ where: { id: exp2Var1.id } })).toMatchObject({
label: "input1-renamed",
});
// It shouldn't mess with unrelated experiments
expect(await prisma.testScenario.findUnique({ where: { id: exp1Scenario.id } })).toMatchObject({
visible: true,
variableValues: { input1: "test" },
});
// Make sure there are a total of 4 scenarios for exp2
expect(
await prisma.testScenario.count({
where: {
experimentId: exp2.id,
},
}),
).toBe(3);
// It shouldn't mess with the existing scenarios, except to hide them
expect(await prisma.testScenario.findUnique({ where: { id: exp2Scenario.id } })).toMatchObject({
visible: false,
variableValues: { input1: "test1", otherInput: "otherTest" },
});
// It should create a new scenario with the new variable name
const newScenario1 = await prisma.testScenario.findFirst({
where: {
experimentId: exp2.id,
variableValues: { equals: { "input1-renamed": "test1", otherInput: "otherTest" } },
},
});
expect(newScenario1).toMatchObject({
visible: true,
});
const newScenario2 = await prisma.testScenario.findFirst({
where: {
experimentId: exp2.id,
variableValues: { equals: { otherInput: "otherTest2" } },
},
});
expect(newScenario2).toMatchObject({
visible: false,
});
});

49
app/src/server/api/routers/templateVariables.router.ts
View File

@@ -1,49 +0,0 @@
import { z } from "zod";
import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/api/trpc";
import { prisma } from "~/server/db";
import { requireCanModifyExperiment, requireCanViewExperiment } from "~/utils/accessControl";
export const templateVarsRouter = createTRPCRouter({
create: protectedProcedure
.input(z.object({ experimentId: z.string(), label: z.string() }))
.mutation(async ({ input, ctx }) => {
await requireCanModifyExperiment(input.experimentId, ctx);
await prisma.templateVariable.create({
data: {
experimentId: input.experimentId,
label: input.label,
},
});
}),
delete: protectedProcedure
.input(z.object({ id: z.string() }))
.mutation(async ({ input, ctx }) => {
const { experimentId } = await prisma.templateVariable.findUniqueOrThrow({
where: { id: input.id },
});
await requireCanModifyExperiment(experimentId, ctx);
await prisma.templateVariable.delete({ where: { id: input.id } });
}),
list: publicProcedure
.input(z.object({ experimentId: z.string() }))
.query(async ({ input, ctx }) => {
await requireCanViewExperiment(input.experimentId, ctx);
return await prisma.templateVariable.findMany({
where: {
experimentId: input.experimentId,
},
orderBy: {
createdAt: "asc",
},
select: {
id: true,
label: true,
},
});
}),
});

63
app/src/server/scripts/test-queries.ts
View File

@@ -1,63 +0,0 @@
import dayjs from "dayjs";
import { prisma } from "../db";
const projectId = "1234";
// Find all calls in the last 24 hours
const responses = await prisma.loggedCall.findMany({
where: {
projectId: projectId,
startTime: {
gt: dayjs()
.subtract(24 * 3600)
.toDate(),
},
},
include: {
modelResponse: true,
},
orderBy: {
startTime: "desc",
},
});
// Find all calls in the last 24 hours with promptId 'hello-world'
const helloWorld = await prisma.loggedCall.findMany({
where: {
projectId: projectId,
startTime: {
gt: dayjs()
.subtract(24 * 3600)
.toDate(),
},
tags: {
some: {
name: "promptId",
value: "hello-world",
},
},
},
include: {
modelResponse: true,
},
orderBy: {
startTime: "desc",
},
});
// Total spent on OpenAI in the last month
const totalSpent = await prisma.loggedCallModelResponse.aggregate({
_sum: {
totalCost: true,
},
where: {
originalLoggedCall: {
projectId: projectId,
},
startTime: {
gt: dayjs()
.subtract(30 * 24 * 3600)
.toDate(),
},
},
});

6
app/src/server/utils/error.ts
View File

@@ -1,6 +0,0 @@
export default function userError(message: string): { status: "error"; message: string } {
return {
status: "error",
message,
};
}

4
app/src/state/persist.ts
View File

@@ -1,5 +1,5 @@
import { PersistOptions } from "zustand/middleware/persist";
import { State } from "./store";
import { type PersistOptions } from "zustand/middleware/persist";
import { type State } from "./store";
export const stateToPersist = {
selectedProjectId: null as string | null,

27
app/src/state/sharedVariantEditor.slice.ts
View File

@@ -8,9 +8,9 @@ export const editorBackground = "#fafafa";
export type SharedVariantEditorSlice = {
monaco: null | ReturnType<typeof loader.__getMonacoInstance>;
loadMonaco: () => Promise<void>;
scenarios: RouterOutputs["scenarios"]["list"]["scenarios"];
scenarioVars: RouterOutputs["scenarioVars"]["list"];
updateScenariosModel: () => void;
setScenarios: (scenarios: RouterOutputs["scenarios"]["list"]["scenarios"]) => void;
setScenarioVars: (scenarioVars: RouterOutputs["scenarioVars"]["list"]) => void;
};
export const createVariantEditorSlice: SliceCreator<SharedVariantEditorSlice> = (set, get) => ({
@@ -60,10 +60,10 @@ export const createVariantEditorSlice: SliceCreator<SharedVariantEditorSlice> =
});
get().sharedVariantEditor.updateScenariosModel();
},
scenarios: [],
setScenarios: (scenarios) => {
scenarioVars: [],
setScenarioVars: (scenarios) => {
set((state) => {
state.sharedVariantEditor.scenarios = scenarios;
state.sharedVariantEditor.scenarioVars = scenarios;
});
get().sharedVariantEditor.updateScenariosModel();
@@ -73,17 +73,16 @@ export const createVariantEditorSlice: SliceCreator<SharedVariantEditorSlice> =
const monaco = get().sharedVariantEditor.monaco;
if (!monaco) return;
const modelContents = `
const scenarios = ${JSON.stringify(
get().sharedVariantEditor.scenarios.map((s) => s.variableValues),
null,
2,
)} as const;
type Scenario = typeof scenarios[number];
declare var scenario: Scenario | { [key: string]: string };
const modelContents = `
declare var scenario: {
${get()
.sharedVariantEditor.scenarioVars.map((s) => `${s.label}: string;`)
.join("\n")}
};
`;
console.log(modelContents);
const scenariosModel = monaco.editor.getModel(monaco.Uri.parse("file:///scenarios.ts"));
if (scenariosModel) {

2
app/src/state/store.ts
View File

@@ -7,7 +7,7 @@ import {
createVariantEditorSlice,
} from "./sharedVariantEditor.slice";
import { type APIClient } from "~/utils/api";
import { persistOptions, stateToPersist } from "./persist";
import { persistOptions, type stateToPersist } from "./persist";
export type State = {
drawerOpen: boolean;

10
app/src/state/sync.tsx
View File

@@ -1,16 +1,16 @@
import { useEffect } from "react";
import { api } from "~/utils/api";
import { useScenarios } from "~/utils/hooks";
import { useScenarioVars } from "~/utils/hooks";
import { useAppStore } from "./store";
export function useSyncVariantEditor() {
const scenarios = useScenarios();
const scenarioVars = useScenarioVars();
useEffect(() => {
if (scenarios.data) {
useAppStore.getState().sharedVariantEditor.setScenarios(scenarios.data.scenarios);
if (scenarioVars.data) {
useAppStore.getState().sharedVariantEditor.setScenarioVars(scenarioVars.data);
}
}, [scenarios.data]);
}, [scenarioVars.data]);
}
export function SyncAppStore() {

5
app/src/tests/helpers/loadEnv.ts Normal file
View File

@@ -0,0 +1,5 @@
import { configDotenv } from "dotenv";
configDotenv({
path: ".env.test",
});

13
app/src/tests/helpers/setup.ts Normal file
View File

@@ -0,0 +1,13 @@
import "./loadEnv";
import { sql } from "kysely";
import { beforeEach } from "vitest";
import { kysely } from "~/server/db";
// Reset all Prisma data
const resetDb = async () => {
await sql`truncate "Experiment" cascade;`.execute(kysely);
};
beforeEach(async () => {
await resetDb();
});

18
app/src/theme/ChakraThemeProvider.tsx
View File

@@ -1,4 +1,9 @@
import { extendTheme, defineStyleConfig, ChakraProvider } from "@chakra-ui/react";
import {
extendTheme,
defineStyleConfig,
ChakraProvider,
createStandaloneToast,
} from "@chakra-ui/react";
import "@fontsource/inconsolata";
import { modalAnatomy } from "@chakra-ui/anatomy";
import { createMultiStyleConfigHelpers } from "@chakra-ui/styled-system";
@@ -63,6 +68,15 @@ const theme = extendTheme({
},
});
const { ToastContainer, toast } = createStandaloneToast(theme);
export { toast };
export const ChakraThemeProvider = ({ children }: { children: JSX.Element }) => {
return <ChakraProvider theme={theme}>{children}</ChakraProvider>;
return (
<ChakraProvider theme={theme}>
<ToastContainer />
{children}
</ChakraProvider>
);
};

9
app/src/utils/hooks.ts
View File

@@ -157,3 +157,12 @@ export const useSelectedProject = () => {
{ enabled: !!selectedProjectId },
);
};
export const useScenarioVars = () => {
const experiment = useExperiment();
return api.scenarioVars.list.useQuery(
{ experimentId: experiment.data?.id ?? "" },
{ enabled: experiment.data?.id != null },
);
};

31
app/src/utils/standardResponses.ts Normal file
View File

@@ -0,0 +1,31 @@
import { toast } from "~/theme/ChakraThemeProvider";
export function error(message: string): { status: "error"; message: string } {
return {
status: "error",
message,
};
}
export function success<T>(payload: T): { status: "success"; payload: T };
export function success(payload?: undefined): { status: "success"; payload: undefined };
export function success<T>(payload?: T) {
return { status: "success", payload };
}
type SuccessType<T> = ReturnType<typeof success<T>>;
type ErrorType = ReturnType<typeof error>;
// Used client-side to report generic errors
export function maybeReportError<T>(response: SuccessType<T> | ErrorType): response is ErrorType {
if (response.status === "error") {
toast({
description: response.message,
status: "error",
duration: 5000,
isClosable: true,
});
return true;
}
return false;
}

4
app/vitest.config.ts
View File

@@ -4,6 +4,10 @@ import { configDefaults, defineConfig, type UserConfig } from "vitest/config";
const config = defineConfig({
test: {
...configDefaults, // Extending Vitest's default options
setupFiles: ["./src/tests/helpers/setup.ts"],
// Unfortunately using threads seems to cause issues with isolated-vm
threads: false,
},
plugins: [tsconfigPaths()],
}) as UserConfig;