Fix seeds and update eval field names

This commit is contained in:
Kyle Corbitt
2023-07-17 14:14:20 -07:00
parent 6b84a59372
commit 54369dba54
18 changed files with 136 additions and 80 deletions

View File

@@ -1,4 +1,6 @@
import { prisma } from "~/server/db";
import dedent from "dedent";
import { generateNewCell } from "~/server/utils/generateNewCell";
const experimentId = "11111111-1111-1111-1111-111111111111";
@@ -9,7 +11,7 @@ await prisma.experiment.deleteMany({
},
});
const experiment = await prisma.experiment.create({
await prisma.experiment.create({
data: {
id: experimentId,
label: "Country Capitals Example",
@@ -37,28 +39,34 @@ await prisma.promptVariant.createMany({
label: "Prompt Variant 1",
sortIndex: 0,
model: "gpt-3.5-turbo-0613",
constructFn: `prompt = {
model: "gpt-3.5-turbo-0613",
messages: [{ role: "user", content: "What is the capital of {{country}}?" }],
temperature: 0,
}`,
constructFn: dedent`
prompt = {
model: "gpt-3.5-turbo-0613",
messages: [
{
role: "user",
content: \`What is the capital of ${"$"}{scenario.country}?\`
}
],
temperature: 0,
}`,
},
{
experimentId,
label: "Prompt Variant 2",
sortIndex: 1,
model: "gpt-3.5-turbo-0613",
constructFn: `prompt = {
model: "gpt-3.5-turbo-0613",
messages: [
{
role: "user",
content:
"What is the capital of {{country}}? Return just the city name and nothing else.",
},
],
temperature: 0,
}`,
constructFn: dedent`
prompt = {
model: "gpt-3.5-turbo-0613",
messages: [
{
role: "user",
content: \`What is the capital of ${"$"}{scenario.country}? Return just the city name and nothing else.\`
}
],
temperature: 0,
}`,
},
],
});
@@ -109,3 +117,26 @@ await prisma.testScenario.createMany({
},
],
});
const variants = await prisma.promptVariant.findMany({
where: {
experimentId,
},
});
const scenarios = await prisma.testScenario.findMany({
where: {
experimentId,
},
});
await Promise.all(
variants
.flatMap((variant) =>
scenarios.map((scenario) => ({
promptVariantId: variant.id,
testScenarioId: scenario.id,
})),
)
.map((cell) => generateNewCell(cell.promptVariantId, cell.testScenarioId)),
);