Fix seeds and update eval field names
This commit is contained in:
@@ -1,4 +1,6 @@
|
||||
import { prisma } from "~/server/db";
|
||||
import dedent from "dedent";
|
||||
import { generateNewCell } from "~/server/utils/generateNewCell";
|
||||
|
||||
const experimentId = "11111111-1111-1111-1111-111111111111";
|
||||
|
||||
@@ -9,7 +11,7 @@ await prisma.experiment.deleteMany({
|
||||
},
|
||||
});
|
||||
|
||||
const experiment = await prisma.experiment.create({
|
||||
await prisma.experiment.create({
|
||||
data: {
|
||||
id: experimentId,
|
||||
label: "Country Capitals Example",
|
||||
@@ -37,28 +39,34 @@ await prisma.promptVariant.createMany({
|
||||
label: "Prompt Variant 1",
|
||||
sortIndex: 0,
|
||||
model: "gpt-3.5-turbo-0613",
|
||||
constructFn: `prompt = {
|
||||
model: "gpt-3.5-turbo-0613",
|
||||
messages: [{ role: "user", content: "What is the capital of {{country}}?" }],
|
||||
temperature: 0,
|
||||
}`,
|
||||
constructFn: dedent`
|
||||
prompt = {
|
||||
model: "gpt-3.5-turbo-0613",
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: \`What is the capital of ${"$"}{scenario.country}?\`
|
||||
}
|
||||
],
|
||||
temperature: 0,
|
||||
}`,
|
||||
},
|
||||
{
|
||||
experimentId,
|
||||
label: "Prompt Variant 2",
|
||||
sortIndex: 1,
|
||||
model: "gpt-3.5-turbo-0613",
|
||||
constructFn: `prompt = {
|
||||
model: "gpt-3.5-turbo-0613",
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content:
|
||||
"What is the capital of {{country}}? Return just the city name and nothing else.",
|
||||
},
|
||||
],
|
||||
temperature: 0,
|
||||
}`,
|
||||
constructFn: dedent`
|
||||
prompt = {
|
||||
model: "gpt-3.5-turbo-0613",
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: \`What is the capital of ${"$"}{scenario.country}? Return just the city name and nothing else.\`
|
||||
}
|
||||
],
|
||||
temperature: 0,
|
||||
}`,
|
||||
},
|
||||
],
|
||||
});
|
||||
@@ -109,3 +117,26 @@ await prisma.testScenario.createMany({
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const variants = await prisma.promptVariant.findMany({
|
||||
where: {
|
||||
experimentId,
|
||||
},
|
||||
});
|
||||
|
||||
const scenarios = await prisma.testScenario.findMany({
|
||||
where: {
|
||||
experimentId,
|
||||
},
|
||||
});
|
||||
|
||||
await Promise.all(
|
||||
variants
|
||||
.flatMap((variant) =>
|
||||
scenarios.map((scenario) => ({
|
||||
promptVariantId: variant.id,
|
||||
testScenarioId: scenario.id,
|
||||
})),
|
||||
)
|
||||
.map((cell) => generateNewCell(cell.promptVariantId, cell.testScenarioId)),
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user