16aa6672fc
We'll probably need a concept of organizations at some point in the future, but in practice the way we're using these in the codebase right now is as a project, so this renames it to that to avoid confusion.
115 lines
2.8 KiB
TypeScript
115 lines
2.8 KiB
TypeScript
import { prisma } from "~/server/db";
|
|
import dedent from "dedent";
|
|
import fs from "fs";
|
|
import { parse } from "csv-parse/sync";
|
|
import { promptConstructorVersion } from "~/promptConstructor/version";
|
|
|
|
const defaultId = "11111111-1111-1111-1111-111111111112";
|
|
|
|
await prisma.project.deleteMany({
|
|
where: { id: defaultId },
|
|
});
|
|
|
|
// If there's an existing project, just seed into it
|
|
const project =
|
|
(await prisma.project.findFirst({})) ??
|
|
(await prisma.project.create({
|
|
data: { id: defaultId },
|
|
}));
|
|
|
|
type Scenario = {
|
|
text: string;
|
|
sentiment: string;
|
|
emotion: string;
|
|
};
|
|
|
|
const experimentName = `Twitter Sentiment Analysis`;
|
|
const oldExperiment = await prisma.experiment.findFirst({
|
|
where: {
|
|
label: experimentName,
|
|
projectId: project.id,
|
|
},
|
|
});
|
|
if (oldExperiment) {
|
|
await prisma.experiment.deleteMany({
|
|
where: { id: oldExperiment.id },
|
|
});
|
|
}
|
|
|
|
const experiment = await prisma.experiment.create({
|
|
data: {
|
|
id: oldExperiment?.id ?? undefined,
|
|
label: experimentName,
|
|
projectId: project.id,
|
|
},
|
|
});
|
|
|
|
const content = fs.readFileSync("./prisma/datasets/validated_tweets.csv", "utf8");
|
|
|
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
const records: any[] = parse(content, { delimiter: ",", from_line: 2 });
|
|
|
|
console.log("records", records);
|
|
|
|
const scenarios: Scenario[] = records.map((row) => ({
|
|
text: row[0],
|
|
sentiment: row[1],
|
|
emotion: row[2],
|
|
}));
|
|
|
|
console.log("scenarios", scenarios.length);
|
|
|
|
await prisma.testScenario.createMany({
|
|
data: scenarios.slice(0, 30).map((scenario, i) => ({
|
|
experimentId: experiment.id,
|
|
sortIndex: i,
|
|
variableValues: {
|
|
text: scenario.text,
|
|
sentiment: scenario.sentiment,
|
|
emotion: scenario.emotion,
|
|
},
|
|
})),
|
|
});
|
|
|
|
await prisma.templateVariable.createMany({
|
|
data: ["text", "sentiment", "emotion"].map((label) => ({
|
|
experimentId: experiment.id,
|
|
label,
|
|
})),
|
|
});
|
|
|
|
await prisma.promptVariant.createMany({
|
|
data: [
|
|
{
|
|
experimentId: experiment.id,
|
|
label: "Prompt Variant 1",
|
|
sortIndex: 0,
|
|
model: "gpt-3.5-turbo-0613",
|
|
modelProvider: "openai/ChatCompletion",
|
|
promptConstructorVersion,
|
|
promptConstructor: dedent`
|
|
definePrompt("openai/ChatCompletion", {
|
|
model: "gpt-3.5-turbo-0613",
|
|
messages: [
|
|
{
|
|
role: "user",
|
|
content: \`Text: ${"$"}{scenario.text}\n\nRespond with the sentiment (negative|neutral|positive) and emotion (optimism|joy|anger|sadness) of the tweet in this format: "answer: <sentiment>-<emotion>".\`
|
|
}
|
|
],
|
|
temperature: 0,
|
|
})`,
|
|
},
|
|
],
|
|
});
|
|
|
|
await prisma.evaluation.createMany({
|
|
data: [
|
|
{
|
|
experimentId: experiment.id,
|
|
label: "Eval",
|
|
evalType: "CONTAINS",
|
|
value: "answer: {{sentiment}}-{{emotion}}",
|
|
},
|
|
],
|
|
});
|