1dcdba04a6
Allows for the creation of user accounts. A few notes on the specifics: - Experiments are the main access control objects. If you can view an experiment, you can view all its prompts/scenarios/evals. If you can edit it, you can edit or delete all of those as well. - Experiments are owned by Organizations in the database. Organizations can have multiple members and members can have roles of ADMIN, MEMBER or VIEWER. - Organizations can either be "personal" or general. Each user has a "personal" organization created as soon as they try to create an experiment. There's currently no UI support for creating general orgs or adding users to them; they're just in the database to future-proof all the ACL logic. - You can require that a user is signed-in to see a route using the `protectedProcedure` helper. When you use `protectedProcedure`, you also have to call `ctx.markAccessControlRun()` (or delegate to a function that does it for you; see accessControl.ts). This is to remind us to actually check for access control when we define a new endpoint.
150 lines
2.9 KiB
TypeScript
150 lines
2.9 KiB
TypeScript
import { prisma } from "~/server/db";
|
|
import dedent from "dedent";
|
|
import { generateNewCell } from "~/server/utils/generateNewCell";
|
|
|
|
const defaultId = "11111111-1111-1111-1111-111111111111";
|
|
|
|
await prisma.organization.deleteMany({
|
|
where: { id: defaultId },
|
|
});
|
|
await prisma.organization.create({
|
|
data: { id: defaultId },
|
|
});
|
|
|
|
await prisma.experiment.deleteMany({
|
|
where: {
|
|
id: defaultId,
|
|
},
|
|
});
|
|
|
|
await prisma.experiment.create({
|
|
data: {
|
|
id: defaultId,
|
|
label: "Country Capitals Example",
|
|
organizationId: defaultId,
|
|
},
|
|
});
|
|
|
|
await prisma.scenarioVariantCell.deleteMany({
|
|
where: {
|
|
promptVariant: {
|
|
experimentId: defaultId,
|
|
},
|
|
},
|
|
});
|
|
|
|
await prisma.promptVariant.deleteMany({
|
|
where: {
|
|
experimentId: defaultId,
|
|
},
|
|
});
|
|
|
|
await prisma.promptVariant.createMany({
|
|
data: [
|
|
{
|
|
experimentId: defaultId,
|
|
label: "Prompt Variant 1",
|
|
sortIndex: 0,
|
|
model: "gpt-3.5-turbo-0613",
|
|
constructFn: dedent`
|
|
prompt = {
|
|
model: "gpt-3.5-turbo-0613",
|
|
messages: [
|
|
{
|
|
role: "user",
|
|
content: \`What is the capital of ${"$"}{scenario.country}?\`
|
|
}
|
|
],
|
|
temperature: 0,
|
|
}`,
|
|
},
|
|
{
|
|
experimentId: defaultId,
|
|
label: "Prompt Variant 2",
|
|
sortIndex: 1,
|
|
model: "gpt-3.5-turbo-0613",
|
|
constructFn: dedent`
|
|
prompt = {
|
|
model: "gpt-3.5-turbo-0613",
|
|
messages: [
|
|
{
|
|
role: "user",
|
|
content: \`What is the capital of ${"$"}{scenario.country}? Return just the city name and nothing else.\`
|
|
}
|
|
],
|
|
temperature: 0,
|
|
}`,
|
|
},
|
|
],
|
|
});
|
|
|
|
await prisma.templateVariable.deleteMany({
|
|
where: {
|
|
experimentId: defaultId,
|
|
},
|
|
});
|
|
|
|
await prisma.templateVariable.createMany({
|
|
data: [
|
|
{
|
|
experimentId: defaultId,
|
|
label: "country",
|
|
},
|
|
],
|
|
});
|
|
|
|
await prisma.testScenario.deleteMany({
|
|
where: {
|
|
experimentId: defaultId,
|
|
},
|
|
});
|
|
|
|
await prisma.testScenario.createMany({
|
|
data: [
|
|
{
|
|
experimentId: defaultId,
|
|
sortIndex: 0,
|
|
variableValues: {
|
|
country: "Spain",
|
|
},
|
|
},
|
|
{
|
|
experimentId: defaultId,
|
|
sortIndex: 1,
|
|
variableValues: {
|
|
country: "USA",
|
|
},
|
|
},
|
|
{
|
|
experimentId: defaultId,
|
|
sortIndex: 2,
|
|
variableValues: {
|
|
country: "Chile",
|
|
},
|
|
},
|
|
],
|
|
});
|
|
|
|
const variants = await prisma.promptVariant.findMany({
|
|
where: {
|
|
experimentId: defaultId,
|
|
},
|
|
});
|
|
|
|
const scenarios = await prisma.testScenario.findMany({
|
|
where: {
|
|
experimentId: defaultId,
|
|
},
|
|
});
|
|
|
|
await Promise.all(
|
|
variants
|
|
.flatMap((variant) =>
|
|
scenarios.map((scenario) => ({
|
|
promptVariantId: variant.id,
|
|
testScenarioId: scenario.id,
|
|
})),
|
|
)
|
|
.map((cell) => generateNewCell(cell.promptVariantId, cell.testScenarioId)),
|
|
);
|