d6b97b29f7
* Move DeleteButton into a separate file * Rename plural relations * Add ability to fork * Fork automatically after auth upon return * Add experiment card skeleton * Create HeaderButtons component * return no header buttons while experiment loading * Fix prettier * Remove unused variable * Remove newline * Default json values to undefined * Change header styles * Fix prettier * Give AddScenario icon less width * Move useEffect * Skip invalidating experiments list after forking * Require user to be able to view experiment to fork it * Move experiment creation into same transaction * Only return the forked experiment id * Put delete button in experiment settings drawer * Move useEffect hook
80 lines
2.0 KiB
TypeScript
80 lines
2.0 KiB
TypeScript
import { type ModelOutput, type Evaluation } from "@prisma/client";
|
|
import { prisma } from "../db";
|
|
import { runOneEval } from "./runOneEval";
|
|
import { type Scenario } from "~/components/OutputsTable/types";
|
|
|
|
const saveResult = async (evaluation: Evaluation, scenario: Scenario, modelOutput: ModelOutput) => {
|
|
const result = await runOneEval(evaluation, scenario, modelOutput);
|
|
return await prisma.outputEvaluation.upsert({
|
|
where: {
|
|
modelOutputId_evaluationId: {
|
|
modelOutputId: modelOutput.id,
|
|
evaluationId: evaluation.id,
|
|
},
|
|
},
|
|
create: {
|
|
modelOutputId: modelOutput.id,
|
|
evaluationId: evaluation.id,
|
|
...result,
|
|
},
|
|
update: {
|
|
...result,
|
|
},
|
|
});
|
|
};
|
|
|
|
export const runEvalsForOutput = async (
|
|
experimentId: string,
|
|
scenario: Scenario,
|
|
modelOutput: ModelOutput,
|
|
) => {
|
|
const evaluations = await prisma.evaluation.findMany({
|
|
where: { experimentId },
|
|
});
|
|
|
|
await Promise.all(
|
|
evaluations.map(async (evaluation) => await saveResult(evaluation, scenario, modelOutput)),
|
|
);
|
|
};
|
|
|
|
export const runAllEvals = async (experimentId: string) => {
|
|
const outputs = await prisma.modelOutput.findMany({
|
|
where: {
|
|
scenarioVariantCell: {
|
|
promptVariant: {
|
|
experimentId,
|
|
visible: true,
|
|
},
|
|
testScenario: {
|
|
visible: true,
|
|
},
|
|
},
|
|
},
|
|
include: {
|
|
scenarioVariantCell: {
|
|
include: {
|
|
testScenario: true,
|
|
},
|
|
},
|
|
outputEvaluations: true,
|
|
},
|
|
});
|
|
const evals = await prisma.evaluation.findMany({
|
|
where: { experimentId },
|
|
});
|
|
|
|
await Promise.all(
|
|
outputs.map(async (output) => {
|
|
const unrunEvals = evals.filter(
|
|
(evaluation) => !output.outputEvaluations.find((e) => e.evaluationId === evaluation.id),
|
|
);
|
|
|
|
await Promise.all(
|
|
unrunEvals.map(async (evaluation) => {
|
|
await saveResult(evaluation, output.scenarioVariantCell.testScenario, output);
|
|
}),
|
|
);
|
|
}),
|
|
);
|
|
};
|