diff --git a/src/components/OutputsTable/EditEvaluations.tsx b/src/components/OutputsTable/EditEvaluations.tsx
index 9a134ec..6f95f40 100644
--- a/src/components/OutputsTable/EditEvaluations.tsx
+++ b/src/components/OutputsTable/EditEvaluations.tsx
@@ -65,14 +65,14 @@ export function EvaluationEditor(props: {
Match String
-
- This string will be interpreted as a regex and checked against each model output.
-
setValues((values) => ({ ...values, matchString: e.target.value }))}
/>
+
+ This string will be interpreted as a regex and checked against each model output.
+
diff --git a/src/components/OutputsTable/OutputCell.tsx b/src/components/OutputsTable/OutputCell.tsx
index 15cc745..ffca43b 100644
--- a/src/components/OutputsTable/OutputCell.tsx
+++ b/src/components/OutputsTable/OutputCell.tsx
@@ -1,11 +1,11 @@
-import { api } from "~/utils/api";
+import { type RouterOutputs, api } from "~/utils/api";
import { type PromptVariant, type Scenario } from "./types";
import { Spinner, Text, Box, Center, Flex, Icon, HStack } from "@chakra-ui/react";
-import { useExperiment } from "~/utils/hooks";
+import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
import SyntaxHighlighter from "react-syntax-highlighter";
import { docco } from "react-syntax-highlighter/dist/cjs/styles/hljs";
import stringify from "json-stringify-pretty-compact";
-import { useMemo, type ReactElement } from "react";
+import { useMemo, type ReactElement, useState, useEffect } from "react";
import { BsCheck, BsClock, BsX, BsCurrencyDollar } from "react-icons/bs";
import { type ModelOutput } from "@prisma/client";
import { type ChatCompletion } from "openai/resources/chat";
@@ -24,6 +24,7 @@ export default function OutputCell({
scenario: Scenario;
variant: PromptVariant;
}): ReactElement | null {
+ const utils = api.useContext();
const experiment = useExperiment();
const vars = api.templateVars.list.useQuery({
experimentId: experiment.data?.id ?? "",
@@ -53,41 +54,47 @@ export default function OutputCell({
return generateChannel();
}, [shouldStream]);
- const output = api.outputs.get.useQuery(
- {
+ const outputMutation = api.outputs.get.useMutation();
+
+ const [output, setOutput] = useState(null);
+
+ const [fetchOutput, fetchingOutput] = useHandledAsyncCallback(async () => {
+ const output = await outputMutation.mutateAsync({
scenarioId: scenario.id,
variantId: variant.id,
channel,
- },
- { enabled: disabledReason === null }
- );
+ });
+ setOutput(output);
+ await utils.evaluations.results.invalidate();
+ }, [outputMutation, scenario.id, variant.id, channel]);
+
+ useEffect(fetchOutput, []);
// Disconnect from socket if we're not streaming anymore
- const streamedMessage = useSocket(output.isLoading ? channel : undefined);
+ const streamedMessage = useSocket(fetchingOutput ? channel : undefined);
const streamedContent = streamedMessage?.choices?.[0]?.message?.content;
if (!vars) return null;
if (disabledReason) return {disabledReason} ;
- if (output.isLoading && !streamedMessage)
+ if (fetchingOutput && !streamedMessage)
return (
);
- if (!output.data && !output.isLoading)
- return Error retrieving output ;
+ if (!output && !fetchingOutput) return Error retrieving output ;
- if (output.data && output.data.errorMessage) {
- return Error: {output.data.errorMessage} ;
+ if (output && output.errorMessage) {
+ return Error: {output.errorMessage} ;
}
- const response = output.data?.output as unknown as ChatCompletion;
+ const response = output?.output as unknown as ChatCompletion;
const message = response?.choices?.[0]?.message;
- if (output.data && message?.function_call) {
+ if (output && message?.function_call) {
const rawArgs = message.function_call.arguments ?? "null";
let parsedArgs: string;
try {
@@ -115,18 +122,17 @@ export default function OutputCell({
{ maxLength: 40 }
)}
-
+
);
}
- const contentToDisplay =
- message?.content ?? streamedContent ?? JSON.stringify(output.data?.output);
+ const contentToDisplay = message?.content ?? streamedContent ?? JSON.stringify(output?.output);
return (
{contentToDisplay}
- {output.data && }
+ {output && }
);
}
diff --git a/src/components/OutputsTable/ScenarioEditor.tsx b/src/components/OutputsTable/ScenarioEditor.tsx
index 0fd2c8b..9242a99 100644
--- a/src/components/OutputsTable/ScenarioEditor.tsx
+++ b/src/components/OutputsTable/ScenarioEditor.tsx
@@ -26,9 +26,9 @@ export default function ScenarioEditor({
const [values, setValues] = useState>(savedValues);
const experiment = useExperiment();
- const vars = api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" }).data;
+ const vars = api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
- const variableLabels = vars?.map((v) => v.label) ?? [];
+ const variableLabels = vars.data?.map((v) => v.label) ?? [];
const hasChanged = !isEqual(savedValues, values);
@@ -117,7 +117,7 @@ export default function ScenarioEditor({
/>
{variableLabels.length === 0 ? (
- No scenario variables configured
+ {vars.data ? "No scenario variables configured" : "Loading..."}
) : (
{variableLabels.map((key) => {
diff --git a/src/server/api/routers/modelOutputs.router.ts b/src/server/api/routers/modelOutputs.router.ts
index c2f1e57..a726712 100644
--- a/src/server/api/routers/modelOutputs.router.ts
+++ b/src/server/api/routers/modelOutputs.router.ts
@@ -13,7 +13,7 @@ export const modelOutputsRouter = createTRPCRouter({
.input(
z.object({ scenarioId: z.string(), variantId: z.string(), channel: z.string().optional() })
)
- .query(async ({ input }) => {
+ .mutation(async ({ input }) => {
const existing = await prisma.modelOutput.findUnique({
where: {
promptVariantId_testScenarioId: {
diff --git a/src/server/utils/evaluateOutput.ts b/src/server/utils/evaluateOutput.ts
index 404df09..0dd3266 100644
--- a/src/server/utils/evaluateOutput.ts
+++ b/src/server/utils/evaluateOutput.ts
@@ -8,7 +8,7 @@ export const evaluateOutput = (
evaluation: Evaluation
): boolean => {
const output = modelOutput.output as unknown as ChatCompletion;
- const message = output.choices?.[0]?.message;
+ const message = output?.choices?.[0]?.message;
if (!message) return false;