move app to app/ subdir

app/src/components/OutputsTable/AddVariantButton.tsx

@@ -0,0 +1,57 @@
+import { Box, Flex, Icon, Spinner } from "@chakra-ui/react";
+import { BsPlus } from "react-icons/bs";
+import { Text } from "@chakra-ui/react";
+import { api } from "~/utils/api";
+import {
+  useExperiment,
+  useExperimentAccess,
+  useHandledAsyncCallback,
+  useVisibleScenarioIds,
+} from "~/utils/hooks";
+import { cellPadding } from "../constants";
+import { ActionButton } from "./ScenariosHeader";
+
+export default function AddVariantButton() {
+  const experiment = useExperiment();
+  const mutation = api.promptVariants.create.useMutation();
+  const utils = api.useContext();
+  const visibleScenarios = useVisibleScenarioIds();
+
+  const [onClick, loading] = useHandledAsyncCallback(async () => {
+    if (!experiment.data) return;
+    await mutation.mutateAsync({
+      experimentId: experiment.data.id,
+      streamScenarios: visibleScenarios,
+    });
+    await utils.promptVariants.list.invalidate();
+  }, [mutation]);
+
+  const { canModify } = useExperimentAccess();
+  if (!canModify) return <Box w={cellPadding.x} />;
+
+  return (
+    <Flex w="100%" justifyContent="flex-end">
+      <ActionButton
+        onClick={onClick}
+        py={5}
+        leftIcon={<Icon as={loading ? Spinner : BsPlus} boxSize={6} mr={loading ? 1 : 0} />}
+      >
+        <Text display={{ base: "none", md: "flex" }}>Add Variant</Text>
+      </ActionButton>
+      {/* <Button
+        alignItems="center"
+        justifyContent="center"
+        fontWeight="normal"
+        bgColor="transparent"
+        _hover={{ bgColor: "gray.100" }}
+        px={cellPadding.x}
+        onClick={onClick}
+        height="unset"
+        minH={headerMinHeight}
+      >
+        <Icon as={loading ? Spinner : BsPlus} boxSize={6} mr={loading ? 1 : 0} />
+        <Text display={{ base: "none", md: "flex" }}>Add Variant</Text>
+      </Button> */}
+    </Flex>
+  );
+}

app/src/components/OutputsTable/EditEvaluations.tsx

@@ -0,0 +1,240 @@
+import {
+  Text,
+  Button,
+  HStack,
+  Heading,
+  Icon,
+  Input,
+  Stack,
+  VStack,
+  FormControl,
+  FormLabel,
+  Select,
+  FormHelperText,
+  Code,
+} from "@chakra-ui/react";
+import { type Evaluation, EvalType } from "@prisma/client";
+import { useCallback, useState } from "react";
+import { BsPencil, BsX } from "react-icons/bs";
+import { api } from "~/utils/api";
+import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+import AutoResizeTextArea from "../AutoResizeTextArea";
+
+type EvalValues = Pick<Evaluation, "label" | "value" | "evalType">;
+
+export function EvaluationEditor(props: {
+  evaluation: Evaluation | null;
+  defaultName?: string;
+  onSave: (id: string | undefined, vals: EvalValues) => void;
+  onCancel: () => void;
+}) {
+  const [values, setValues] = useState<EvalValues>({
+    label: props.evaluation?.label ?? props.defaultName ?? "",
+    value: props.evaluation?.value ?? "",
+    evalType: props.evaluation?.evalType ?? "CONTAINS",
+  });
+
+  return (
+    <VStack borderTopWidth={1} borderColor="gray.200" py={4}>
+      <HStack w="100%">
+        <FormControl flex={1}>
+          <FormLabel fontSize="sm">Eval Name</FormLabel>
+          <Input
+            size="sm"
+            value={values.label}
+            onChange={(e) => setValues((values) => ({ ...values, label: e.target.value }))}
+          />
+        </FormControl>
+        <FormControl flex={1}>
+          <FormLabel fontSize="sm">Eval Type</FormLabel>
+          <Select
+            size="sm"
+            value={values.evalType}
+            onChange={(e) =>
+              setValues((values) => ({
+                ...values,
+                evalType: e.target.value as EvalType,
+              }))
+            }
+          >
+            {Object.values(EvalType).map((type) => (
+              <option key={type} value={type}>
+                {type}
+              </option>
+            ))}
+          </Select>
+        </FormControl>
+      </HStack>
+      {["CONTAINS", "DOES_NOT_CONTAIN"].includes(values.evalType) && (
+        <FormControl>
+          <FormLabel fontSize="sm">Match String</FormLabel>
+          <Input
+            size="sm"
+            value={values.value}
+            onChange={(e) => setValues((values) => ({ ...values, value: e.target.value }))}
+          />
+          <FormHelperText>
+            This string will be interpreted as a regex and checked against each model output. You
+            can include scenario variables using <Code>{"{{curly_braces}}"}</Code>
+          </FormHelperText>
+        </FormControl>
+      )}
+      {values.evalType === "GPT4_EVAL" && (
+        <FormControl pt={2}>
+          <FormLabel fontSize="sm">GPT4 Instructions</FormLabel>
+          <AutoResizeTextArea
+            size="sm"
+            value={values.value}
+            onChange={(e) => setValues((values) => ({ ...values, value: e.target.value }))}
+            minRows={3}
+          />
+          <FormHelperText>
+            Give instructions to GPT-4 for how to evaluate your prompt. It will have access to the
+            full scenario as well as the output it is evaluating. It will <strong>not</strong> have
+            access to the specific prompt variant, so be sure to be clear about the task you want it
+            to perform.
+          </FormHelperText>
+        </FormControl>
+      )}
+      <HStack alignSelf="flex-end">
+        <Button size="sm" onClick={props.onCancel} colorScheme="gray">
+          Cancel
+        </Button>
+        <Button
+          size="sm"
+          onClick={() => props.onSave(props.evaluation?.id, values)}
+          colorScheme="blue"
+        >
+          Save
+        </Button>
+      </HStack>
+    </VStack>
+  );
+}
+
+export default function EditEvaluations() {
+  const experiment = useExperiment();
+  const evaluations =
+    api.evaluations.list.useQuery({ experimentId: experiment.data?.id ?? "" }).data ?? [];
+
+  const [editingId, setEditingId] = useState<string | null>(null);
+
+  const utils = api.useContext();
+  const createMutation = api.evaluations.create.useMutation();
+  const updateMutation = api.evaluations.update.useMutation();
+
+  const deleteMutation = api.evaluations.delete.useMutation();
+  const [onDelete] = useHandledAsyncCallback(async (id: string) => {
+    await deleteMutation.mutateAsync({ id });
+    await utils.evaluations.list.invalidate();
+    await utils.promptVariants.stats.invalidate();
+  }, []);
+
+  const [onSave] = useHandledAsyncCallback(async (id: string | undefined, vals: EvalValues) => {
+    setEditingId(null);
+    if (!experiment.data?.id) return;
+
+    if (id) {
+      await updateMutation.mutateAsync({
+        id,
+        updates: vals,
+      });
+    } else {
+      await createMutation.mutateAsync({
+        experimentId: experiment.data.id,
+        ...vals,
+      });
+    }
+    await utils.evaluations.list.invalidate();
+    await utils.promptVariants.stats.invalidate();
+    await utils.scenarioVariantCells.get.invalidate();
+  }, []);
+
+  const onCancel = useCallback(() => {
+    setEditingId(null);
+  }, []);
+
+  return (
+    <Stack>
+      <Heading size="sm">Evaluations</Heading>
+      <Stack spacing={4}>
+        <Text fontSize="sm">
+          Evaluations allow you to compare prompt performance in an automated way.
+        </Text>
+        <Stack spacing={2}>
+          {evaluations.map((evaluation) =>
+            editingId == evaluation.id ? (
+              <EvaluationEditor
+                evaluation={evaluation}
+                onSave={onSave}
+                onCancel={onCancel}
+                key={evaluation.id}
+              />
+            ) : (
+              <HStack
+                fontSize="sm"
+                borderTopWidth={1}
+                borderColor="gray.200"
+                py={4}
+                align="center"
+                key={evaluation.id}
+              >
+                <Text fontWeight="bold">{evaluation.label}</Text>
+                <Text flex={1}>
+                  {evaluation.evalType}: &quot;{evaluation.value}&quot;
+                </Text>
+                <Button
+                  variant="unstyled"
+                  color="gray.400"
+                  height="unset"
+                  width="unset"
+                  minW="unset"
+                  onClick={() => setEditingId(evaluation.id)}
+                  _hover={{
+                    color: "gray.800",
+                    cursor: "pointer",
+                  }}
+                >
+                  <Icon as={BsPencil} boxSize={4} />
+                </Button>
+                <Button
+                  variant="unstyled"
+                  color="gray.400"
+                  height="unset"
+                  width="unset"
+                  minW="unset"
+                  onClick={() => onDelete(evaluation.id)}
+                  _hover={{
+                    color: "gray.800",
+                    cursor: "pointer",
+                  }}
+                >
+                  <Icon as={BsX} boxSize={6} />
+                </Button>
+              </HStack>
+            ),
+          )}
+          {editingId == null && (
+            <Button
+              onClick={() => setEditingId("new")}
+              alignSelf="flex-start"
+              size="sm"
+              mt={4}
+              colorScheme="blue"
+            >
+              Add Evaluation
+            </Button>
+          )}
+          {editingId == "new" && (
+            <EvaluationEditor
+              evaluation={null}
+              defaultName={`Eval${evaluations.length + 1}`}
+              onSave={onSave}
+              onCancel={onCancel}
+            />
+          )}
+        </Stack>
+      </Stack>
+    </Stack>
+  );
+}

app/src/components/OutputsTable/EditScenarioVars.tsx

@@ -0,0 +1,103 @@
+import { Text, Button, HStack, Heading, Icon, Input, Stack } from "@chakra-ui/react";
+import { useState } from "react";
+import { BsCheck, BsX } from "react-icons/bs";
+import { api } from "~/utils/api";
+import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+
+export default function EditScenarioVars() {
+  const experiment = useExperiment();
+  const vars =
+    api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" }).data ?? [];
+
+  const [newVariable, setNewVariable] = useState<string>("");
+  const newVarIsValid = newVariable.length > 0 && !vars.map((v) => v.label).includes(newVariable);
+
+  const utils = api.useContext();
+  const addVarMutation = api.templateVars.create.useMutation();
+  const [onAddVar] = useHandledAsyncCallback(async () => {
+    if (!experiment.data?.id) return;
+    if (!newVarIsValid) return;
+    await addVarMutation.mutateAsync({
+      experimentId: experiment.data.id,
+      label: newVariable,
+    });
+    await utils.templateVars.list.invalidate();
+    setNewVariable("");
+  }, [addVarMutation, experiment.data?.id, newVarIsValid, newVariable]);
+
+  const deleteMutation = api.templateVars.delete.useMutation();
+  const [onDeleteVar] = useHandledAsyncCallback(async (id: string) => {
+    await deleteMutation.mutateAsync({ id });
+    await utils.templateVars.list.invalidate();
+  }, []);
+
+  return (
+    <Stack>
+      <Heading size="sm">Scenario Variables</Heading>
+      <Stack spacing={2}>
+        <Text fontSize="sm">
+          Scenario variables can be used in your prompt variants as well as evaluations.
+        </Text>
+        <HStack spacing={0}>
+          <Input
+            placeholder="Add Scenario Variable"
+            size="sm"
+            borderTopRadius={0}
+            borderRightRadius={0}
+            value={newVariable}
+            onChange={(e) => setNewVariable(e.target.value)}
+            onKeyDown={(e) => {
+              if (e.key === "Enter") {
+                e.preventDefault();
+                onAddVar();
+              }
+              // If the user types a space, replace it with an underscore
+              if (e.key === " ") {
+                e.preventDefault();
+                setNewVariable((v) => v + "_");
+              }
+            }}
+          />
+          <Button
+            size="xs"
+            height="100%"
+            borderLeftRadius={0}
+            isDisabled={!newVarIsValid}
+            onClick={onAddVar}
+          >
+            <Icon as={BsCheck} boxSize={8} />
+          </Button>
+        </HStack>
+
+        <HStack spacing={2} py={4} wrap="wrap">
+          {vars.map((variable) => (
+            <HStack
+              key={variable.id}
+              spacing={0}
+              bgColor="blue.100"
+              color="blue.600"
+              pl={2}
+              pr={0}
+              fontWeight="bold"
+            >
+              <Text fontSize="sm" flex={1}>
+                {variable.label}
+              </Text>
+              <Button
+                size="xs"
+                variant="ghost"
+                colorScheme="blue"
+                p="unset"
+                minW="unset"
+                px="unset"
+                onClick={() => onDeleteVar(variable.id)}
+              >
+                <Icon as={BsX} boxSize={6} color="blue.800" />
+              </Button>
+            </HStack>
+          ))}
+        </HStack>
+      </Stack>
+    </Stack>
+  );
+}

app/src/components/OutputsTable/FloatingLabelInput.tsx

@@ -0,0 +1,46 @@
+import { FormLabel, FormControl, type TextareaProps } from "@chakra-ui/react";
+import { useState } from "react";
+import AutoResizeTextArea from "../AutoResizeTextArea";
+
+export const FloatingLabelInput = ({
+  label,
+  value,
+  ...props
+}: { label: string; value: string } & TextareaProps) => {
+  const [isFocused, setIsFocused] = useState(false);
+
+  return (
+    <FormControl position="relative">
+      <FormLabel
+        position="absolute"
+        left="10px"
+        top={isFocused || !!value ? 0 : 3}
+        transform={isFocused || !!value ? "translateY(-50%)" : "translateY(0)"}
+        fontSize={isFocused || !!value ? "12px" : "16px"}
+        transition="all 0.15s"
+        zIndex="5"
+        bg="white"
+        px={1}
+        lineHeight="1"
+        pointerEvents="none"
+        color={isFocused ? "blue.500" : "gray.500"}
+      >
+        {label}
+      </FormLabel>
+      <AutoResizeTextArea
+        px={3}
+        pt={3}
+        pb={2}
+        onFocus={() => setIsFocused(true)}
+        onBlur={() => setIsFocused(false)}
+        borderRadius="md"
+        borderColor={isFocused ? "blue.500" : "gray.400"}
+        autoComplete="off"
+        value={value}
+        overflowY="auto"
+        overflowX="hidden"
+        {...props}
+      />
+    </FormControl>
+  );
+};

app/src/components/OutputsTable/OutputCell/OutputCell.tsx

@@ -0,0 +1,197 @@
+import { api } from "~/utils/api";
+import { type PromptVariant, type Scenario } from "../types";
+import { type StackProps, Text, VStack } from "@chakra-ui/react";
+import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+import SyntaxHighlighter from "react-syntax-highlighter";
+import { docco } from "react-syntax-highlighter/dist/cjs/styles/hljs";
+import stringify from "json-stringify-pretty-compact";
+import { type ReactElement, useState, useEffect, Fragment, useCallback } from "react";
+import useSocket from "~/utils/useSocket";
+import { OutputStats } from "./OutputStats";
+import { RetryCountdown } from "./RetryCountdown";
+import frontendModelProviders from "~/modelProviders/frontendModelProviders";
+import { ResponseLog } from "./ResponseLog";
+import { CellOptions } from "./TopActions";
+
+const WAITING_MESSAGE_INTERVAL = 20000;
+
+export default function OutputCell({
+  scenario,
+  variant,
+}: {
+  scenario: Scenario;
+  variant: PromptVariant;
+}): ReactElement | null {
+  const utils = api.useContext();
+  const experiment = useExperiment();
+  const vars = api.templateVars.list.useQuery({
+    experimentId: experiment.data?.id ?? "",
+  }).data;
+
+  const scenarioVariables = scenario.variableValues as Record<string, string>;
+  const templateHasVariables =
+    vars?.length === 0 || vars?.some((v) => scenarioVariables[v.label] !== undefined);
+
+  let disabledReason: string | null = null;
+
+  if (!templateHasVariables) disabledReason = "Add a value to the scenario variables to see output";
+
+  const [refetchInterval, setRefetchInterval] = useState(0);
+  const { data: cell, isLoading: queryLoading } = api.scenarioVariantCells.get.useQuery(
+    { scenarioId: scenario.id, variantId: variant.id },
+    { refetchInterval },
+  );
+
+  const provider =
+    frontendModelProviders[variant.modelProvider as keyof typeof frontendModelProviders];
+
+  type OutputSchema = Parameters<typeof provider.normalizeOutput>[0];
+
+  const { mutateAsync: hardRefetchMutate } = api.scenarioVariantCells.forceRefetch.useMutation();
+  const [hardRefetch, hardRefetching] = useHandledAsyncCallback(async () => {
+    await hardRefetchMutate({ scenarioId: scenario.id, variantId: variant.id });
+    await utils.scenarioVariantCells.get.invalidate({
+      scenarioId: scenario.id,
+      variantId: variant.id,
+    });
+    await utils.promptVariants.stats.invalidate({
+      variantId: variant.id,
+    });
+  }, [hardRefetchMutate, scenario.id, variant.id]);
+
+  const fetchingOutput = queryLoading || hardRefetching;
+
+  const awaitingOutput =
+    !cell ||
+    !cell.evalsComplete ||
+    cell.retrievalStatus === "PENDING" ||
+    cell.retrievalStatus === "IN_PROGRESS" ||
+    hardRefetching;
+  useEffect(() => setRefetchInterval(awaitingOutput ? 1000 : 0), [awaitingOutput]);
+
+  // TODO: disconnect from socket if we're not streaming anymore
+  const streamedMessage = useSocket<OutputSchema>(cell?.id);
+
+  const mostRecentResponse = cell?.modelResponses[cell.modelResponses.length - 1];
+
+  const CellWrapper = useCallback(
+    ({ children, ...props }: StackProps) => (
+      <VStack w="full" alignItems="flex-start" {...props} px={2} py={2} h="100%">
+        {cell && (
+          <CellOptions refetchingOutput={hardRefetching} refetchOutput={hardRefetch} cell={cell} />
+        )}
+        <VStack w="full" alignItems="flex-start" maxH={500} overflowY="auto" flex={1}>
+          {children}
+        </VStack>
+        {mostRecentResponse && (
+          <OutputStats modelResponse={mostRecentResponse} scenario={scenario} />
+        )}
+      </VStack>
+    ),
+    [hardRefetching, hardRefetch, mostRecentResponse, scenario, cell],
+  );
+
+  if (!vars) return null;
+
+  if (!cell && !fetchingOutput)
+    return (
+      <CellWrapper>
+        <Text color="gray.500">Error retrieving output</Text>
+      </CellWrapper>
+    );
+
+  if (cell && cell.errorMessage) {
+    return (
+      <CellWrapper>
+        <Text color="red.500">{cell.errorMessage}</Text>
+      </CellWrapper>
+    );
+  }
+
+  if (disabledReason) return <Text color="gray.500">{disabledReason}</Text>;
+
+  const showLogs = !streamedMessage && !mostRecentResponse?.output;
+
+  if (showLogs)
+    return (
+      <CellWrapper alignItems="flex-start" fontFamily="inconsolata, monospace" spacing={0}>
+        {cell?.jobQueuedAt && <ResponseLog time={cell.jobQueuedAt} title="Job queued" />}
+        {cell?.jobStartedAt && <ResponseLog time={cell.jobStartedAt} title="Job started" />}
+        {cell?.modelResponses?.map((response) => {
+          let numWaitingMessages = 0;
+          const relativeWaitingTime = response.receivedAt
+            ? response.receivedAt.getTime()
+            : Date.now();
+          if (response.requestedAt) {
+            numWaitingMessages = Math.floor(
+              (relativeWaitingTime - response.requestedAt.getTime()) / WAITING_MESSAGE_INTERVAL,
+            );
+          }
+          return (
+            <Fragment key={response.id}>
+              {response.requestedAt && (
+                <ResponseLog time={response.requestedAt} title="Request sent to API" />
+              )}
+              {response.requestedAt &&
+                Array.from({ length: numWaitingMessages }, (_, i) => (
+                  <ResponseLog
+                    key={`waiting-${i}`}
+                    time={
+                      new Date(
+                        (response.requestedAt?.getTime?.() ?? 0) +
+                          (i + 1) * WAITING_MESSAGE_INTERVAL,
+                      )
+                    }
+                    title="Waiting for response..."
+                  />
+                ))}
+              {response.receivedAt && (
+                <ResponseLog
+                  time={response.receivedAt}
+                  title="Response received from API"
+                  message={`statusCode: ${response.statusCode ?? ""}\n ${
+                    response.errorMessage ?? ""
+                  }`}
+                />
+              )}
+            </Fragment>
+          );
+        }) ?? null}
+        {mostRecentResponse?.retryTime && (
+          <RetryCountdown retryTime={mostRecentResponse.retryTime} />
+        )}
+      </CellWrapper>
+    );
+
+  const normalizedOutput = mostRecentResponse?.output
+    ? provider.normalizeOutput(mostRecentResponse?.output)
+    : streamedMessage
+    ? provider.normalizeOutput(streamedMessage)
+    : null;
+
+  if (mostRecentResponse?.output && normalizedOutput?.type === "json") {
+    return (
+      <CellWrapper>
+        <SyntaxHighlighter
+          customStyle={{ overflowX: "unset", width: "100%", flex: 1 }}
+          language="json"
+          style={docco}
+          lineProps={{
+            style: { wordBreak: "break-all", whiteSpace: "pre-wrap" },
+          }}
+          wrapLines
+        >
+          {stringify(normalizedOutput.value, { maxLength: 40 })}
+        </SyntaxHighlighter>
+      </CellWrapper>
+    );
+  }
+
+  const contentToDisplay = (normalizedOutput?.type === "text" && normalizedOutput.value) || "";
+
+  return (
+    <CellWrapper>
+      <Text>{contentToDisplay}</Text>
+    </CellWrapper>
+  );
+}

app/src/components/OutputsTable/OutputCell/OutputStats.tsx

@@ -0,0 +1,76 @@
+import { type Scenario } from "../types";
+import { type RouterOutputs } from "~/utils/api";
+import { HStack, Icon, Text, Tooltip } from "@chakra-ui/react";
+import { BsCheck, BsClock, BsCurrencyDollar, BsX } from "react-icons/bs";
+import { CostTooltip } from "~/components/tooltip/CostTooltip";
+
+const SHOW_TIME = true;
+
+export const OutputStats = ({
+  modelResponse,
+}: {
+  modelResponse: NonNullable<
+    NonNullable<RouterOutputs["scenarioVariantCells"]["get"]>["modelResponses"][0]
+  >;
+  scenario: Scenario;
+}) => {
+  const timeToComplete =
+    modelResponse.receivedAt && modelResponse.requestedAt
+      ? modelResponse.receivedAt.getTime() - modelResponse.requestedAt.getTime()
+      : 0;
+
+  const promptTokens = modelResponse.promptTokens;
+  const completionTokens = modelResponse.completionTokens;
+
+  return (
+    <HStack
+      w="full"
+      align="center"
+      color="gray.500"
+      fontSize="2xs"
+      mt={{ base: 0, md: 1 }}
+      alignItems="flex-end"
+    >
+      <HStack flex={1} flexWrap="wrap">
+        {modelResponse.outputEvaluations.map((evaluation) => {
+          const passed = evaluation.result > 0.5;
+          return (
+            <Tooltip
+              isDisabled={!evaluation.details}
+              label={evaluation.details}
+              key={evaluation.id}
+              shouldWrapChildren
+            >
+              <HStack spacing={0}>
+                <Text>{evaluation.evaluation.label}</Text>
+                <Icon
+                  as={passed ? BsCheck : BsX}
+                  color={passed ? "green.500" : "red.500"}
+                  boxSize={6}
+                />
+              </HStack>
+            </Tooltip>
+          );
+        })}
+      </HStack>
+      {modelResponse.cost && (
+        <CostTooltip
+          promptTokens={promptTokens}
+          completionTokens={completionTokens}
+          cost={modelResponse.cost}
+        >
+          <HStack spacing={0}>
+            <Icon as={BsCurrencyDollar} />
+            <Text mr={1}>{modelResponse.cost.toFixed(3)}</Text>
+          </HStack>
+        </CostTooltip>
+      )}
+      {SHOW_TIME && (
+        <HStack spacing={0.5}>
+          <Icon as={BsClock} />
+          <Text>{(timeToComplete / 1000).toFixed(2)}s</Text>
+        </HStack>
+      )}
+    </HStack>
+  );
+};

app/src/components/OutputsTable/OutputCell/PromptModal.tsx

@@ -0,0 +1,36 @@
+import {
+  Modal,
+  ModalBody,
+  ModalCloseButton,
+  ModalContent,
+  ModalHeader,
+  ModalOverlay,
+  type UseDisclosureReturn,
+} from "@chakra-ui/react";
+import { type RouterOutputs } from "~/utils/api";
+import { JSONTree } from "react-json-tree";
+
+export default function ExpandedModal(props: {
+  cell: NonNullable<RouterOutputs["scenarioVariantCells"]["get"]>;
+  disclosure: UseDisclosureReturn;
+}) {
+  return (
+    <Modal isOpen={props.disclosure.isOpen} onClose={props.disclosure.onClose} size="2xl">
+      <ModalOverlay />
+      <ModalContent>
+        <ModalHeader>Prompt</ModalHeader>
+        <ModalCloseButton />
+        <ModalBody>
+          <JSONTree
+            data={props.cell.prompt}
+            invertTheme={true}
+            theme="chalk"
+            shouldExpandNodeInitially={() => true}
+            getItemString={() => ""}
+            hideRoot
+          />
+        </ModalBody>
+      </ModalContent>
+    </Modal>
+  );
+}

app/src/components/OutputsTable/OutputCell/ResponseLog.tsx

@@ -0,0 +1,22 @@
+import { HStack, VStack, Text } from "@chakra-ui/react";
+import dayjs from "dayjs";
+
+export const ResponseLog = ({
+  time,
+  title,
+  message,
+}: {
+  time: Date;
+  title: string;
+  message?: string;
+}) => {
+  return (
+    <VStack spacing={0} alignItems="flex-start">
+      <HStack>
+        <Text>{dayjs(time).format("HH:mm:ss")}</Text>
+        <Text>{title}</Text>
+      </HStack>
+      {message && <Text pl={4}>{message}</Text>}
+    </VStack>
+  );
+};

app/src/components/OutputsTable/OutputCell/RetryCountdown.tsx

@@ -0,0 +1,39 @@
+import { Text } from "@chakra-ui/react";
+import { useEffect, useState } from "react";
+import pluralize from "pluralize";
+
+export const RetryCountdown = ({ retryTime }: { retryTime: Date }) => {
+  const [msToWait, setMsToWait] = useState(0);
+
+  useEffect(() => {
+    const initialWaitTime = retryTime.getTime() - Date.now();
+    const msModuloOneSecond = initialWaitTime % 1000;
+    let remainingTime = initialWaitTime - msModuloOneSecond;
+    setMsToWait(remainingTime);
+
+    let interval: NodeJS.Timeout;
+    const timeout = setTimeout(() => {
+      interval = setInterval(() => {
+        remainingTime -= 1000;
+        setMsToWait(remainingTime);
+
+        if (remainingTime <= 0) {
+          clearInterval(interval);
+        }
+      }, 1000);
+    }, msModuloOneSecond);
+
+    return () => {
+      clearInterval(interval);
+      clearTimeout(timeout);
+    };
+  }, [retryTime]);
+
+  if (msToWait <= 0) return null;
+
+  return (
+    <Text color="red.600" fontSize="sm">
+      Retrying in {pluralize("second", Math.ceil(msToWait / 1000), true)}...
+    </Text>
+  );
+};

app/src/components/OutputsTable/OutputCell/TopActions.tsx

@@ -0,0 +1,53 @@
+import { HStack, Icon, IconButton, Spinner, Tooltip, useDisclosure } from "@chakra-ui/react";
+import { BsArrowClockwise, BsInfoCircle } from "react-icons/bs";
+import { useExperimentAccess } from "~/utils/hooks";
+import ExpandedModal from "./PromptModal";
+import { type RouterOutputs } from "~/utils/api";
+
+export const CellOptions = ({
+  cell,
+  refetchingOutput,
+  refetchOutput,
+}: {
+  cell: RouterOutputs["scenarioVariantCells"]["get"];
+  refetchingOutput: boolean;
+  refetchOutput: () => void;
+}) => {
+  const { canModify } = useExperimentAccess();
+
+  const modalDisclosure = useDisclosure();
+
+  return (
+    <HStack justifyContent="flex-end" w="full">
+      {cell && (
+        <>
+          <Tooltip label="See Prompt">
+            <IconButton
+              aria-label="See Prompt"
+              icon={<Icon as={BsInfoCircle} boxSize={4} />}
+              onClick={modalDisclosure.onOpen}
+              size="xs"
+              colorScheme="gray"
+              color="gray.500"
+              variant="ghost"
+            />
+          </Tooltip>
+          <ExpandedModal cell={cell} disclosure={modalDisclosure} />
+        </>
+      )}
+      {canModify && (
+        <Tooltip label="Refetch output">
+          <IconButton
+            size="xs"
+            color="gray.500"
+            variant="ghost"
+            cursor="pointer"
+            onClick={refetchOutput}
+            aria-label="refetch output"
+            icon={<Icon as={refetchingOutput ? Spinner : BsArrowClockwise} boxSize={4} />}
+          />
+        </Tooltip>
+      )}
+    </HStack>
+  );
+};

app/src/components/OutputsTable/ScenarioEditor.tsx

@@ -0,0 +1,207 @@
+import { isEqual } from "lodash-es";
+import { useEffect, useState, type DragEvent } from "react";
+import { api } from "~/utils/api";
+import { useExperiment, useExperimentAccess, useHandledAsyncCallback } from "~/utils/hooks";
+import { type Scenario } from "./types";
+
+import {
+  Box,
+  Button,
+  HStack,
+  Icon,
+  IconButton,
+  Spinner,
+  Text,
+  Tooltip,
+  VStack,
+} from "@chakra-ui/react";
+import { BsArrowsAngleExpand, BsX } from "react-icons/bs";
+import { cellPadding } from "../constants";
+import { FloatingLabelInput } from "./FloatingLabelInput";
+import { ScenarioEditorModal } from "./ScenarioEditorModal";
+
+export default function ScenarioEditor({
+  scenario,
+  ...props
+}: {
+  scenario: Scenario;
+  hovered: boolean;
+  canHide: boolean;
+}) {
+  const { canModify } = useExperimentAccess();
+
+  const savedValues = scenario.variableValues as Record<string, string>;
+  const utils = api.useContext();
+  const [isDragTarget, setIsDragTarget] = useState(false);
+  const [variableInputHovered, setVariableInputHovered] = useState(false);
+
+  const [values, setValues] = useState<Record<string, string>>(savedValues);
+
+  useEffect(() => {
+    if (savedValues) setValues(savedValues);
+  }, [savedValues]);
+
+  const experiment = useExperiment();
+  const vars = api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
+
+  const variableLabels = vars.data?.map((v) => v.label) ?? [];
+
+  const hasChanged = !isEqual(savedValues, values);
+
+  const mutation = api.scenarios.replaceWithValues.useMutation();
+
+  const [onSave] = useHandledAsyncCallback(async () => {
+    await mutation.mutateAsync({
+      id: scenario.id,
+      values,
+    });
+    await utils.scenarios.list.invalidate();
+  }, [mutation, values]);
+
+  const hideMutation = api.scenarios.hide.useMutation();
+  const [onHide, hidingInProgress] = useHandledAsyncCallback(async () => {
+    await hideMutation.mutateAsync({
+      id: scenario.id,
+    });
+    await utils.scenarios.list.invalidate();
+    await utils.promptVariants.stats.invalidate();
+  }, [hideMutation, scenario.id]);
+
+  const reorderMutation = api.scenarios.reorder.useMutation();
+  const [onReorder] = useHandledAsyncCallback(
+    async (e: DragEvent<HTMLDivElement>) => {
+      e.preventDefault();
+      setIsDragTarget(false);
+      const draggedId = e.dataTransfer.getData("text/plain");
+      const droppedId = scenario.id;
+      if (!draggedId || !droppedId || draggedId === droppedId) return;
+      await reorderMutation.mutateAsync({
+        draggedId,
+        droppedId,
+      });
+      await utils.scenarios.list.invalidate();
+    },
+    [reorderMutation, scenario.id],
+  );
+
+  const [scenarioEditorModalOpen, setScenarioEditorModalOpen] = useState(false);
+
+  return (
+    <>
+      <HStack
+        alignItems="flex-start"
+        px={cellPadding.x}
+        py={cellPadding.y}
+        spacing={0}
+        height="100%"
+        draggable={!variableInputHovered}
+        onDragStart={(e) => {
+          e.dataTransfer.setData("text/plain", scenario.id);
+          e.currentTarget.style.opacity = "0.4";
+        }}
+        onDragEnd={(e) => {
+          e.currentTarget.style.opacity = "1";
+        }}
+        onDragOver={(e) => {
+          e.preventDefault();
+          setIsDragTarget(true);
+        }}
+        onDragLeave={() => {
+          setIsDragTarget(false);
+        }}
+        onDrop={onReorder}
+        backgroundColor={isDragTarget ? "gray.100" : "transparent"}
+      >
+        {variableLabels.length === 0 ? (
+          <Box color="gray.500">
+            {vars.data ? "No scenario variables configured" : "Loading..."}
+          </Box>
+        ) : (
+          <VStack spacing={4} flex={1} py={2}>
+            <HStack justifyContent="space-between" w="100%" align="center" spacing={0}>
+              <Text flex={1}>Scenario</Text>
+              <Tooltip label="Expand" hasArrow>
+                <IconButton
+                  aria-label="Expand"
+                  icon={<Icon as={BsArrowsAngleExpand} boxSize={3} />}
+                  onClick={() => setScenarioEditorModalOpen(true)}
+                  size="xs"
+                  colorScheme="gray"
+                  color="gray.500"
+                  variant="ghost"
+                />
+              </Tooltip>
+              {canModify && props.canHide && (
+                <Tooltip label="Delete" hasArrow>
+                  <IconButton
+                    aria-label="Delete"
+                    icon={
+                      <Icon
+                        as={hidingInProgress ? Spinner : BsX}
+                        boxSize={hidingInProgress ? 4 : 6}
+                      />
+                    }
+                    onClick={onHide}
+                    size="xs"
+                    display="flex"
+                    colorScheme="gray"
+                    color="gray.500"
+                    variant="ghost"
+                  />
+                </Tooltip>
+              )}
+            </HStack>
+            {variableLabels.map((key) => {
+              const value = values[key] ?? "";
+              return (
+                <FloatingLabelInput
+                  key={key}
+                  label={key}
+                  isDisabled={!canModify}
+                  style={{ width: "100%" }}
+                  maxHeight={32}
+                  value={value}
+                  onChange={(e) => {
+                    setValues((prev) => ({ ...prev, [key]: e.target.value }));
+                  }}
+                  onKeyDown={(e) => {
+                    if (e.key === "Enter" && (e.metaKey || e.ctrlKey)) {
+                      e.preventDefault();
+                      e.currentTarget.blur();
+                      onSave();
+                    }
+                  }}
+                  onMouseEnter={() => setVariableInputHovered(true)}
+                  onMouseLeave={() => setVariableInputHovered(false)}
+                />
+              );
+            })}
+            {hasChanged && (
+              <HStack justify="right">
+                <Button
+                  size="sm"
+                  onMouseDown={() => {
+                    setValues(savedValues);
+                  }}
+                  colorScheme="gray"
+                >
+                  Reset
+                </Button>
+                <Button size="sm" onMouseDown={onSave} colorScheme="blue">
+                  Save
+                </Button>
+              </HStack>
+            )}
+          </VStack>
+        )}
+      </HStack>
+      {scenarioEditorModalOpen && (
+        <ScenarioEditorModal
+          scenarioId={scenario.id}
+          initialValues={savedValues}
+          onClose={() => setScenarioEditorModalOpen(false)}
+        />
+      )}
+    </>
+  );
+}

app/src/components/OutputsTable/ScenarioEditorModal.tsx

@@ -0,0 +1,123 @@
+import {
+  Button,
+  HStack,
+  Modal,
+  ModalBody,
+  ModalCloseButton,
+  ModalContent,
+  ModalFooter,
+  ModalHeader,
+  ModalOverlay,
+  Spinner,
+  Text,
+  VStack,
+} from "@chakra-ui/react";
+import { useEffect, useState } from "react";
+import { isEqual } from "lodash-es";
+
+import { api } from "~/utils/api";
+import {
+  useScenario,
+  useHandledAsyncCallback,
+  useExperiment,
+  useExperimentAccess,
+} from "~/utils/hooks";
+import { FloatingLabelInput } from "./FloatingLabelInput";
+
+export const ScenarioEditorModal = ({
+  scenarioId,
+  initialValues,
+  onClose,
+}: {
+  scenarioId: string;
+  initialValues: Record<string, string>;
+  onClose: () => void;
+}) => {
+  const utils = api.useContext();
+  const experiment = useExperiment();
+  const { canModify } = useExperimentAccess();
+  const scenario = useScenario(scenarioId);
+
+  const savedValues = scenario.data?.variableValues as Record<string, string>;
+
+  const [values, setValues] = useState<Record<string, string>>(initialValues);
+
+  useEffect(() => {
+    if (savedValues) setValues(savedValues);
+  }, [savedValues]);
+
+  const hasChanged = !isEqual(savedValues, values);
+
+  const mutation = api.scenarios.replaceWithValues.useMutation();
+
+  const [onSave, saving] = useHandledAsyncCallback(async () => {
+    await mutation.mutateAsync({
+      id: scenarioId,
+      values,
+    });
+    await utils.scenarios.list.invalidate();
+  }, [mutation, values]);
+
+  const vars = api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" });
+  const variableLabels = vars.data?.map((v) => v.label) ?? [];
+
+  return (
+    <Modal
+      isOpen
+      onClose={onClose}
+      size={{ base: "xl", sm: "2xl", md: "3xl", lg: "5xl", xl: "7xl" }}
+    >
+      <ModalOverlay />
+      <ModalContent w={1200}>
+        <ModalHeader />
+        <ModalCloseButton />
+        <ModalBody maxW="unset">
+          <VStack spacing={8}>
+            {values &&
+              variableLabels.map((key) => {
+                const value = values[key] ?? "";
+                return (
+                  <FloatingLabelInput
+                    key={key}
+                    label={key}
+                    isDisabled={!canModify}
+                    _disabled={{ opacity: 1 }}
+                    style={{ width: "100%" }}
+                    value={value}
+                    onChange={(e) => {
+                      setValues((prev) => ({ ...prev, [key]: e.target.value }));
+                    }}
+                    onKeyDown={(e) => {
+                      if (e.key === "Enter" && (e.metaKey || e.ctrlKey)) {
+                        e.preventDefault();
+                        e.currentTarget.blur();
+                        onSave();
+                      }
+                    }}
+                  />
+                );
+              })}
+          </VStack>
+        </ModalBody>
+
+        <ModalFooter>
+          {canModify && (
+            <HStack>
+              <Button
+                colorScheme="gray"
+                onClick={() => setValues(savedValues)}
+                minW={24}
+                isDisabled={!hasChanged}
+              >
+                <Text>Reset</Text>
+              </Button>
+              <Button colorScheme="blue" onClick={onSave} minW={24} isDisabled={!hasChanged}>
+                {saving ? <Spinner boxSize={4} /> : <Text>Save</Text>}
+              </Button>
+            </HStack>
+          )}
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};

app/src/components/OutputsTable/ScenarioPaginator.tsx

@@ -0,0 +1,21 @@
+import { useScenarios } from "~/utils/hooks";
+import Paginator from "../Paginator";
+
+const ScenarioPaginator = () => {
+  const { data } = useScenarios();
+
+  if (!data) return null;
+
+  const { scenarios, startIndex, lastPage, count } = data;
+
+  return (
+    <Paginator
+      numItemsLoaded={scenarios.length}
+      startIndex={startIndex}
+      lastPage={lastPage}
+      count={count}
+    />
+  );
+};
+
+export default ScenarioPaginator;

app/src/components/OutputsTable/ScenarioRow.tsx

@@ -0,0 +1,48 @@
+import { GridItem } from "@chakra-ui/react";
+import React, { useState } from "react";
+import OutputCell from "./OutputCell/OutputCell";
+import ScenarioEditor from "./ScenarioEditor";
+import type { PromptVariant, Scenario } from "./types";
+import { borders } from "./styles";
+
+const ScenarioRow = (props: {
+  scenario: Scenario;
+  variants: PromptVariant[];
+  canHide: boolean;
+  rowStart: number;
+}) => {
+  const [isHovered, setIsHovered] = useState(false);
+
+  const highlightStyle = { backgroundColor: "gray.50" };
+
+  return (
+    <>
+      <GridItem
+        onMouseEnter={() => setIsHovered(true)}
+        onMouseLeave={() => setIsHovered(false)}
+        sx={isHovered ? highlightStyle : undefined}
+        borderLeftWidth={1}
+        {...borders}
+        rowStart={props.rowStart}
+        colStart={1}
+      >
+        <ScenarioEditor scenario={props.scenario} hovered={isHovered} canHide={props.canHide} />
+      </GridItem>
+      {props.variants.map((variant, i) => (
+        <GridItem
+          key={variant.id}
+          onMouseEnter={() => setIsHovered(true)}
+          onMouseLeave={() => setIsHovered(false)}
+          sx={isHovered ? highlightStyle : undefined}
+          rowStart={props.rowStart}
+          colStart={i + 2}
+          {...borders}
+        >
+          <OutputCell key={variant.id} scenario={props.scenario} variant={variant} />
+        </GridItem>
+      ))}
+    </>
+  );
+};
+
+export default ScenarioRow;

app/src/components/OutputsTable/ScenariosHeader.tsx

@@ -0,0 +1,82 @@
+import {
+  Button,
+  type ButtonProps,
+  HStack,
+  Text,
+  Icon,
+  Menu,
+  MenuButton,
+  MenuList,
+  MenuItem,
+  IconButton,
+  Spinner,
+} from "@chakra-ui/react";
+import { cellPadding } from "../constants";
+import {
+  useExperiment,
+  useExperimentAccess,
+  useHandledAsyncCallback,
+  useScenarios,
+} from "~/utils/hooks";
+import { BsGear, BsPencil, BsPlus, BsStars } from "react-icons/bs";
+import { useAppStore } from "~/state/store";
+import { api } from "~/utils/api";
+
+export const ActionButton = (props: ButtonProps) => (
+  <Button size="sm" variant="ghost" color="gray.600" {...props} />
+);
+
+export const ScenariosHeader = () => {
+  const openDrawer = useAppStore((s) => s.openDrawer);
+  const { canModify } = useExperimentAccess();
+  const scenarios = useScenarios();
+
+  const experiment = useExperiment();
+  const createScenarioMutation = api.scenarios.create.useMutation();
+  const utils = api.useContext();
+
+  const [onAddScenario, loading] = useHandledAsyncCallback(
+    async (autogenerate: boolean) => {
+      if (!experiment.data) return;
+      await createScenarioMutation.mutateAsync({
+        experimentId: experiment.data.id,
+        autogenerate,
+      });
+      await utils.scenarios.list.invalidate();
+    },
+    [createScenarioMutation],
+  );
+
+  return (
+    <HStack w="100%" pb={cellPadding.y} pt={0} align="center" spacing={0}>
+      <Text fontSize={16} fontWeight="bold">
+        Scenarios ({scenarios.data?.count})
+      </Text>
+      {canModify && (
+        <Menu>
+          <MenuButton
+            as={IconButton}
+            mt={1}
+            variant="ghost"
+            aria-label="Edit Scenarios"
+            icon={<Icon as={loading ? Spinner : BsGear} />}
+          />
+          <MenuList fontSize="md" zIndex="dropdown" mt={-3}>
+            <MenuItem
+              icon={<Icon as={BsPlus} boxSize={6} mx="-5px" />}
+              onClick={() => onAddScenario(false)}
+            >
+              Add Scenario
+            </MenuItem>
+            <MenuItem icon={<BsStars />} onClick={() => onAddScenario(true)}>
+              Autogenerate Scenario
+            </MenuItem>
+            <MenuItem icon={<BsPencil />} onClick={openDrawer}>
+              Edit Vars
+            </MenuItem>
+          </MenuList>
+        </Menu>
+      )}
+    </HStack>
+  );
+};

app/src/components/OutputsTable/VariantEditor.tsx

@@ -0,0 +1,239 @@
+import {
+  Box,
+  Button,
+  HStack,
+  IconButton,
+  Spinner,
+  Text,
+  Tooltip,
+  useToast,
+} from "@chakra-ui/react";
+import { useCallback, useEffect, useRef, useState } from "react";
+import { FiMaximize, FiMinimize } from "react-icons/fi";
+import { editorBackground } from "~/state/sharedVariantEditor.slice";
+import { useAppStore } from "~/state/store";
+import { api } from "~/utils/api";
+import {
+  useExperimentAccess,
+  useHandledAsyncCallback,
+  useModifierKeyLabel,
+  useVisibleScenarioIds,
+} from "~/utils/hooks";
+import { type PromptVariant } from "./types";
+
+export default function VariantEditor(props: { variant: PromptVariant }) {
+  const { canModify } = useExperimentAccess();
+  const monaco = useAppStore.use.sharedVariantEditor.monaco();
+  const editorRef = useRef<ReturnType<NonNullable<typeof monaco>["editor"]["create"]> | null>(null);
+  const containerRef = useRef<HTMLDivElement | null>(null);
+  const [editorId] = useState(() => `editor_${Math.random().toString(36).substring(7)}`);
+  const [isChanged, setIsChanged] = useState(false);
+
+  const [isFullscreen, setIsFullscreen] = useState(false);
+
+  const toggleFullscreen = useCallback(() => {
+    setIsFullscreen((prev) => !prev);
+    editorRef.current?.focus();
+  }, [setIsFullscreen]);
+
+  useEffect(() => {
+    const handleEsc = (event: KeyboardEvent) => {
+      if (event.key === "Escape" && isFullscreen) {
+        toggleFullscreen();
+      }
+    };
+
+    window.addEventListener("keydown", handleEsc);
+    return () => window.removeEventListener("keydown", handleEsc);
+  }, [isFullscreen, toggleFullscreen]);
+
+  const lastSavedFn = props.variant.promptConstructor;
+
+  const modifierKey = useModifierKeyLabel();
+
+  const checkForChanges = useCallback(() => {
+    if (!editorRef.current) return;
+    const currentFn = editorRef.current.getValue();
+    setIsChanged(currentFn.length > 0 && currentFn !== lastSavedFn);
+  }, [lastSavedFn]);
+
+  const matchUpdatedSavedFn = useCallback(() => {
+    if (!editorRef.current) return;
+    editorRef.current.setValue(lastSavedFn);
+    setIsChanged(false);
+  }, [lastSavedFn]);
+
+  useEffect(matchUpdatedSavedFn, [matchUpdatedSavedFn, lastSavedFn]);
+
+  const replaceVariant = api.promptVariants.replaceVariant.useMutation();
+  const utils = api.useContext();
+  const toast = useToast();
+  const visibleScenarios = useVisibleScenarioIds();
+
+  const [onSave, saveInProgress] = useHandledAsyncCallback(async () => {
+    if (!editorRef.current) return;
+
+    await editorRef.current.getAction("editor.action.formatDocument")?.run();
+
+    const currentFn = editorRef.current.getValue();
+
+    if (!currentFn) return;
+
+    // Check if the editor has any typescript errors
+    const model = editorRef.current.getModel();
+    if (!model) return;
+
+    // Make sure the user defined the prompt with the string "prompt\w*=" somewhere
+    const promptRegex = /definePrompt\(/;
+    if (!promptRegex.test(currentFn)) {
+      toast({
+        title: "Missing prompt",
+        description: "Please define the prompt (eg. `definePrompt(...`",
+        status: "error",
+      });
+      return;
+    }
+
+    const resp = await replaceVariant.mutateAsync({
+      id: props.variant.id,
+      promptConstructor: currentFn,
+      streamScenarios: visibleScenarios,
+    });
+    if (resp.status === "error") {
+      return toast({
+        title: "Error saving variant",
+        description: resp.message,
+        status: "error",
+      });
+    }
+
+    setIsChanged(false);
+
+    await utils.promptVariants.list.invalidate();
+  }, [checkForChanges]);
+
+  useEffect(() => {
+    if (monaco) {
+      const container = document.getElementById(editorId) as HTMLElement;
+
+      editorRef.current = monaco.editor.create(container, {
+        value: lastSavedFn,
+        language: "typescript",
+        theme: "customTheme",
+        lineNumbers: "off",
+        minimap: { enabled: false },
+        wrappingIndent: "indent",
+        wrappingStrategy: "advanced",
+        wordWrap: "on",
+        folding: false,
+        scrollbar: {
+          alwaysConsumeMouseWheel: false,
+          verticalScrollbarSize: 0,
+        },
+        wordWrapBreakAfterCharacters: "",
+        wordWrapBreakBeforeCharacters: "",
+        quickSuggestions: true,
+        readOnly: !canModify,
+      });
+
+      // Workaround because otherwise the commands only work on whatever
+      // editor was loaded on the page last.
+      // https://github.com/microsoft/monaco-editor/issues/2947#issuecomment-1422265201
+      editorRef.current.onDidFocusEditorText(() => {
+        editorRef.current?.addCommand(monaco.KeyMod.CtrlCmd | monaco.KeyCode.KeyS, onSave);
+
+        editorRef.current?.addCommand(
+          monaco.KeyMod.CtrlCmd | monaco.KeyMod.Shift | monaco.KeyCode.KeyF,
+          toggleFullscreen,
+        );
+
+        // Exit fullscreen with escape
+        editorRef.current?.addCommand(monaco.KeyCode.Escape, () => {
+          if (isFullscreen) {
+            toggleFullscreen();
+          }
+        });
+      });
+
+      editorRef.current.onDidChangeModelContent(checkForChanges);
+
+      const resizeObserver = new ResizeObserver(() => {
+        editorRef.current?.layout();
+      });
+      resizeObserver.observe(container);
+
+      return () => {
+        resizeObserver.disconnect();
+        editorRef.current?.dispose();
+      };
+    }
+
+    // We intentionally skip the onSave and props.savedConfig dependencies here because
+    // we don't want to re-render the editor from scratch
+    /* eslint-disable-next-line react-hooks/exhaustive-deps */
+  }, [monaco, editorId]);
+
+  useEffect(() => {
+    if (!editorRef.current) return;
+    editorRef.current.updateOptions({
+      readOnly: !canModify,
+    });
+  }, [canModify]);
+
+  return (
+    <Box
+      w="100%"
+      ref={containerRef}
+      sx={
+        isFullscreen
+          ? {
+              position: "fixed",
+              top: 0,
+              left: 0,
+              right: 0,
+              bottom: 0,
+            }
+          : { h: "400px", w: "100%" }
+      }
+      bgColor={editorBackground}
+      zIndex={isFullscreen ? 1000 : "unset"}
+      pos="relative"
+      _hover={{ ".fullscreen-toggle": { opacity: 1 } }}
+    >
+      <Box id={editorId} w="100%" h="100%" />
+      <Tooltip label={`${modifierKey} + ⇧ + F`}>
+        <IconButton
+          className="fullscreen-toggle"
+          aria-label="Minimize"
+          icon={isFullscreen ? <FiMinimize /> : <FiMaximize />}
+          position="absolute"
+          top={2}
+          right={2}
+          onClick={toggleFullscreen}
+          opacity={0}
+          transition="opacity 0.2s"
+        />
+      </Tooltip>
+
+      {isChanged && (
+        <HStack pos="absolute" bottom={2} right={2}>
+          <Button
+            colorScheme="gray"
+            size="sm"
+            onClick={() => {
+              editorRef.current?.setValue(lastSavedFn);
+              checkForChanges();
+            }}
+          >
+            Reset
+          </Button>
+          <Tooltip label={`${modifierKey} + S`}>
+            <Button size="sm" onClick={onSave} colorScheme="blue" w={16} disabled={saveInProgress}>
+              {saveInProgress ? <Spinner boxSize={4} /> : <Text>Save</Text>}
+            </Button>
+          </Tooltip>
+        </HStack>
+      )}
+    </Box>
+  );
+}

app/src/components/OutputsTable/VariantStats.tsx

@@ -0,0 +1,83 @@
+import { HStack, Icon, Text, useToken } from "@chakra-ui/react";
+import { type PromptVariant } from "./types";
+import { cellPadding } from "../constants";
+import { api } from "~/utils/api";
+import chroma from "chroma-js";
+import { BsCurrencyDollar } from "react-icons/bs";
+import { CostTooltip } from "../tooltip/CostTooltip";
+import { useEffect, useState } from "react";
+
+export default function VariantStats(props: { variant: PromptVariant }) {
+  const [refetchInterval, setRefetchInterval] = useState(0);
+  const { data } = api.promptVariants.stats.useQuery(
+    {
+      variantId: props.variant.id,
+    },
+    {
+      initialData: {
+        evalResults: [],
+        overallCost: 0,
+        promptTokens: 0,
+        completionTokens: 0,
+        scenarioCount: 0,
+        outputCount: 0,
+        awaitingEvals: false,
+      },
+      refetchInterval,
+    },
+  );
+
+  // Poll every two seconds while we are waiting for LLM retrievals to finish
+  useEffect(() => setRefetchInterval(data.awaitingEvals ? 5000 : 0), [data.awaitingEvals]);
+
+  const [passColor, neutralColor, failColor] = useToken("colors", [
+    "green.500",
+    "gray.500",
+    "red.500",
+  ]);
+
+  const scale = chroma.scale([failColor, neutralColor, passColor]).domain([0, 0.5, 1]);
+
+  const showNumFinished = data.scenarioCount > 0 && data.scenarioCount !== data.outputCount;
+
+  return (
+    <HStack
+      justifyContent="space-between"
+      alignItems="flex-end"
+      mx="2"
+      fontSize="xs"
+      py={cellPadding.y}
+    >
+      <HStack px={cellPadding.x} flexWrap="wrap">
+        {showNumFinished && (
+          <Text>
+            {data.outputCount} / {data.scenarioCount}
+          </Text>
+        )}
+        {data.evalResults.map((result) => {
+          const passedFrac = result.passCount / result.totalCount;
+          return (
+            <HStack key={result.id}>
+              <Text>{result.label}</Text>
+              <Text color={scale(passedFrac).hex()} fontWeight="bold">
+                {(passedFrac * 100).toFixed(1)}%
+              </Text>
+            </HStack>
+          );
+        })}
+      </HStack>
+      {data.overallCost && (
+        <CostTooltip
+          promptTokens={data.promptTokens}
+          completionTokens={data.completionTokens}
+          cost={data.overallCost}
+        >
+          <HStack spacing={0} align="center" color="gray.500">
+            <Icon as={BsCurrencyDollar} />
+            <Text mr={1}>{data.overallCost.toFixed(3)}</Text>
+          </HStack>
+        </CostTooltip>
+      )}
+    </HStack>
+  );
+}

app/src/components/OutputsTable/index.tsx

@@ -0,0 +1,107 @@
+import { Grid, GridItem, type GridItemProps } from "@chakra-ui/react";
+import { api } from "~/utils/api";
+import AddVariantButton from "./AddVariantButton";
+import ScenarioRow from "./ScenarioRow";
+import VariantEditor from "./VariantEditor";
+import VariantHeader from "../VariantHeader/VariantHeader";
+import VariantStats from "./VariantStats";
+import { ScenariosHeader } from "./ScenariosHeader";
+import { borders } from "./styles";
+import { useScenarios } from "~/utils/hooks";
+import ScenarioPaginator from "./ScenarioPaginator";
+import { Fragment } from "react";
+
+export default function OutputsTable({ experimentId }: { experimentId: string | undefined }) {
+  const variants = api.promptVariants.list.useQuery(
+    { experimentId: experimentId as string },
+    { enabled: !!experimentId },
+  );
+
+  const scenarios = useScenarios();
+
+  if (!variants.data || !scenarios.data) return null;
+
+  const allCols = variants.data.length + 2;
+  const variantHeaderRows = 3;
+  const scenarioHeaderRows = 1;
+  const scenarioFooterRows = 1;
+  const visibleScenariosCount = scenarios.data.scenarios.length;
+  const allRows =
+    variantHeaderRows + scenarioHeaderRows + visibleScenariosCount + scenarioFooterRows;
+
+  return (
+    <Grid
+      pt={4}
+      pb={24}
+      pl={8}
+      display="grid"
+      gridTemplateColumns={`250px repeat(${variants.data.length}, minmax(320px, 1fr)) auto`}
+      sx={{
+        "> *": {
+          borderColor: "gray.300",
+        },
+      }}
+      fontSize="sm"
+    >
+      <GridItem rowSpan={variantHeaderRows}>
+        <AddVariantButton />
+      </GridItem>
+
+      {variants.data.map((variant, i) => {
+        const sharedProps: GridItemProps = {
+          ...borders,
+          colStart: i + 2,
+          borderLeftWidth: i === 0 ? 1 : 0,
+          marginLeft: i === 0 ? "-1px" : 0,
+          backgroundColor: "gray.100",
+        };
+        return (
+          <Fragment key={variant.uiId}>
+            <VariantHeader
+              variant={variant}
+              canHide={variants.data.length > 1}
+              rowStart={1}
+              {...sharedProps}
+            />
+            <GridItem rowStart={2} {...sharedProps}>
+              <VariantEditor variant={variant} />
+            </GridItem>
+            <GridItem rowStart={3} {...sharedProps}>
+              <VariantStats variant={variant} />
+            </GridItem>
+          </Fragment>
+        );
+      })}
+
+      <GridItem
+        colSpan={allCols - 1}
+        rowStart={variantHeaderRows + 1}
+        colStart={1}
+        {...borders}
+        borderRightWidth={0}
+      >
+        <ScenariosHeader />
+      </GridItem>
+
+      {scenarios.data.scenarios.map((scenario, i) => (
+        <ScenarioRow
+          rowStart={i + variantHeaderRows + scenarioHeaderRows + 2}
+          key={scenario.uiId}
+          scenario={scenario}
+          variants={variants.data}
+          canHide={visibleScenariosCount > 1}
+        />
+      ))}
+      <GridItem
+        rowStart={variantHeaderRows + scenarioHeaderRows + visibleScenariosCount + 2}
+        colStart={1}
+        colSpan={allCols}
+      >
+        <ScenarioPaginator />
+      </GridItem>
+
+      {/* Add some extra padding on the right, because when the table is too wide to fit in the viewport `pr` on the Grid isn't respected. */}
+      <GridItem rowStart={1} colStart={allCols} rowSpan={allRows} w={4} borderBottomWidth={0} />
+    </Grid>
+  );
+}

app/src/components/OutputsTable/styles.ts

@@ -0,0 +1,6 @@
+import { type GridItemProps } from "@chakra-ui/react";
+
+export const borders: GridItemProps = {
+  borderRightWidth: 1,
+  borderBottomWidth: 1,
+};

app/src/components/OutputsTable/types.ts

@@ -0,0 +1,5 @@
+import { type RouterOutputs } from "~/utils/api";
+
+export type PromptVariant = NonNullable<RouterOutputs["promptVariants"]["list"]>[0];
+
+export type Scenario = NonNullable<RouterOutputs["scenarios"]["list"]>["scenarios"][0];