settings drawer

.eslintrc.cjs

@@ -32,6 +32,7 @@ const config = {
     ],
 
     "unused-imports/no-unused-imports": "error",
+    "@typescript-eslint/no-unused-vars": "off",
     "unused-imports/no-unused-vars": [
       "warn",
       { vars: "all", varsIgnorePattern: "^_", args: "after-used", argsIgnorePattern: "^_" },

package.json

@@ -51,7 +51,8 @@
     "socket.io-client": "^4.7.1",
     "superjson": "1.12.2",
     "tsx": "^3.12.7",
-    "zod": "^3.21.4"
+    "zod": "^3.21.4",
+    "zustand": "^4.3.9"
   },
   "devDependencies": {
     "@openapi-contrib/openapi-schema-to-json-schema": "^4.0.5",

pnpm-lock.yaml

@@ -119,6 +119,9 @@ dependencies:
   zod:
     specifier: ^3.21.4
     version: 3.21.4
+  zustand:
+    specifier: ^4.3.9
+    version: 4.3.9(react@18.2.0)
 
 devDependencies:
   '@openapi-contrib/openapi-schema-to-json-schema':
@@ -5964,3 +5967,19 @@ packages:
   /zod@3.21.4:
     resolution: {integrity: sha512-m46AKbrzKVzOzs/DZgVnG5H55N1sv1M8qZU3A8RIKbs3mrACDNeIOeilDymVb2HdmP8uwshOCF4uJ8uM9rCqJw==}
     dev: false
+
+  /zustand@4.3.9(react@18.2.0):
+    resolution: {integrity: sha512-Tat5r8jOMG1Vcsj8uldMyqYKC5IZvQif8zetmLHs9WoZlntTHmIoNM8TpLRY31ExncuUvUOXehd0kvahkuHjDw==}
+    engines: {node: '>=12.7.0'}
+    peerDependencies:
+      immer: '>=9.0'
+      react: '>=16.8'
+    peerDependenciesMeta:
+      immer:
+        optional: true
+      react:
+        optional: true
+    dependencies:
+      react: 18.2.0
+      use-sync-external-store: 1.2.0(react@18.2.0)
+    dev: false

src/components/OutputsTable/EditEvaluations.tsx

@@ -0,0 +1,44 @@
+import { Text, Heading, Stack } from "@chakra-ui/react";
+import { useState } from "react";
+import { api } from "~/utils/api";
+import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+import { useStore } from "~/utils/store";
+
+export default function EditEvaluations() {
+  const experiment = useExperiment();
+  const vars =
+    api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" }).data ?? [];
+
+  const [newVariable, setNewVariable] = useState<string>("");
+  const newVarIsValid = newVariable.length > 0 && !vars.map((v) => v.label).includes(newVariable);
+
+  const utils = api.useContext();
+  const addVarMutation = api.templateVars.create.useMutation();
+  const [onAddVar] = useHandledAsyncCallback(async () => {
+    if (!experiment.data?.id) return;
+    if (!newVarIsValid) return;
+    await addVarMutation.mutateAsync({
+      experimentId: experiment.data.id,
+      label: newVariable,
+    });
+    await utils.templateVars.list.invalidate();
+    setNewVariable("");
+  }, [addVarMutation, experiment.data?.id, newVarIsValid, newVariable]);
+
+  const deleteMutation = api.templateVars.delete.useMutation();
+  const [onDeleteVar] = useHandledAsyncCallback(async (id: string) => {
+    await deleteMutation.mutateAsync({ id });
+    await utils.templateVars.list.invalidate();
+  }, []);
+
+  const closeDrawer = useStore((state) => state.closeDrawer);
+
+  return (
+    <Stack>
+      <Heading size="sm">Edit Evaluations</Heading>
+      <Stack spacing={2} pt={2}>
+        <Text fontSize="sm"></Text>
+      </Stack>
+    </Stack>
+  );
+}

src/components/OutputsTable/EditScenarioVars.tsx

@@ -0,0 +1,104 @@
+import { Text, Button, HStack, Heading, Icon, Input, Stack, Code } from "@chakra-ui/react";
+import { useState } from "react";
+import { BsCheck, BsX } from "react-icons/bs";
+import { api } from "~/utils/api";
+import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+
+export default function EditScenarioVars() {
+  const experiment = useExperiment();
+  const vars =
+    api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" }).data ?? [];
+
+  const [newVariable, setNewVariable] = useState<string>("");
+  const newVarIsValid = newVariable.length > 0 && !vars.map((v) => v.label).includes(newVariable);
+
+  const utils = api.useContext();
+  const addVarMutation = api.templateVars.create.useMutation();
+  const [onAddVar] = useHandledAsyncCallback(async () => {
+    if (!experiment.data?.id) return;
+    if (!newVarIsValid) return;
+    await addVarMutation.mutateAsync({
+      experimentId: experiment.data.id,
+      label: newVariable,
+    });
+    await utils.templateVars.list.invalidate();
+    setNewVariable("");
+  }, [addVarMutation, experiment.data?.id, newVarIsValid, newVariable]);
+
+  const deleteMutation = api.templateVars.delete.useMutation();
+  const [onDeleteVar] = useHandledAsyncCallback(async (id: string) => {
+    await deleteMutation.mutateAsync({ id });
+    await utils.templateVars.list.invalidate();
+  }, []);
+
+  return (
+    <Stack>
+      <Heading size="sm">Edit Scenario Variables</Heading>
+      <Stack spacing={2} pt={2}>
+        <Text fontSize="sm">
+          Scenario variables can be used in your prompt variants as well as evaluations. Reference
+          them using <Code>{"{{curly_braces}}"}</Code>.
+        </Text>
+        <HStack spacing={0}>
+          <Input
+            placeholder="Add Scenario Variable"
+            size="sm"
+            borderTopRadius={0}
+            borderRightRadius={0}
+            value={newVariable}
+            onChange={(e) => setNewVariable(e.target.value)}
+            onKeyDown={(e) => {
+              if (e.key === "Enter") {
+                e.preventDefault();
+                onAddVar();
+              }
+              // If the user types a space, replace it with an underscore
+              if (e.key === " ") {
+                e.preventDefault();
+                setNewVariable((v) => v + "_");
+              }
+            }}
+          />
+          <Button
+            size="xs"
+            height="100%"
+            borderLeftRadius={0}
+            isDisabled={!newVarIsValid}
+            onClick={onAddVar}
+          >
+            <Icon as={BsCheck} boxSize={8} />
+          </Button>
+        </HStack>
+
+        <HStack spacing={2} py={4} wrap="wrap">
+          {vars.map((variable) => (
+            <HStack
+              key={variable.id}
+              spacing={0}
+              bgColor="blue.100"
+              color="blue.600"
+              pl={2}
+              pr={0}
+              fontWeight="bold"
+            >
+              <Text fontSize="sm" flex={1}>
+                {variable.label}
+              </Text>
+              <Button
+                size="xs"
+                variant="ghost"
+                colorScheme="blue"
+                p="unset"
+                minW="unset"
+                px="unset"
+                onClick={() => onDeleteVar(variable.id)}
+              >
+                <Icon as={BsX} boxSize={6} color="blue.800" />
+              </Button>
+            </HStack>
+          ))}
+        </HStack>
+      </Stack>
+    </Stack>
+  );
+}

src/components/OutputsTable/OutputCell.tsx

@@ -114,7 +114,8 @@ export default function OutputCell({
     );
   }
 
-  const contentToDisplay = message?.content ?? streamedContent ?? JSON.stringify(output.data?.output);
+  const contentToDisplay =
+    message?.content ?? streamedContent ?? JSON.stringify(output.data?.output);
 
   return (
     <Flex w="100%" h="100%" direction="column" justifyContent="space-between" whiteSpace="pre-wrap">

src/components/OutputsTable/SettingsDrawer.tsx

@@ -0,0 +1,32 @@
+import {
+  Drawer,
+  DrawerBody,
+  DrawerCloseButton,
+  DrawerContent,
+  DrawerHeader,
+  DrawerOverlay,
+  Heading,
+} from "@chakra-ui/react";
+import { useStore } from "~/utils/store";
+import EditScenarioVars from "./EditScenarioVars";
+
+export default function SettingsDrawer() {
+  const isOpen = useStore((state) => state.drawerOpen);
+  const closeDrawer = useStore((state) => state.closeDrawer);
+
+  return (
+    <Drawer isOpen={isOpen} placement="right" onClose={closeDrawer} size="md">
+      <DrawerOverlay />
+      <DrawerContent>
+        <DrawerCloseButton />
+        <DrawerHeader>
+          <Heading size="md">Settings</Heading>
+        </DrawerHeader>
+        <DrawerBody>
+          <EditScenarioVars />
+          {/* <EditEvaluations /> */}
+        </DrawerBody>
+      </DrawerContent>
+    </Drawer>
+  );
+}

src/components/OutputsTable/index.tsx

@@ -1,11 +1,20 @@
-import { Box, Grid, GridItem, Heading, type SystemStyleObject } from "@chakra-ui/react";
-import ScenarioHeader from "~/server/ScenarioHeader";
+import {
+  Button,
+  Grid,
+  GridItem,
+  HStack,
+  Heading,
+  type SystemStyleObject,
+} from "@chakra-ui/react";
 import { api } from "~/utils/api";
 import NewScenarioButton from "./NewScenarioButton";
 import NewVariantButton from "./NewVariantButton";
 import ScenarioRow from "./ScenarioRow";
 import VariantConfigEditor from "./VariantConfigEditor";
 import VariantHeader from "./VariantHeader";
+import { cellPadding } from "../constants";
+import { BsPencil } from "react-icons/bs";
+import { useStore } from "~/utils/store";
 
 const stickyHeaderStyle: SystemStyleObject = {
   position: "sticky",
@@ -19,6 +28,7 @@ export default function OutputsTable({ experimentId }: { experimentId: string |
     { experimentId: experimentId as string },
     { enabled: !!experimentId }
   );
+  const openDrawer = useStore((s) => s.openDrawer);
 
   const scenarios = api.scenarios.list.useQuery(
     { experimentId: experimentId as string },
@@ -42,8 +52,28 @@ export default function OutputsTable({ experimentId }: { experimentId: string |
         },
       }}
     >
-      <GridItem display="flex" alignItems="flex-end" rowSpan={2}>
-        <ScenarioHeader />
+      <GridItem
+        display="flex"
+        alignItems="flex-end"
+        rowSpan={2}
+        px={cellPadding.x}
+        py={cellPadding.y}
+      >
+        <HStack w="100%">
+          <Heading size="sm" fontWeight="bold" flex={1}>
+            Scenario
+          </Heading>
+          <Button
+            size="xs"
+            variant="ghost"
+            color="gray.500"
+            aria-label="Edit"
+            leftIcon={<BsPencil />}
+            onClick={openDrawer}
+          >
+            Edit Vars
+          </Button>
+        </HStack>
       </GridItem>
 
       {variants.data.map((variant) => (

src/pages/experiments/[id].tsx

@@ -11,12 +11,14 @@ import {
 } from "@chakra-ui/react";
 import { useRouter } from "next/router";
 import { useState, useEffect } from "react";
-import { BsTrash } from "react-icons/bs";
+import { BsGearFill, BsTrash } from "react-icons/bs";
 import { RiFlaskLine } from "react-icons/ri";
 import OutputsTable from "~/components/OutputsTable";
+import SettingsDrawer from "~/components/OutputsTable/SettingsDrawer";
 import AppShell from "~/components/nav/AppShell";
 import { api } from "~/utils/api";
 import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
+import { useStore } from "~/utils/store";
 
 const DeleteButton = (props: { experimentId: string }) => {
   const mutation = api.experiments.delete.useMutation();
@@ -44,6 +46,7 @@ export default function Experiment() {
   const router = useRouter();
   const experiment = useExperiment();
   const utils = api.useContext();
+  const openDrawer = useStore((s) => s.openDrawer);
 
   const [label, setLabel] = useState(experiment.data?.label || "");
   useEffect(() => {
@@ -98,8 +101,19 @@ export default function Experiment() {
               />
             </BreadcrumbItem>
           </Breadcrumb>
+          <Button
+            size="sm"
+            variant="ghost"
+            colorScheme="gray"
+            fontWeight="normal"
+            onClick={openDrawer}
+            leftIcon={<Icon as={BsGearFill} boxSize={4} color="gray.600" />}
+          >
+            Edit Vars & Evals
+          </Button>
           <DeleteButton experimentId={router.query.id as string} />
         </HStack>
+        <SettingsDrawer />
         <OutputsTable experimentId={router.query.id as string | undefined} />
       </Box>
     </AppShell>

src/server/ScenarioHeader.tsx

@@ -1,124 +0,0 @@
-import { Text, Button, HStack, Heading, Icon, Input, Stack, Code } from "@chakra-ui/react";
-import { useState } from "react";
-import { BsCheck, BsChevronDown, BsX } from "react-icons/bs";
-import { cellPadding } from "~/components/constants";
-import { api } from "~/utils/api";
-import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
-
-export default function ScenarioHeader() {
-  const experiment = useExperiment();
-  const vars =
-    api.templateVars.list.useQuery({ experimentId: experiment.data?.id ?? "" }).data ?? [];
-
-  const [newVariable, setNewVariable] = useState<string>("");
-  const newVarIsValid = newVariable.length > 0 && !vars.map((v) => v.label).includes(newVariable);
-
-  const [editing, setEditing] = useState(false);
-
-  const utils = api.useContext();
-  const addVarMutation = api.templateVars.create.useMutation();
-  const [onAddVar] = useHandledAsyncCallback(async () => {
-    if (!experiment.data?.id) return;
-    if (!newVarIsValid) return;
-    await addVarMutation.mutateAsync({
-      experimentId: experiment.data.id,
-      label: newVariable,
-    });
-    await utils.templateVars.list.invalidate();
-    setNewVariable("");
-  }, [addVarMutation, experiment.data?.id, newVarIsValid, newVariable]);
-
-  const deleteMutation = api.templateVars.delete.useMutation();
-  const [onDeleteVar] = useHandledAsyncCallback(async (id: string) => {
-    await deleteMutation.mutateAsync({ id });
-    await utils.templateVars.list.invalidate();
-  }, []);
-
-  return (
-    <Stack flex={1} px={cellPadding.x} py={cellPadding.y}>
-      <HStack>
-        <Heading size="sm" fontWeight="bold" flex={1}>
-          Scenario
-        </Heading>
-        {
-          <Button
-            size="xs"
-            variant="outline"
-            colorScheme="blue"
-            rightIcon={<Icon as={editing ? BsCheck : BsChevronDown} />}
-            onClick={() => setEditing((editing) => !editing)}
-          >
-            {editing ? "Finish" : "Edit Vars"}
-          </Button>
-        }
-      </HStack>
-      {editing && (
-        <Stack spacing={2} pt={2}>
-          <Text fontSize="sm">
-            Define scenario variables. Reference them from your prompt variants using{" "}
-            <Code>{`{{curly_braces}}`}</Code>.
-          </Text>
-          <HStack spacing={0}>
-            <Input
-              placeholder="Add Variable Name"
-              size="sm"
-              borderTopRadius={0}
-              borderRightRadius={0}
-              value={newVariable}
-              onChange={(e) => setNewVariable(e.target.value)}
-              onKeyDown={(e) => {
-                if (e.key === "Enter") {
-                  e.preventDefault();
-                  onAddVar();
-                }
-                // If the user types a space, replace it with an underscore
-                if (e.key === " ") {
-                  e.preventDefault();
-                  setNewVariable((v) => v + "_");
-                }
-              }}
-            />
-            <Button
-              size="xs"
-              height="100%"
-              borderLeftRadius={0}
-              isDisabled={!newVarIsValid}
-              onClick={onAddVar}
-            >
-              <Icon as={BsCheck} boxSize={8} />
-            </Button>
-          </HStack>
-
-          <HStack spacing={2} py={4} wrap="wrap">
-            {vars.map((variable) => (
-              <HStack
-                key={variable.id}
-                spacing={0}
-                bgColor="blue.100"
-                color="blue.600"
-                pl={2}
-                pr={0}
-                fontWeight="bold"
-              >
-                <Text fontSize="sm" flex={1}>
-                  {variable.label}
-                </Text>
-                <Button
-                  size="xs"
-                  variant="ghost"
-                  colorScheme="blue"
-                  p="unset"
-                  minW="unset"
-                  px="unset"
-                  onClick={() => onDeleteVar(variable.id)}
-                >
-                  <Icon as={BsX} boxSize={6} color="blue.800" />
-                </Button>
-              </HStack>
-            ))}
-          </HStack>
-        </Stack>
-      )}
-    </Stack>
-  );
-}

src/utils/store.ts

@@ -0,0 +1,13 @@
+import { create } from "zustand";
+
+type StoreState = {
+  drawerOpen: boolean;
+  openDrawer: () => void;
+  closeDrawer: () => void;
+};
+
+export const useStore = create<StoreState>()((set) => ({
+  drawerOpen: true,
+  openDrawer: () => set({ drawerOpen: true }),
+  closeDrawer: () => set({ drawerOpen: false }),
+}));