Merge pull request #123 from OpenPipe/add-openapi

Add Logged Calls and projects

app/src/components/CopiableCode.tsx

@@ -0,0 +1,40 @@
+import { HStack, Icon, IconButton, Tooltip, Text } from "@chakra-ui/react";
+import { useState } from "react";
+import { MdContentCopy } from "react-icons/md";
+import { useHandledAsyncCallback } from "~/utils/hooks";
+
+const CopiableCode = ({ code }: { code: string }) => {
+  const [copied, setCopied] = useState(false);
+
+  const [copyToClipboard] = useHandledAsyncCallback(async () => {
+    await navigator.clipboard.writeText(code);
+    setCopied(true);
+  }, [code]);
+  return (
+    <HStack
+      backgroundColor="blackAlpha.800"
+      color="white"
+      borderRadius={4}
+      padding={3}
+      w="full"
+      justifyContent="space-between"
+    >
+      <Text fontFamily="inconsolata" fontWeight="bold" letterSpacing={0.5}>
+        {code}
+      </Text>
+      <Tooltip closeOnClick={false} label={copied ? "Copied!" : "Copy to clipboard"}>
+        <IconButton
+          aria-label="Copy"
+          icon={<Icon as={MdContentCopy} boxSize={5} />}
+          size="xs"
+          colorScheme="white"
+          variant="ghost"
+          onClick={copyToClipboard}
+          onMouseLeave={() => setCopied(false)}
+        />
+      </Tooltip>
+    </HStack>
+  );
+};
+
+export default CopiableCode;

app/src/components/ExperimentSettingsDrawer/DeleteButton.tsx

@@ -14,6 +14,7 @@ import {
 import { useRouter } from "next/router";
 import { useRef } from "react";
 import { BsTrash } from "react-icons/bs";
+import { useAppStore } from "~/state/store";
 import { api } from "~/utils/api";
 import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
 
@@ -23,6 +24,8 @@ export const DeleteButton = () => {
   const utils = api.useContext();
   const router = useRouter();
 
+  const closeDrawer = useAppStore((s) => s.closeDrawer);
+
   const { isOpen, onOpen, onClose } = useDisclosure();
   const cancelRef = useRef<HTMLButtonElement>(null);
 
@@ -31,6 +34,8 @@ export const DeleteButton = () => {
     await mutation.mutateAsync({ id: experiment.data.id });
     await utils.experiments.list.invalidate();
     await router.push({ pathname: "/experiments" });
+    closeDrawer();
+
     onClose();
   }, [mutation, experiment.data?.id, router]);
 

app/src/components/StatsCard.tsx

@@ -0,0 +1,26 @@
+import { VStack, HStack, type StackProps, Text, Divider } from "@chakra-ui/react";
+import Link, { type LinkProps } from "next/link";
+
+const StatsCard = ({
+  title,
+  href,
+  children,
+  ...rest
+}: { title: string; href: string } & StackProps & LinkProps) => {
+  return (
+    <VStack flex={1} borderWidth={1} padding={4} borderRadius={4} borderColor="gray.300" {...rest}>
+      <HStack w="full" justifyContent="space-between">
+        <Text fontSize="md" fontWeight="bold">
+          {title}
+        </Text>
+        <Link href={href}>
+          <Text color="blue">View all</Text>
+        </Link>
+      </HStack>
+      <Divider />
+      {children}
+    </VStack>
+  );
+};
+
+export default StatsCard;

app/src/components/dashboard/LoggedCallTable.tsx

@@ -0,0 +1,201 @@
+import {
+  Box,
+  Card,
+  CardHeader,
+  Heading,
+  Table,
+  Tbody,
+  Td,
+  Th,
+  Thead,
+  Tr,
+  Tooltip,
+  Collapse,
+  HStack,
+  VStack,
+  IconButton,
+  useToast,
+  Icon,
+  Button,
+  ButtonGroup,
+} from "@chakra-ui/react";
+import dayjs from "dayjs";
+import relativeTime from "dayjs/plugin/relativeTime";
+import { ChevronUpIcon, ChevronDownIcon, CopyIcon } from "lucide-react";
+import { useMemo, useState } from "react";
+import { type RouterOutputs, api } from "~/utils/api";
+import SyntaxHighlighter from "react-syntax-highlighter";
+import { atelierCaveLight } from "react-syntax-highlighter/dist/cjs/styles/hljs";
+import stringify from "json-stringify-pretty-compact";
+import Link from "next/link";
+
+dayjs.extend(relativeTime);
+
+type LoggedCall = RouterOutputs["dashboard"]["loggedCalls"][0];
+
+const FormattedJson = ({ json }: { json: any }) => {
+  const jsonString = stringify(json, { maxLength: 40 });
+  const toast = useToast();
+
+  const copyToClipboard = async (text: string) => {
+    try {
+      await navigator.clipboard.writeText(text);
+      toast({
+        title: "Copied to clipboard",
+        status: "success",
+        duration: 2000,
+      });
+    } catch (err) {
+      toast({
+        title: "Failed to copy to clipboard",
+        status: "error",
+        duration: 2000,
+      });
+    }
+  };
+
+  return (
+    <Box position="relative" fontSize="sm" borderRadius="md" overflow="hidden">
+      <SyntaxHighlighter
+        customStyle={{ overflowX: "unset" }}
+        language="json"
+        style={atelierCaveLight}
+        lineProps={{
+          style: { wordBreak: "break-all", whiteSpace: "pre-wrap" },
+        }}
+        wrapLines
+      >
+        {jsonString}
+      </SyntaxHighlighter>
+      <IconButton
+        aria-label="Copy"
+        icon={<CopyIcon />}
+        position="absolute"
+        top={1}
+        right={1}
+        size="xs"
+        variant="ghost"
+        onClick={() => void copyToClipboard(jsonString)}
+      />
+    </Box>
+  );
+};
+
+function TableRow({
+  loggedCall,
+  isExpanded,
+  onToggle,
+}: {
+  loggedCall: LoggedCall;
+  isExpanded: boolean;
+  onToggle: () => void;
+}) {
+  const isError = loggedCall.modelResponse?.respStatus !== 200;
+  const timeAgo = dayjs(loggedCall.startTime).fromNow();
+  const fullTime = dayjs(loggedCall.startTime).toString();
+
+  const model = useMemo(
+    () => loggedCall.tags.find((tag) => tag.name.startsWith("$model"))?.value,
+    [loggedCall.tags],
+  );
+
+  return (
+    <>
+      <Tr
+        onClick={onToggle}
+        key={loggedCall.id}
+        _hover={{ bgColor: "gray.100", cursor: "pointer" }}
+        sx={{
+          "> td": { borderBottom: "none" },
+        }}
+      >
+        <Td>
+          <Icon boxSize={6} as={isExpanded ? ChevronUpIcon : ChevronDownIcon} />
+        </Td>
+        <Td>
+          <Tooltip label={fullTime} placement="top">
+            <Box whiteSpace="nowrap" minW="120px">
+              {timeAgo}
+            </Box>
+          </Tooltip>
+        </Td>
+        <Td width="100%">{model}</Td>
+        <Td isNumeric>{((loggedCall.modelResponse?.durationMs ?? 0) / 1000).toFixed(2)}s</Td>
+        <Td isNumeric>{loggedCall.modelResponse?.inputTokens}</Td>
+        <Td isNumeric>{loggedCall.modelResponse?.outputTokens}</Td>
+        <Td sx={{ color: isError ? "red.500" : "green.500", fontWeight: "semibold" }} isNumeric>
+          {loggedCall.modelResponse?.respStatus ?? "No response"}
+        </Td>
+      </Tr>
+      <Tr>
+        <Td colSpan={8} p={0}>
+          <Collapse in={isExpanded} unmountOnExit={true}>
+            <VStack p={4} align="stretch">
+              <HStack align="stretch">
+                <VStack flex={1} align="stretch">
+                  <Heading size="sm">Input</Heading>
+                  <FormattedJson json={loggedCall.modelResponse?.reqPayload} />
+                </VStack>
+                <VStack flex={1} align="stretch">
+                  <Heading size="sm">Output</Heading>
+                  <FormattedJson json={loggedCall.modelResponse?.respPayload} />
+                </VStack>
+              </HStack>
+              <ButtonGroup alignSelf="flex-end">
+                <Button as={Link} colorScheme="blue" href={{ pathname: "/experiments" }}>
+                  Experiments
+                </Button>
+              </ButtonGroup>
+            </VStack>
+          </Collapse>
+        </Td>
+      </Tr>
+    </>
+  );
+}
+
+export default function LoggedCallTable() {
+  const [expandedRow, setExpandedRow] = useState<string | null>(null);
+  const loggedCalls = api.dashboard.loggedCalls.useQuery({});
+
+  return (
+    <Card variant="outline" width="100%" overflow="hidden">
+      <CardHeader>
+        <Heading as="h3" size="sm">
+          Logged Calls
+        </Heading>
+      </CardHeader>
+      <Table>
+        <Thead>
+          <Tr>
+            <Th />
+            <Th>Time</Th>
+            <Th>Model</Th>
+            <Th isNumeric>Duration</Th>
+            <Th isNumeric>Input tokens</Th>
+            <Th isNumeric>Output tokens</Th>
+            <Th isNumeric>Status</Th>
+          </Tr>
+        </Thead>
+        <Tbody>
+          {loggedCalls.data?.map((loggedCall) => {
+            return (
+              <TableRow
+                key={loggedCall.id}
+                loggedCall={loggedCall}
+                isExpanded={loggedCall.id === expandedRow}
+                onToggle={() => {
+                  if (loggedCall.id === expandedRow) {
+                    setExpandedRow(null);
+                  } else {
+                    setExpandedRow(loggedCall.id);
+                  }
+                }}
+              />
+            );
+          })}
+        </Tbody>
+      </Table>
+    </Card>
+  );
+}

app/src/components/datasets/DatasetCard.tsx

@@ -15,6 +15,7 @@ import { useRouter } from "next/router";
 import { BsPlusSquare } from "react-icons/bs";
 import { api } from "~/utils/api";
 import { useHandledAsyncCallback } from "~/utils/hooks";
+import { useAppStore } from "~/state/store";
 
 type DatasetData = {
   name: string;
@@ -71,11 +72,12 @@ const CountLabel = ({ label, count }: { label: string; count: number }) => {
 
 export const NewDatasetCard = () => {
   const router = useRouter();
+  const selectedOrgId = useAppStore((s) => s.selectedOrgId);
   const createMutation = api.datasets.create.useMutation();
   const [createDataset, isLoading] = useHandledAsyncCallback(async () => {
-    const newDataset = await createMutation.mutateAsync({ label: "New Dataset" });
+    const newDataset = await createMutation.mutateAsync({ organizationId: selectedOrgId ?? "" });
     await router.push({ pathname: "/data/[id]", query: { id: newDataset.id } });
-  }, [createMutation, router]);
+  }, [createMutation, router, selectedOrgId]);
 
   return (
     <AspectRatio ratio={1.2} w="full">

app/src/components/experiments/ExperimentCard.tsx

@@ -15,6 +15,7 @@ import { useRouter } from "next/router";
 import { BsPlusSquare } from "react-icons/bs";
 import { api } from "~/utils/api";
 import { useHandledAsyncCallback } from "~/utils/hooks";
+import { useAppStore } from "~/state/store";
 
 type ExperimentData = {
   testScenarioCount: number;
@@ -75,11 +76,17 @@ const CountLabel = ({ label, count }: { label: string; count: number }) => {
 
 export const NewExperimentCard = () => {
   const router = useRouter();
+  const selectedOrgId = useAppStore((s) => s.selectedOrgId);
   const createMutation = api.experiments.create.useMutation();
   const [createExperiment, isLoading] = useHandledAsyncCallback(async () => {
-    const newExperiment = await createMutation.mutateAsync({ label: "New Experiment" });
-    await router.push({ pathname: "/experiments/[id]", query: { id: newExperiment.id } });
-  }, [createMutation, router]);
+    const newExperiment = await createMutation.mutateAsync({
+      organizationId: selectedOrgId ?? "",
+    });
+    await router.push({
+      pathname: "/experiments/[id]",
+      query: { id: newExperiment.id },
+    });
+  }, [createMutation, router, selectedOrgId]);
 
   return (
     <AspectRatio ratio={1.2} w="full">

app/src/components/experiments/ExperimentHeaderButtons/useOnForkButtonPressed.tsx

@@ -3,18 +3,23 @@ import { api } from "~/utils/api";
 import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
 import { signIn, useSession } from "next-auth/react";
 import { useRouter } from "next/router";
+import { useAppStore } from "~/state/store";
 
 export const useOnForkButtonPressed = () => {
   const router = useRouter();
 
   const user = useSession().data;
   const experiment = useExperiment();
+  const selectedOrgId = useAppStore((state) => state.selectedOrgId);
 
   const forkMutation = api.experiments.fork.useMutation();
 
   const [onFork, isForking] = useHandledAsyncCallback(async () => {
-    if (!experiment.data?.id) return;
-    const forkedExperimentId = await forkMutation.mutateAsync({ id: experiment.data.id });
+    if (!experiment.data?.id || !selectedOrgId) return;
+    const forkedExperimentId = await forkMutation.mutateAsync({
+      id: experiment.data.id,
+      organizationId: selectedOrgId,
+    });
     await router.push({ pathname: "/experiments/[id]", query: { id: forkedExperimentId } });
   }, [forkMutation, experiment.data?.id, router]);
 

app/src/components/nav/AppShell.tsx

@@ -7,48 +7,22 @@ import {
   Image,
   Text,
   Box,
-  type BoxProps,
   Link as ChakraLink,
   Flex,
 } from "@chakra-ui/react";
 import Head from "next/head";
-import Link, { type LinkProps } from "next/link";
-import { BsGithub, BsPersonCircle } from "react-icons/bs";
-import { useRouter } from "next/router";
-import { type IconType } from "react-icons";
+import Link from "next/link";
+import { BsGearFill, BsGithub, BsPersonCircle } from "react-icons/bs";
+import { IoStatsChartOutline } from "react-icons/io5";
 import { RiDatabase2Line, RiFlaskLine } from "react-icons/ri";
 import { signIn, useSession } from "next-auth/react";
 import UserMenu from "./UserMenu";
 import { env } from "~/env.mjs";
+import ProjectMenu from "./ProjectMenu";
+import NavSidebarOption from "./NavSidebarOption";
+import IconLink from "./IconLink";
 
-type IconLinkProps = BoxProps & LinkProps & { label?: string; icon: IconType; href: string };
-
-const IconLink = ({ icon, label, href, color, ...props }: IconLinkProps) => {
-  const router = useRouter();
-  const isActive = href && router.pathname.startsWith(href);
-  return (
-    <Link href={href} style={{ width: "100%" }}>
-      <HStack
-        w="full"
-        p={4}
-        color={color}
-        as={ChakraLink}
-        bgColor={isActive ? "gray.200" : "transparent"}
-        _hover={{ bgColor: "gray.300", textDecoration: "none" }}
-        justifyContent="start"
-        cursor="pointer"
-        {...props}
-      >
-        <Icon as={icon} boxSize={6} mr={2} />
-        <Text fontWeight="bold" fontSize="sm">
-          {label}
-        </Text>
-      </HStack>
-    </Link>
-  );
-};
-
-const Divider = () => <Box h="1px" bgColor="gray.200" />;
+const Divider = () => <Box h="1px" bgColor="gray.300" w="full" />;
 
 const NavSidebar = () => {
   const user = useSession().data;
@@ -56,22 +30,31 @@ const NavSidebar = () => {
   return (
     <VStack
       align="stretch"
-      bgColor="gray.100"
+      bgColor="gray.50"
       py={2}
+      px={2}
       pb={0}
       height="100%"
-      w={{ base: "56px", md: "200px" }}
+      w={{ base: "56px", md: "240px" }}
       overflow="hidden"
+      borderRightWidth={1}
+      borderColor="gray.300"
     >
-      <HStack as={Link} href="/" _hover={{ textDecoration: "none" }} spacing={0} px={4} py={2}>
+      <HStack as={Link} href="/" _hover={{ textDecoration: "none" }} spacing={0} px={2} py={2}>
         <Image src="/logo.svg" alt="" boxSize={6} mr={4} />
         <Heading size="md" fontFamily="inconsolata, monospace">
           OpenPipe
         </Heading>
       </HStack>
-      <VStack spacing={0} align="flex-start" overflowY="auto" overflowX="hidden" flex={1}>
+      <Divider />
+      <VStack align="flex-start" overflowY="auto" overflowX="hidden" flex={1}>
         {user != null && (
           <>
+            <ProjectMenu />
+            <Divider />
+            {env.NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS && (
+              <IconLink icon={IoStatsChartOutline} label="Logged Calls" href="/logged-calls" beta />
+            )}
             <IconLink icon={RiFlaskLine} label="Experiments" href="/experiments" />
             {env.NEXT_PUBLIC_SHOW_DATA && (
               <IconLink icon={RiDatabase2Line} label="Data" href="/data" />
@@ -79,29 +62,39 @@ const NavSidebar = () => {
           </>
         )}
         {user === null && (
-          <HStack
-            w="full"
-            p={4}
-            as={ChakraLink}
-            _hover={{ bgColor: "gray.300", textDecoration: "none" }}
-            justifyContent="start"
-            cursor="pointer"
-            onClick={() => {
-              signIn("github").catch(console.error);
-            }}
-          >
-            <Icon as={BsPersonCircle} boxSize={6} mr={2} />
-            <Text fontWeight="bold" fontSize="sm">
-              Sign In
-            </Text>
-          </HStack>
+          <NavSidebarOption>
+            <HStack
+              w="full"
+              p={4}
+              as={ChakraLink}
+              justifyContent="start"
+              onClick={() => {
+                signIn("github").catch(console.error);
+              }}
+            >
+              <Icon as={BsPersonCircle} boxSize={6} mr={2} />
+              <Text fontWeight="bold" fontSize="sm">
+                Sign In
+              </Text>
+            </HStack>
+          </NavSidebarOption>
         )}
       </VStack>
-      {user ? (
-        <UserMenu user={user} borderColor={"gray.200"} borderTopWidth={1} borderBottomWidth={1} />
-      ) : (
-        <Divider />
-      )}
+      <VStack w="full" alignItems="flex-start" spacing={0}>
+        <Text
+          pl={2}
+          pb={2}
+          fontSize="xs"
+          fontWeight="bold"
+          color="gray.500"
+          display={{ base: "none", md: "flex" }}
+        >
+          CONFIGURATION
+        </Text>
+        <IconLink icon={BsGearFill} label="Project Settings" href="/project/settings" />
+      </VStack>
+      {user && <UserMenu user={user} borderColor={"gray.200"} />}
+      <Divider />
       <VStack spacing={0} align="center">
         <ChakraLink
           href="https://github.com/openpipe/openpipe"
@@ -117,7 +110,15 @@ const NavSidebar = () => {
   );
 };
 
-export default function AppShell(props: { children: React.ReactNode; title?: string }) {
+export default function AppShell({
+  children,
+  title,
+  requireAuth,
+}: {
+  children: React.ReactNode;
+  title?: string;
+  requireAuth?: boolean;
+}) {
   const [vh, setVh] = useState("100vh"); // Default height to prevent flicker on initial render
 
   useEffect(() => {
@@ -137,14 +138,23 @@ export default function AppShell(props: { children: React.ReactNode; title?: str
     };
   }, []);
 
+  const user = useSession().data;
+  const authLoading = useSession().status === "loading";
+
+  useEffect(() => {
+    if (requireAuth && user === null && !authLoading) {
+      signIn("github").catch(console.error);
+    }
+  }, [requireAuth, user, authLoading]);
+
   return (
     <Flex h={vh} w="100vw">
       <Head>
-        <title>{props.title ? `${props.title} | OpenPipe` : "OpenPipe"}</title>
+        <title>{title ? `${title} | OpenPipe` : "OpenPipe"}</title>
       </Head>
       <NavSidebar />
       <Box h="100%" flex={1} overflowY="auto">
-        {props.children}
+        {children}
       </Box>
     </Flex>
   );

app/src/components/nav/IconLink.tsx

@@ -0,0 +1,31 @@
+import { Icon, HStack, Text, type BoxProps } from "@chakra-ui/react";
+import Link, { type LinkProps } from "next/link";
+import { type IconType } from "react-icons";
+import NavSidebarOption from "./NavSidebarOption";
+
+type IconLinkProps = BoxProps &
+  LinkProps & { label?: string; icon: IconType; href: string; beta?: boolean };
+
+const IconLink = ({ icon, label, href, color, beta, ...props }: IconLinkProps) => {
+  return (
+    <Link href={href} style={{ width: "100%" }}>
+      <NavSidebarOption activeHrefPattern={href}>
+        <HStack w="full" justifyContent="space-between" p={2} color={color} {...props}>
+          <HStack w="full" justifyContent="start">
+            <Icon as={icon} boxSize={6} mr={2} />
+            <Text fontSize="sm" display={{ base: "none", md: "block" }}>
+              {label}
+            </Text>
+          </HStack>
+          {beta && (
+            <Text fontSize="xs" ml={2} fontWeight="bold" color="orange.400">
+              BETA
+            </Text>
+          )}
+        </HStack>
+      </NavSidebarOption>
+    </Link>
+  );
+};
+
+export default IconLink;

app/src/components/nav/NavSidebarOption.tsx

@@ -0,0 +1,27 @@
+import { Box, type BoxProps } from "@chakra-ui/react";
+import { useRouter } from "next/router";
+
+const NavSidebarOption = ({
+  activeHrefPattern,
+  disableHoverEffect,
+  ...props
+}: { activeHrefPattern?: string; disableHoverEffect?: boolean } & BoxProps) => {
+  const router = useRouter();
+  const isActive = activeHrefPattern && router.pathname.startsWith(activeHrefPattern);
+  return (
+    <Box
+      w="full"
+      fontWeight={isActive ? "bold" : "500"}
+      bgColor={isActive ? "gray.200" : "transparent"}
+      _hover={disableHoverEffect ? undefined : { bgColor: "gray.200", textDecoration: "none" }}
+      justifyContent="start"
+      cursor="pointer"
+      borderRadius={4}
+      {...props}
+    >
+      {props.children}
+    </Box>
+  );
+};
+
+export default NavSidebarOption;

app/src/components/nav/PageHeaderContainer.tsx

@@ -0,0 +1,19 @@
+import { Flex, type FlexProps } from "@chakra-ui/react";
+
+const PageHeaderContainer = (props: FlexProps) => {
+  return (
+    <Flex
+      px={8}
+      py={2}
+      minH={16}
+      w="full"
+      direction={{ base: "column", sm: "row" }}
+      alignItems={{ base: "flex-start", sm: "center" }}
+      justifyContent="space-between"
+      fontWeight="500"
+      {...props}
+    />
+  );
+};
+
+export default PageHeaderContainer;

app/src/components/nav/ProjectBreadcrumbContents.tsx

@@ -0,0 +1,28 @@
+import { HStack, Flex, Text } from "@chakra-ui/react";
+import { useSelectedOrg } from "~/utils/hooks";
+
+// Have to export only contents here instead of full BreadcrumbItem because Chakra doesn't
+// recognize a BreadcrumbItem exported with this component as a valid child of Breadcrumb.
+export default function ProjectBreadcrumbContents({ orgName = "" }: { orgName?: string }) {
+  const { data: selectedOrg } = useSelectedOrg();
+
+  orgName = orgName || selectedOrg?.name || "";
+
+  return (
+    <HStack w="full">
+      <Flex
+        p={1}
+        borderRadius={4}
+        backgroundColor="orange.100"
+        boxSize={6}
+        alignItems="center"
+        justifyContent="center"
+      >
+        <Text>{orgName[0]?.toUpperCase()}</Text>
+      </Flex>
+      <Text display={{ base: "none", md: "block" }} py={1}>
+        {orgName}
+      </Text>
+    </HStack>
+  );
+}

app/src/components/nav/ProjectMenu.tsx

@@ -0,0 +1,178 @@
+import {
+  HStack,
+  VStack,
+  Text,
+  Popover,
+  PopoverTrigger,
+  PopoverContent,
+  Flex,
+  IconButton,
+  Icon,
+  Divider,
+  Button,
+  useDisclosure,
+  Spinner,
+} from "@chakra-ui/react";
+import React, { useEffect, useState } from "react";
+import Link from "next/link";
+import { AiFillCaretDown } from "react-icons/ai";
+import { BsGear, BsPlus } from "react-icons/bs";
+import { type Organization } from "@prisma/client";
+
+import { useAppStore } from "~/state/store";
+import { api } from "~/utils/api";
+import NavSidebarOption from "./NavSidebarOption";
+import { useHandledAsyncCallback, useSelectedOrg } from "~/utils/hooks";
+import { useRouter } from "next/router";
+
+export default function ProjectMenu() {
+  const router = useRouter();
+  const isActive = router.pathname.startsWith("/home");
+  const utils = api.useContext();
+
+  const selectedOrgId = useAppStore((s) => s.selectedOrgId);
+  const setSelectedOrgId = useAppStore((s) => s.setSelectedOrgId);
+
+  const { data: orgs } = api.organizations.list.useQuery();
+
+  useEffect(() => {
+    if (orgs && orgs[0] && (!selectedOrgId || !orgs.find((org) => org.id === selectedOrgId))) {
+      setSelectedOrgId(orgs[0].id);
+    }
+  }, [selectedOrgId, setSelectedOrgId, orgs]);
+
+  const { data: selectedOrg } = useSelectedOrg();
+
+  const popover = useDisclosure();
+
+  const createMutation = api.organizations.create.useMutation();
+  const [createProject, isLoading] = useHandledAsyncCallback(async () => {
+    const newOrg = await createMutation.mutateAsync({ name: "New Project" });
+    await utils.organizations.list.invalidate();
+    setSelectedOrgId(newOrg.id);
+    await router.push({ pathname: "/project/settings" });
+  }, [createMutation, router]);
+
+  return (
+    <VStack w="full" alignItems="flex-start" spacing={0}>
+      <Text
+        pl={2}
+        pb={2}
+        fontSize="xs"
+        fontWeight="bold"
+        color="gray.500"
+        display={{ base: "none", md: "flex" }}
+      >
+        PROJECT
+      </Text>
+      <NavSidebarOption>
+        <Popover
+          placement="bottom-start"
+          isOpen={popover.isOpen}
+          onClose={popover.onClose}
+          closeOnBlur
+        >
+          <PopoverTrigger>
+            <HStack w="full" justifyContent="space-between" onClick={popover.onToggle}>
+              <Flex
+                p={1}
+                borderRadius={4}
+                backgroundColor="orange.100"
+                minW={{ base: 10, md: 8 }}
+                minH={{ base: 10, md: 8 }}
+                m={{ base: 0, md: 1 }}
+                alignItems="center"
+                justifyContent="center"
+                // onClick={sidebarExpanded ? undefined : openMenu}
+              >
+                <Text>{selectedOrg?.name[0]?.toUpperCase()}</Text>
+              </Flex>
+              <Text fontSize="sm" display={{ base: "none", md: "block" }} py={1}>
+                {selectedOrg?.name}
+              </Text>
+              <Icon as={AiFillCaretDown} boxSize={3} size="xs" color="gray.500" mr={2} />
+            </HStack>
+          </PopoverTrigger>
+          <PopoverContent
+            _focusVisible={{ boxShadow: "unset" }}
+            minW={0}
+            borderColor="blue.400"
+            w="full"
+          >
+            <VStack alignItems="flex-start" spacing={2} py={4} px={2}>
+              <Text color="gray.500" fontSize="xs" fontWeight="bold" pb={1}>
+                PROJECTS
+              </Text>
+              <Divider />
+              <VStack spacing={0} w="full">
+                {orgs?.map((org) => (
+                  <ProjectOption
+                    key={org.id}
+                    org={org}
+                    isActive={org.id === selectedOrgId}
+                    onClose={popover.onClose}
+                  />
+                ))}
+              </VStack>
+              <HStack
+                as={Button}
+                variant="ghost"
+                colorScheme="blue"
+                color="blue.400"
+                pr={8}
+                w="full"
+                onClick={createProject}
+              >
+                <Icon as={isLoading ? Spinner : BsPlus} boxSize={6} />
+                <Text>New project</Text>
+              </HStack>
+            </VStack>
+          </PopoverContent>
+        </Popover>
+      </NavSidebarOption>
+    </VStack>
+  );
+}
+
+const ProjectOption = ({
+  org,
+  isActive,
+  onClose,
+}: {
+  org: Organization;
+  isActive: boolean;
+  onClose: () => void;
+}) => {
+  const setSelectedOrgId = useAppStore((s) => s.setSelectedOrgId);
+  const [gearHovered, setGearHovered] = useState(false);
+  return (
+    <HStack
+      as={Link}
+      href="/experiments"
+      onClick={() => {
+        setSelectedOrgId(org.id);
+        onClose();
+      }}
+      w="full"
+      justifyContent="space-between"
+      bgColor={isActive ? "gray.100" : "transparent"}
+      _hover={gearHovered ? undefined : { bgColor: "gray.200", textDecoration: "none" }}
+      p={2}
+    >
+      <Text>{org.name}</Text>
+      <IconButton
+        as={Link}
+        href="/project/settings"
+        aria-label={`Open ${org.name} settings`}
+        icon={<Icon as={BsGear} boxSize={5} strokeWidth={0.5} color="gray.500" />}
+        variant="ghost"
+        size="xs"
+        p={0}
+        onMouseEnter={() => setGearHovered(true)}
+        onMouseLeave={() => setGearHovered(false)}
+        _hover={{ bgColor: isActive ? "gray.300" : "gray.100", transitionDelay: 0 }}
+        borderRadius={4}
+      />
+    </HStack>
+  );
+};

app/src/components/nav/UserMenu.tsx

@@ -8,16 +8,15 @@ import {
   PopoverTrigger,
   PopoverContent,
   Link,
-  useColorMode,
   type StackProps,
+  Box,
 } from "@chakra-ui/react";
 import { type Session } from "next-auth";
 import { signOut } from "next-auth/react";
 import { BsBoxArrowRight, BsChevronRight, BsPersonCircle } from "react-icons/bs";
+import NavSidebarOption from "./NavSidebarOption";
 
 export default function UserMenu({ user, ...rest }: { user: Session } & StackProps) {
-  const { colorMode } = useColorMode();
-
   const profileImage = user.user.image ? (
     <Image src={user.user.image} alt="profile picture" boxSize={8} borderRadius="50%" />
   ) : (
@@ -28,28 +27,28 @@ export default function UserMenu({ user, ...rest }: { user: Session } & StackPro
     <>
       <Popover placement="right">
         <PopoverTrigger>
-          <HStack
-            // Weird values to make mobile look right; can clean up when we make the sidebar disappear on mobile
-            px={3}
-            spacing={3}
-            py={2}
-            {...rest}
-            cursor="pointer"
-            _hover={{
-              bgColor: colorMode === "light" ? "gray.200" : "gray.700",
-            }}
-          >
-            {profileImage}
-            <VStack spacing={0} align="start" flex={1} flexShrink={1}>
-              <Text fontWeight="bold" fontSize="sm">
-                {user.user.name}
-              </Text>
-              <Text color="gray.500" fontSize="xs">
-                {user.user.email}
-              </Text>
-            </VStack>
-            <Icon as={BsChevronRight} boxSize={4} color="gray.500" />
-          </HStack>
+          <Box>
+            <NavSidebarOption>
+              <HStack
+                // Weird values to make mobile look right; can clean up when we make the sidebar disappear on mobile
+                py={2}
+                px={1}
+                spacing={3}
+                {...rest}
+              >
+                {profileImage}
+                <VStack spacing={0} align="start" flex={1} flexShrink={1}>
+                  <Text fontWeight="bold" fontSize="sm">
+                    {user.user.name}
+                  </Text>
+                  <Text color="gray.500" fontSize="xs">
+                    {/* {user.user.email} */}
+                  </Text>
+                </VStack>
+                <Icon as={BsChevronRight} boxSize={4} color="gray.500" />
+              </HStack>
+            </NavSidebarOption>
+          </Box>
         </PopoverTrigger>
         <PopoverContent _focusVisible={{ boxShadow: "unset", outline: "unset" }} maxW="200px">
           <VStack align="stretch" spacing={0}>

app/src/components/projectSettings/DeleteProjectDialog.tsx

@@ -0,0 +1,89 @@
+import {
+  Button,
+  AlertDialog,
+  AlertDialogBody,
+  AlertDialogFooter,
+  AlertDialogHeader,
+  AlertDialogContent,
+  AlertDialogOverlay,
+  Input,
+  Text,
+  VStack,
+  Box,
+  Spinner,
+} from "@chakra-ui/react";
+
+import { useRouter } from "next/router";
+import { useRef, useState } from "react";
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback, useSelectedOrg } from "~/utils/hooks";
+
+export const DeleteProjectDialog = ({
+  isOpen,
+  onClose,
+}: {
+  isOpen: boolean;
+  onClose: () => void;
+}) => {
+  const selectedOrg = useSelectedOrg();
+  const deleteMutation = api.organizations.delete.useMutation();
+  const utils = api.useContext();
+  const router = useRouter();
+
+  const cancelRef = useRef<HTMLButtonElement>(null);
+
+  const [onDeleteConfirm, isDeleting] = useHandledAsyncCallback(async () => {
+    if (!selectedOrg.data?.id) return;
+    await deleteMutation.mutateAsync({ id: selectedOrg.data.id });
+    await utils.organizations.list.invalidate();
+    await router.push({ pathname: "/experiments" });
+    onClose();
+  }, [deleteMutation, selectedOrg, router]);
+
+  const [nameToDelete, setNameToDelete] = useState("");
+
+  return (
+    <AlertDialog isOpen={isOpen} leastDestructiveRef={cancelRef} onClose={onClose}>
+      <AlertDialogOverlay>
+        <AlertDialogContent>
+          <AlertDialogHeader fontSize="lg" fontWeight="bold">
+            Delete Project
+          </AlertDialogHeader>
+
+          <AlertDialogBody>
+            <VStack spacing={4} alignItems="flex-start">
+              <Text>
+                If you delete this project all the associated data and experiments will be deleted
+                as well. If you are sure that you want to delete this project, please type the name
+                of the project below.
+              </Text>
+              <Box bgColor="orange.100" w="full" p={2} borderRadius={4}>
+                <Text fontFamily="inconsolata">{selectedOrg.data?.name}</Text>
+              </Box>
+              <Input
+                placeholder={selectedOrg.data?.name}
+                value={nameToDelete}
+                onChange={(e) => setNameToDelete(e.target.value)}
+              />
+            </VStack>
+          </AlertDialogBody>
+
+          <AlertDialogFooter>
+            <Button ref={cancelRef} onClick={onClose}>
+              Cancel
+            </Button>
+            <Button
+              colorScheme="red"
+              onClick={onDeleteConfirm}
+              ml={3}
+              isDisabled={nameToDelete !== selectedOrg.data?.name}
+              w={20}
+            >
+              {isDeleting ? <Spinner /> : "Delete"}
+            </Button>
+          </AlertDialogFooter>
+        </AlertDialogContent>
+      </AlertDialogOverlay>
+    </AlertDialog>
+  );
+};

app/src/env.mjs

@@ -20,6 +20,7 @@ export const env = createEnv({
     REPLICATE_API_TOKEN: z.string().default("placeholder"),
     ANTHROPIC_API_KEY: z.string().default("placeholder"),
     SENTRY_AUTH_TOKEN: z.string().optional(),
+    OPENPIPE_API_KEY: z.string().optional(),
   },
 
   /**
@@ -33,6 +34,7 @@ export const env = createEnv({
     NEXT_PUBLIC_HOST: z.string().url().default("http://localhost:3000"),
     NEXT_PUBLIC_SENTRY_DSN: z.string().optional(),
     NEXT_PUBLIC_SHOW_DATA: z.string().optional(),
+    NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS: z.string().optional(),
   },
 
   /**
@@ -54,6 +56,8 @@ export const env = createEnv({
     ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY,
     NEXT_PUBLIC_SENTRY_DSN: process.env.NEXT_PUBLIC_SENTRY_DSN,
     SENTRY_AUTH_TOKEN: process.env.SENTRY_AUTH_TOKEN,
+    OPENPIPE_API_KEY: process.env.OPENPIPE_API_KEY,
+    NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS: process.env.NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS,
   },
   /**
    * Run `build` or `dev` with `SKIP_ENV_VALIDATION` to skip env validation.

app/src/pages/api/[...trpc].ts

@@ -0,0 +1,22 @@
+import { type NextApiRequest, type NextApiResponse } from "next";
+import cors from "nextjs-cors";
+import { createOpenApiNextHandler } from "trpc-openapi";
+import { createProcedureCache } from "trpc-openapi/dist/adapters/node-http/procedures";
+import { appRouter } from "~/server/api/root.router";
+import { createTRPCContext } from "~/server/api/trpc";
+
+const openApiHandler = createOpenApiNextHandler({
+  router: appRouter,
+  createContext: createTRPCContext,
+});
+
+const cache = createProcedureCache(appRouter);
+
+const handler = async (req: NextApiRequest, res: NextApiResponse) => {
+  // Setup CORS
+  await cors(req, res);
+
+  return openApiHandler(req, res);
+};
+
+export default handler;

app/src/pages/api/openapi.json.ts

@@ -0,0 +1,16 @@
+import { type NextApiRequest, type NextApiResponse } from "next";
+import { generateOpenApiDocument } from "trpc-openapi";
+import { appRouter } from "~/server/api/root.router";
+
+export const openApiDocument = generateOpenApiDocument(appRouter, {
+  title: "OpenPipe API",
+  description: "The public API for reporting API calls to OpenPipe",
+  version: "0.1.0",
+  baseUrl: "https://app.openpipe.ai/api",
+});
+// Respond with our OpenAPI schema
+const hander = (req: NextApiRequest, res: NextApiResponse) => {
+  res.status(200).send(openApiDocument);
+};
+
+export default hander;

app/src/pages/data/[id].tsx

@@ -18,6 +18,8 @@ import { api } from "~/utils/api";
 import { useDataset, useHandledAsyncCallback } from "~/utils/hooks";
 import DatasetEntriesTable from "~/components/datasets/DatasetEntriesTable";
 import { DatasetHeaderButtons } from "~/components/datasets/DatasetHeaderButtons/DatasetHeaderButtons";
+import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
+import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
 
 export default function Dataset() {
   const router = useRouter();
@@ -55,15 +57,11 @@ export default function Dataset() {
   return (
     <AppShell title={dataset.data?.name}>
       <VStack h="full">
-        <Flex
-          pl={4}
-          pr={8}
-          py={2}
-          w="full"
-          direction={{ base: "column", sm: "row" }}
-          alignItems={{ base: "flex-start", sm: "center" }}
-        >
-          <Breadcrumb flex={1} mt={1}>
+        <PageHeaderContainer>
+          <Breadcrumb>
+            <BreadcrumbItem>
+              <ProjectBreadcrumbContents orgName={dataset.data?.organization?.name} />
+            </BreadcrumbItem>
             <BreadcrumbItem>
               <Link href="/data">
                 <Flex alignItems="center" _hover={{ textDecoration: "underline" }}>
@@ -89,8 +87,8 @@ export default function Dataset() {
             </BreadcrumbItem>
           </Breadcrumb>
           <DatasetHeaderButtons />
-        </Flex>
-        <Box w="full" overflowX="auto" flex={1} pl={4} pr={8} pt={8} pb={16}>
+        </PageHeaderContainer>
+        <Box w="full" overflowX="auto" flex={1} px={8} pt={8} pb={16}>
           {datasetId && <DatasetEntriesTable />}
         </Box>
       </VStack>

app/src/pages/data/index.tsx

@@ -1,83 +1,49 @@
-import {
-  SimpleGrid,
-  Icon,
-  VStack,
-  Breadcrumb,
-  BreadcrumbItem,
-  Flex,
-  Center,
-  Text,
-  Link,
-  HStack,
-} from "@chakra-ui/react";
+import { SimpleGrid, Icon, Breadcrumb, BreadcrumbItem, Flex } from "@chakra-ui/react";
 import AppShell from "~/components/nav/AppShell";
-import { api } from "~/utils/api";
-import { signIn, useSession } from "next-auth/react";
 import { RiDatabase2Line } from "react-icons/ri";
 import {
   DatasetCard,
   DatasetCardSkeleton,
   NewDatasetCard,
 } from "~/components/datasets/DatasetCard";
+import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
+import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
+import { useDatasets } from "~/utils/hooks";
 
 export default function DatasetsPage() {
-  const datasets = api.datasets.list.useQuery();
-
-  const user = useSession().data;
-  const authLoading = useSession().status === "loading";
-
-  if (user === null || authLoading) {
-    return (
-      <AppShell title="Data">
-        <Center h="100%">
-          {!authLoading && (
-            <Text>
-              <Link
-                onClick={() => {
-                  signIn("github").catch(console.error);
-                }}
-                textDecor="underline"
-              >
-                Sign in
-              </Link>{" "}
-              to view or create new datasets!
-            </Text>
-          )}
-        </Center>
-      </AppShell>
-    );
-  }
+  const datasets = useDatasets();
 
   return (
-    <AppShell title="Data">
-      <VStack alignItems={"flex-start"} px={4} py={2}>
-        <HStack minH={8} align="center" pt={2}>
-          <Breadcrumb flex={1}>
-            <BreadcrumbItem>
-              <Flex alignItems="center">
-                <Icon as={RiDatabase2Line} boxSize={4} mr={2} /> Datasets
-              </Flex>
-            </BreadcrumbItem>
-          </Breadcrumb>
-        </HStack>
-        <SimpleGrid w="full" columns={{ base: 1, md: 2, lg: 3, xl: 4 }} spacing={8} p="4">
-          <NewDatasetCard />
-          {datasets.data && !datasets.isLoading ? (
-            datasets?.data?.map((dataset) => (
-              <DatasetCard
-                key={dataset.id}
-                dataset={{ ...dataset, numEntries: dataset._count.datasetEntries }}
-              />
-            ))
-          ) : (
-            <>
-              <DatasetCardSkeleton />
-              <DatasetCardSkeleton />
-              <DatasetCardSkeleton />
-            </>
-          )}
-        </SimpleGrid>
-      </VStack>
+    <AppShell title="Data" requireAuth>
+      <PageHeaderContainer>
+        <Breadcrumb>
+          <BreadcrumbItem>
+            <ProjectBreadcrumbContents />
+          </BreadcrumbItem>
+          <BreadcrumbItem minH={8}>
+            <Flex alignItems="center">
+              <Icon as={RiDatabase2Line} boxSize={4} mr={2} /> Datasets
+            </Flex>
+          </BreadcrumbItem>
+        </Breadcrumb>
+      </PageHeaderContainer>
+      <SimpleGrid w="full" columns={{ base: 1, md: 2, lg: 3, xl: 4 }} spacing={8} py={4} px={8}>
+        <NewDatasetCard />
+        {datasets.data && !datasets.isLoading ? (
+          datasets?.data?.map((dataset) => (
+            <DatasetCard
+              key={dataset.id}
+              dataset={{ ...dataset, numEntries: dataset._count.datasetEntries }}
+            />
+          ))
+        ) : (
+          <>
+            <DatasetCardSkeleton />
+            <DatasetCardSkeleton />
+            <DatasetCardSkeleton />
+          </>
+        )}
+      </SimpleGrid>
     </AppShell>
   );
 }

app/src/pages/experiments/[id].tsx

@@ -23,6 +23,8 @@ import { useAppStore } from "~/state/store";
 import { useSyncVariantEditor } from "~/state/sync";
 import { ExperimentHeaderButtons } from "~/components/experiments/ExperimentHeaderButtons/ExperimentHeaderButtons";
 import Head from "next/head";
+import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
+import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
 
 // TODO: import less to fix deployment with server side props
 // export const getServerSideProps = async (context: GetServerSidePropsContext<{ id: string }>) => {
@@ -104,14 +106,11 @@ export default function Experiment() {
       )}
       <AppShell title={experiment.data?.label}>
         <VStack h="full">
-          <Flex
-            px={4}
-            py={2}
-            w="full"
-            direction={{ base: "column", sm: "row" }}
-            alignItems={{ base: "flex-start", sm: "center" }}
-          >
-            <Breadcrumb flex={1}>
+          <PageHeaderContainer>
+            <Breadcrumb>
+              <BreadcrumbItem>
+                <ProjectBreadcrumbContents orgName={experiment.data?.organization?.name} />
+              </BreadcrumbItem>
               <BreadcrumbItem>
                 <Link href="/experiments">
                   <Flex alignItems="center" _hover={{ textDecoration: "underline" }}>
@@ -143,7 +142,7 @@ export default function Experiment() {
               </BreadcrumbItem>
             </Breadcrumb>
             <ExperimentHeaderButtons />
-          </Flex>
+          </PageHeaderContainer>
           <ExperimentSettingsDrawer />
           <Box w="100%" overflowX="auto" flex={1}>
             <OutputsTable experimentId={router.query.id as string | undefined} />

app/src/pages/experiments/index.tsx

@@ -1,78 +1,44 @@
-import {
-  SimpleGrid,
-  Icon,
-  VStack,
-  Breadcrumb,
-  BreadcrumbItem,
-  Flex,
-  Center,
-  Text,
-  Link,
-  HStack,
-} from "@chakra-ui/react";
+import { SimpleGrid, Icon, Breadcrumb, BreadcrumbItem, Flex } from "@chakra-ui/react";
 import { RiFlaskLine } from "react-icons/ri";
 import AppShell from "~/components/nav/AppShell";
-import { api } from "~/utils/api";
 import {
   ExperimentCard,
   ExperimentCardSkeleton,
   NewExperimentCard,
 } from "~/components/experiments/ExperimentCard";
-import { signIn, useSession } from "next-auth/react";
+import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
+import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
+import { useExperiments } from "~/utils/hooks";
 
 export default function ExperimentsPage() {
-  const experiments = api.experiments.list.useQuery();
-
-  const user = useSession().data;
-  const authLoading = useSession().status === "loading";
-
-  if (user === null || authLoading) {
-    return (
-      <AppShell title="Experiments">
-        <Center h="100%">
-          {!authLoading && (
-            <Text>
-              <Link
-                onClick={() => {
-                  signIn("github").catch(console.error);
-                }}
-                textDecor="underline"
-              >
-                Sign in
-              </Link>{" "}
-              to view or create new experiments!
-            </Text>
-          )}
-        </Center>
-      </AppShell>
-    );
-  }
+  const experiments = useExperiments();
 
   return (
-    <AppShell title="Experiments">
-      <VStack alignItems={"flex-start"} px={4} py={2}>
-        <HStack minH={8} align="center" pt={2}>
-          <Breadcrumb flex={1}>
-            <BreadcrumbItem>
-              <Flex alignItems="center">
-                <Icon as={RiFlaskLine} boxSize={4} mr={2} /> Experiments
-              </Flex>
-            </BreadcrumbItem>
-          </Breadcrumb>
-        </HStack>
-        <SimpleGrid w="full" columns={{ base: 1, md: 2, lg: 3, xl: 4 }} spacing={8} p="4">
-          <NewExperimentCard />
-          {experiments.data && !experiments.isLoading ? (
-            experiments?.data?.map((exp) => <ExperimentCard key={exp.id} exp={exp} />)
-          ) : (
-            <>
-              <ExperimentCardSkeleton />
-              <ExperimentCardSkeleton />
-              <ExperimentCardSkeleton />
-            </>
-          )}
-        </SimpleGrid>
-      </VStack>
+    <AppShell title="Experiments" requireAuth>
+      <PageHeaderContainer>
+        <Breadcrumb>
+          <BreadcrumbItem>
+            <ProjectBreadcrumbContents />
+          </BreadcrumbItem>
+          <BreadcrumbItem minH={8}>
+            <Flex alignItems="center">
+              <Icon as={RiFlaskLine} boxSize={4} mr={2} /> Experiments
+            </Flex>
+          </BreadcrumbItem>
+        </Breadcrumb>
+      </PageHeaderContainer>
+      <SimpleGrid w="full" columns={{ base: 1, md: 2, lg: 3, xl: 4 }} spacing={8} py="4" px={8}>
+        <NewExperimentCard />
+        {experiments.data && !experiments.isLoading ? (
+          experiments?.data?.map((exp) => <ExperimentCard key={exp.id} exp={exp} />)
+        ) : (
+          <>
+            <ExperimentCardSkeleton />
+            <ExperimentCardSkeleton />
+            <ExperimentCardSkeleton />
+          </>
+        )}
+      </SimpleGrid>
     </AppShell>
   );
 }

app/src/pages/logged-calls/index.tsx

@@ -0,0 +1,181 @@
+import {
+  Heading,
+  Text,
+  Stat,
+  StatLabel,
+  StatNumber,
+  VStack,
+  HStack,
+  Card,
+  CardBody,
+  CardHeader,
+  Icon,
+  Table,
+  Tbody,
+  Tr,
+  Td,
+  Divider,
+  Breadcrumb,
+  BreadcrumbItem,
+} from "@chakra-ui/react";
+import {
+  LineChart,
+  Line,
+  XAxis,
+  YAxis,
+  CartesianGrid,
+  Tooltip,
+  Legend,
+  ResponsiveContainer,
+} from "recharts";
+import { Ban, DollarSign, Hash } from "lucide-react";
+import { useMemo } from "react";
+
+import AppShell from "~/components/nav/AppShell";
+import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
+import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
+import { useSelectedOrg } from "~/utils/hooks";
+import dayjs from "~/utils/dayjs";
+import { api } from "~/utils/api";
+import LoggedCallTable from "~/components/dashboard/LoggedCallTable";
+
+export default function LoggedCalls() {
+  const { data: selectedOrg } = useSelectedOrg();
+
+  const stats = api.dashboard.stats.useQuery(
+    { organizationId: selectedOrg?.id ?? "" },
+    { enabled: !!selectedOrg },
+  );
+
+  const data = useMemo(() => {
+    return (
+      stats.data?.periods.map(({ period, numQueries, totalCost }) => ({
+        period,
+        Requests: numQueries,
+        "Total Spent (USD)": parseFloat(totalCost.toString()),
+      })) || []
+    );
+  }, [stats.data]);
+
+  return (
+    <AppShell requireAuth>
+      <PageHeaderContainer>
+        <Breadcrumb>
+          <BreadcrumbItem>
+            <ProjectBreadcrumbContents />
+          </BreadcrumbItem>
+          <BreadcrumbItem isCurrentPage>
+            <Text>Logged Calls</Text>
+          </BreadcrumbItem>
+        </Breadcrumb>
+      </PageHeaderContainer>
+      <VStack px={8} pt={4} alignItems="flex-start" spacing={4}>
+        <Text fontSize="2xl" fontWeight="bold">
+          {selectedOrg?.name}
+        </Text>
+        <Divider />
+        <VStack margin="auto" spacing={4} align="stretch" w="full">
+          <HStack gap={4} align="start">
+            <Card variant="outline" flex={1}>
+              <CardHeader>
+                <Heading as="h3" size="sm">
+                  Usage Statistics
+                </Heading>
+              </CardHeader>
+              <CardBody>
+                <ResponsiveContainer width="100%" height={400}>
+                  <LineChart data={data} margin={{ top: 5, right: 20, left: 10, bottom: 5 }}>
+                    <XAxis
+                      dataKey="period"
+                      tickFormatter={(str: string) => dayjs(str).format("MMM D")}
+                    />
+                    <YAxis yAxisId="left" dataKey="Requests" orientation="left" stroke="#8884d8" />
+                    <YAxis
+                      yAxisId="right"
+                      dataKey="Total Spent (USD)"
+                      orientation="right"
+                      unit="$"
+                      stroke="#82ca9d"
+                    />
+                    <Tooltip />
+                    <Legend />
+                    <CartesianGrid stroke="#f5f5f5" />
+                    <Line
+                      dataKey="Requests"
+                      stroke="#8884d8"
+                      yAxisId="left"
+                      dot={false}
+                      strokeWidth={2}
+                    />
+                    <Line
+                      dataKey="Total Spent (USD)"
+                      stroke="#82ca9d"
+                      yAxisId="right"
+                      dot={false}
+                      strokeWidth={2}
+                    />
+                  </LineChart>
+                </ResponsiveContainer>
+              </CardBody>
+            </Card>
+            <VStack spacing="4" width="300px" align="stretch">
+              <Card variant="outline">
+                <CardBody>
+                  <Stat>
+                    <HStack>
+                      <StatLabel flex={1}>Total Spent</StatLabel>
+                      <Icon as={DollarSign} boxSize={4} color="gray.500" />
+                    </HStack>
+                    <StatNumber>
+                      ${parseFloat(stats.data?.totals?.totalCost?.toString() ?? "0").toFixed(2)}
+                    </StatNumber>
+                  </Stat>
+                </CardBody>
+              </Card>
+              <Card variant="outline">
+                <CardBody>
+                  <Stat>
+                    <HStack>
+                      <StatLabel flex={1}>Total Requests</StatLabel>
+                      <Icon as={Hash} boxSize={4} color="gray.500" />
+                    </HStack>
+                    <StatNumber>
+                      {stats.data?.totals?.numQueries
+                        ? parseInt(stats.data?.totals?.numQueries.toString())?.toLocaleString()
+                        : undefined}
+                    </StatNumber>
+                  </Stat>
+                </CardBody>
+              </Card>
+              <Card variant="outline" overflow="hidden">
+                <Stat>
+                  <CardHeader>
+                    <HStack>
+                      <StatLabel flex={1}>Errors</StatLabel>
+                      <Icon as={Ban} boxSize={4} color="gray.500" />
+                    </HStack>
+                  </CardHeader>
+                  <Table variant="simple">
+                    <Tbody>
+                      {stats.data?.errors?.map((error) => (
+                        <Tr key={error.code}>
+                          <Td>
+                            {error.name} ({error.code})
+                          </Td>
+                          <Td isNumeric color="red.600">
+                            {parseInt(error.count.toString()).toLocaleString()}
+                          </Td>
+                        </Tr>
+                      ))}
+                    </Tbody>
+                  </Table>
+                </Stat>
+              </Card>
+            </VStack>
+          </HStack>
+          <LoggedCallTable />
+        </VStack>
+      </VStack>
+    </AppShell>
+  );
+}

app/src/pages/project/settings/index.tsx

@@ -0,0 +1,152 @@
+import {
+  Breadcrumb,
+  BreadcrumbItem,
+  Text,
+  type TextProps,
+  VStack,
+  HStack,
+  Input,
+  Button,
+  Divider,
+  Icon,
+  useDisclosure,
+} from "@chakra-ui/react";
+import { useEffect, useState } from "react";
+import { BsTrash } from "react-icons/bs";
+
+import AppShell from "~/components/nav/AppShell";
+import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback, useSelectedOrg } from "~/utils/hooks";
+import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
+import CopiableCode from "~/components/CopiableCode";
+import { DeleteProjectDialog } from "~/components/projectSettings/DeleteProjectDialog";
+
+export default function Settings() {
+  const utils = api.useContext();
+  const { data: selectedOrg } = useSelectedOrg();
+
+  const apiKey =
+    selectedOrg?.apiKeys?.length && selectedOrg?.apiKeys[0] ? selectedOrg?.apiKeys[0].apiKey : "";
+
+  const updateMutation = api.organizations.update.useMutation();
+  const [onSaveName] = useHandledAsyncCallback(async () => {
+    if (name && name !== selectedOrg?.name && selectedOrg?.id) {
+      await updateMutation.mutateAsync({
+        id: selectedOrg.id,
+        updates: { name },
+      });
+      await Promise.all([utils.organizations.get.invalidate({ id: selectedOrg.id })]);
+    }
+  }, [updateMutation, selectedOrg]);
+
+  const [name, setName] = useState(selectedOrg?.name);
+  useEffect(() => {
+    setName(selectedOrg?.name);
+  }, [selectedOrg?.name]);
+
+  const deleteProjectOpen = useDisclosure();
+
+  return (
+    <>
+      <AppShell>
+        <PageHeaderContainer>
+          <Breadcrumb>
+            <BreadcrumbItem>
+              <ProjectBreadcrumbContents />
+            </BreadcrumbItem>
+            <BreadcrumbItem isCurrentPage>
+              <Text>Project Settings</Text>
+            </BreadcrumbItem>
+          </Breadcrumb>
+        </PageHeaderContainer>
+        <VStack px={8} pt={4} alignItems="flex-start" spacing={4}>
+          <VStack spacing={0} alignItems="flex-start">
+            <Text fontSize="2xl" fontWeight="bold">
+              Project Settings
+            </Text>
+            <Text fontSize="sm">
+              Configure your project settings. These settings only apply to {selectedOrg?.name}.
+            </Text>
+          </VStack>
+          <VStack
+            w="full"
+            alignItems="flex-start"
+            borderWidth={1}
+            borderRadius={4}
+            borderColor="gray.300"
+            p={6}
+            spacing={6}
+          >
+            <VStack alignItems="flex-start" w="full">
+              <Text fontWeight="bold" fontSize="xl">
+                Display Name
+              </Text>
+              <Input
+                w="full"
+                maxW={600}
+                value={name}
+                onChange={(e) => setName(e.target.value)}
+                borderColor="gray.300"
+              />
+              <Button
+                isDisabled={!name || name === selectedOrg?.name}
+                colorScheme="orange"
+                borderRadius={4}
+                mt={2}
+                _disabled={{
+                  opacity: 0.6,
+                }}
+                onClick={onSaveName}
+              >
+                Rename Project
+              </Button>
+            </VStack>
+            <Divider backgroundColor="gray.300" />
+            <VStack alignItems="flex-start">
+              <Subtitle>Project API Key</Subtitle>
+              <Text fontSize="sm">
+                Use your project API key to authenticate your requests when sending data to
+                OpenPipe. You can set this key in your environment variables, or use it directly in
+                your code.
+              </Text>
+            </VStack>
+            <CopiableCode code={apiKey} />
+            <Divider />
+            {selectedOrg?.personalOrgUserId ? (
+              <VStack alignItems="flex-start">
+                <Subtitle>Personal Project</Subtitle>
+                <Text fontSize="sm">
+                  This project is {selectedOrg?.personalOrgUser?.name}'s personal project. It cannot
+                  be deleted.
+                </Text>
+              </VStack>
+            ) : (
+              <VStack alignItems="flex-start">
+                <Subtitle color="red.600">Danger Zone</Subtitle>
+                <Text fontSize="sm">
+                  Permanently delete your project and all of its data. This action cannot be undone.
+                </Text>
+                <HStack
+                  as={Button}
+                  isDisabled={selectedOrg?.role !== "ADMIN"}
+                  colorScheme="red"
+                  variant="outline"
+                  borderRadius={4}
+                  mt={2}
+                  onClick={deleteProjectOpen.onOpen}
+                >
+                  <Icon as={BsTrash} />
+                  <Text>Delete {selectedOrg?.name}</Text>
+                </HStack>
+              </VStack>
+            )}
+          </VStack>
+        </VStack>
+      </AppShell>
+      <DeleteProjectDialog isOpen={deleteProjectOpen.isOpen} onClose={deleteProjectOpen.onClose} />
+    </>
+  );
+}
+
+const Subtitle = (props: TextProps) => <Text fontWeight="bold" fontSize="xl" {...props} />;

app/src/server/api/root.router.ts

@@ -8,6 +8,9 @@ import { evaluationsRouter } from "./routers/evaluations.router";
 import { worldChampsRouter } from "./routers/worldChamps.router";
 import { datasetsRouter } from "./routers/datasets.router";
 import { datasetEntries } from "./routers/datasetEntries.router";
+import { externalApiRouter } from "./routers/externalApi.router";
+import { organizationsRouter } from "./routers/organizations.router";
+import { dashboardRouter } from "./routers/dashboard.router";
 
 /**
  * This is the primary router for your server.
@@ -24,6 +27,9 @@ export const appRouter = createTRPCRouter({
   worldChamps: worldChampsRouter,
   datasets: datasetsRouter,
   datasetEntries: datasetEntries,
+  organizations: organizationsRouter,
+  dashboard: dashboardRouter,
+  externalApi: externalApiRouter,
 });
 
 // export type definition of API

app/src/server/api/routers/dashboard.router.ts

@@ -0,0 +1,118 @@
+import { sql } from "kysely";
+import { z } from "zod";
+import { createTRPCRouter, publicProcedure } from "~/server/api/trpc";
+import { kysely, prisma } from "~/server/db";
+import dayjs from "~/utils/dayjs";
+
+export const dashboardRouter = createTRPCRouter({
+  stats: publicProcedure
+    .input(
+      z.object({
+        // TODO: actually take startDate into account
+        startDate: z.string().optional(),
+        organizationId: z.string(),
+      }),
+    )
+    .query(async ({ input }) => {
+      // Return the stats group by hour
+      const periods = await kysely
+        .selectFrom("LoggedCall")
+        .leftJoin(
+          "LoggedCallModelResponse",
+          "LoggedCall.id",
+          "LoggedCallModelResponse.originalLoggedCallId",
+        )
+        .where("organizationId", "=", input.organizationId)
+        .select(({ fn }) => [
+          sql<Date>`date_trunc('day', "LoggedCallModelResponse"."startTime")`.as("period"),
+          sql<number>`count("LoggedCall"."id")::int`.as("numQueries"),
+          fn.sum(fn.coalesce("LoggedCallModelResponse.totalCost", sql<number>`0`)).as("totalCost"),
+        ])
+        .groupBy("period")
+        .orderBy("period")
+        .execute();
+
+      let originalDataIndex = periods.length - 1;
+      // *SLAMS DOWN GLASS OF WHISKEY* timezones, amirite?
+      let dayToMatch = dayjs(input.startDate || new Date());
+      // Ensure that the initial date we're matching against is never before the first period
+      if (
+        periods[originalDataIndex] &&
+        dayToMatch.isBefore(periods[originalDataIndex]?.period, "day")
+      ) {
+        dayToMatch = dayjs(periods[originalDataIndex]?.period);
+      }
+      const backfilledPeriods: typeof periods = [];
+
+      // Backfill from now to 14 days ago or the date of the first logged call, whichever is earlier
+      while (
+        backfilledPeriods.length < 14 ||
+        (periods[0]?.period && !dayToMatch.isBefore(periods[0]?.period, "day"))
+      ) {
+        const nextOriginalPeriod = periods[originalDataIndex];
+        if (nextOriginalPeriod && dayjs(nextOriginalPeriod?.period).isSame(dayToMatch, "day")) {
+          backfilledPeriods.unshift(nextOriginalPeriod);
+          originalDataIndex--;
+        } else {
+          backfilledPeriods.unshift({
+            period: dayjs(dayToMatch).toDate(),
+            numQueries: 0,
+            totalCost: 0,
+          });
+        }
+        dayToMatch = dayToMatch.subtract(1, "day");
+      }
+
+      const totals = await kysely
+        .selectFrom("LoggedCall")
+        .leftJoin(
+          "LoggedCallModelResponse",
+          "LoggedCall.id",
+          "LoggedCallModelResponse.originalLoggedCallId",
+        )
+        .where("organizationId", "=", input.organizationId)
+        .select(({ fn }) => [
+          fn.sum(fn.coalesce("LoggedCallModelResponse.totalCost", sql<number>`0`)).as("totalCost"),
+          fn.count("LoggedCall.id").as("numQueries"),
+        ])
+        .executeTakeFirst();
+
+      const errors = await kysely
+        .selectFrom("LoggedCall")
+        .where("organizationId", "=", input.organizationId)
+        .leftJoin(
+          "LoggedCallModelResponse",
+          "LoggedCall.id",
+          "LoggedCallModelResponse.originalLoggedCallId",
+        )
+        .select(({ fn }) => [fn.count("LoggedCall.id").as("count"), "respStatus as code"])
+        .where("respStatus", ">", 200)
+        .groupBy("code")
+        .orderBy("count", "desc")
+        .execute();
+
+      const namedErrors = errors.map((e) => {
+        if (e.code === 429) {
+          return { ...e, name: "Rate limited" };
+        } else if (e.code === 500) {
+          return { ...e, name: "Internal server error" };
+        } else {
+          return { ...e, name: "Other" };
+        }
+      });
+
+      return { periods: backfilledPeriods, totals, errors: namedErrors };
+    }),
+
+  // TODO useInfiniteQuery
+  // https://discord.com/channels/966627436387266600/1122258443886153758/1122258443886153758
+  loggedCalls: publicProcedure.input(z.object({})).query(async ({ input }) => {
+    const loggedCalls = await prisma.loggedCall.findMany({
+      orderBy: { startTime: "desc" },
+      include: { tags: true, modelResponse: true },
+      take: 20,
+    });
+
+    return loggedCalls;
+  }),
+});

app/src/server/api/routers/datasets.router.ts

@@ -3,65 +3,62 @@ import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/
 import { prisma } from "~/server/db";
 import {
   requireCanModifyDataset,
+  requireCanModifyOrganization,
   requireCanViewDataset,
-  requireNothing,
+  requireCanViewOrganization,
 } from "~/utils/accessControl";
-import userOrg from "~/server/utils/userOrg";
 
 export const datasetsRouter = createTRPCRouter({
-  list: protectedProcedure.query(async ({ ctx }) => {
-    // Anyone can list experiments
-    requireNothing(ctx);
+  list: protectedProcedure
+    .input(z.object({ organizationId: z.string() }))
+    .query(async ({ input, ctx }) => {
+      await requireCanViewOrganization(input.organizationId, ctx);
 
-    const datasets = await prisma.dataset.findMany({
-      where: {
-        organization: {
-          organizationUsers: {
-            some: { userId: ctx.session.user.id },
+      const datasets = await prisma.dataset.findMany({
+        where: {
+          organizationId: input.organizationId,
+        },
+        orderBy: {
+          createdAt: "desc",
+        },
+        include: {
+          _count: {
+            select: { datasetEntries: true },
           },
         },
-      },
-      orderBy: {
-        createdAt: "desc",
-      },
-      include: {
-        _count: {
-          select: { datasetEntries: true },
-        },
-      },
-    });
+      });
 
-    return datasets;
-  }),
+      return datasets;
+    }),
 
   get: publicProcedure.input(z.object({ id: z.string() })).query(async ({ input, ctx }) => {
     await requireCanViewDataset(input.id, ctx);
     return await prisma.dataset.findFirstOrThrow({
       where: { id: input.id },
+      include: {
+        organization: true,
+      },
     });
   }),
 
-  create: protectedProcedure.input(z.object({})).mutation(async ({ ctx }) => {
-    // Anyone can create an experiment
-    requireNothing(ctx);
+  create: protectedProcedure
+    .input(z.object({ organizationId: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      await requireCanModifyOrganization(input.organizationId, ctx);
 
-    const numDatasets = await prisma.dataset.count({
-      where: {
-        organization: {
-          organizationUsers: {
-            some: { userId: ctx.session.user.id },
-          },
+      const numDatasets = await prisma.dataset.count({
+        where: {
+          organizationId: input.organizationId,
         },
-      },
-    });
+      });
 
-    return await prisma.dataset.create({
-      data: {
-        name: `Dataset ${numDatasets + 1}`,
-        organizationId: (await userOrg(ctx.session.user.id)).id,
-      },
-    });
-  }),
+      return await prisma.dataset.create({
+        data: {
+          name: `Dataset ${numDatasets + 1}`,
+          organizationId: input.organizationId,
+        },
+      });
+    }),
 
   update: protectedProcedure
     .input(z.object({ id: z.string(), updates: z.object({ name: z.string() }) }))

app/src/server/api/routers/experiments.router.ts

@@ -8,10 +8,10 @@ import { generateNewCell } from "~/server/utils/generateNewCell";
 import {
   canModifyExperiment,
   requireCanModifyExperiment,
+  requireCanModifyOrganization,
   requireCanViewExperiment,
-  requireNothing,
+  requireCanViewOrganization,
 } from "~/utils/accessControl";
-import userOrg from "~/server/utils/userOrg";
 import generateTypes from "~/modelProviders/generateTypes";
 import { promptConstructorVersion } from "~/promptConstructor/version";
 
@@ -43,55 +43,55 @@ export const experimentsRouter = createTRPCRouter({
       testScenarioCount,
     };
   }),
-  list: protectedProcedure.query(async ({ ctx }) => {
-    // Anyone can list experiments
-    requireNothing(ctx);
+  list: protectedProcedure
+    .input(z.object({ organizationId: z.string() }))
+    .query(async ({ input, ctx }) => {
+      await requireCanViewOrganization(input.organizationId, ctx);
 
-    const experiments = await prisma.experiment.findMany({
-      where: {
-        organization: {
-          organizationUsers: {
-            some: { userId: ctx.session.user.id },
-          },
+      const experiments = await prisma.experiment.findMany({
+        where: {
+          organizationId: input.organizationId,
         },
-      },
-      orderBy: {
-        sortIndex: "desc",
-      },
-    });
+        orderBy: {
+          sortIndex: "desc",
+        },
+      });
 
-    // TODO: look for cleaner way to do this. Maybe aggregate?
-    const experimentsWithCounts = await Promise.all(
-      experiments.map(async (experiment) => {
-        const visibleTestScenarioCount = await prisma.testScenario.count({
-          where: {
-            experimentId: experiment.id,
-            visible: true,
-          },
-        });
+      // TODO: look for cleaner way to do this. Maybe aggregate?
+      const experimentsWithCounts = await Promise.all(
+        experiments.map(async (experiment) => {
+          const visibleTestScenarioCount = await prisma.testScenario.count({
+            where: {
+              experimentId: experiment.id,
+              visible: true,
+            },
+          });
 
-        const visiblePromptVariantCount = await prisma.promptVariant.count({
-          where: {
-            experimentId: experiment.id,
-            visible: true,
-          },
-        });
+          const visiblePromptVariantCount = await prisma.promptVariant.count({
+            where: {
+              experimentId: experiment.id,
+              visible: true,
+            },
+          });
 
-        return {
-          ...experiment,
-          testScenarioCount: visibleTestScenarioCount,
-          promptVariantCount: visiblePromptVariantCount,
-        };
-      }),
-    );
+          return {
+            ...experiment,
+            testScenarioCount: visibleTestScenarioCount,
+            promptVariantCount: visiblePromptVariantCount,
+          };
+        }),
+      );
 
-    return experimentsWithCounts;
-  }),
+      return experimentsWithCounts;
+    }),
 
   get: publicProcedure.input(z.object({ id: z.string() })).query(async ({ input, ctx }) => {
     await requireCanViewExperiment(input.id, ctx);
     const experiment = await prisma.experiment.findFirstOrThrow({
       where: { id: input.id },
+      include: {
+        organization: true,
+      },
     });
 
     const canModify = ctx.session?.user.id
@@ -107,222 +107,224 @@ export const experimentsRouter = createTRPCRouter({
     };
   }),
 
-  fork: protectedProcedure.input(z.object({ id: z.string() })).mutation(async ({ input, ctx }) => {
-    await requireCanViewExperiment(input.id, ctx);
+  fork: protectedProcedure
+    .input(z.object({ id: z.string(), organizationId: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      await requireCanViewExperiment(input.id, ctx);
+      await requireCanModifyOrganization(input.organizationId, ctx);
 
-    const [
-      existingExp,
-      existingVariants,
-      existingScenarios,
-      existingCells,
-      evaluations,
-      templateVariables,
-    ] = await prisma.$transaction([
-      prisma.experiment.findUniqueOrThrow({
-        where: {
-          id: input.id,
-        },
-      }),
-      prisma.promptVariant.findMany({
-        where: {
-          experimentId: input.id,
-          visible: true,
-        },
-      }),
-      prisma.testScenario.findMany({
-        where: {
-          experimentId: input.id,
-          visible: true,
-        },
-      }),
-      prisma.scenarioVariantCell.findMany({
-        where: {
-          testScenario: {
-            visible: true,
+      const [
+        existingExp,
+        existingVariants,
+        existingScenarios,
+        existingCells,
+        evaluations,
+        templateVariables,
+      ] = await prisma.$transaction([
+        prisma.experiment.findUniqueOrThrow({
+          where: {
+            id: input.id,
           },
-          promptVariant: {
+        }),
+        prisma.promptVariant.findMany({
+          where: {
             experimentId: input.id,
             visible: true,
           },
-        },
-        include: {
-          modelResponses: {
-            include: {
-              outputEvaluations: true,
+        }),
+        prisma.testScenario.findMany({
+          where: {
+            experimentId: input.id,
+            visible: true,
+          },
+        }),
+        prisma.scenarioVariantCell.findMany({
+          where: {
+            testScenario: {
+              visible: true,
+            },
+            promptVariant: {
+              experimentId: input.id,
+              visible: true,
             },
           },
-        },
-      }),
-      prisma.evaluation.findMany({
-        where: {
-          experimentId: input.id,
-        },
-      }),
-      prisma.templateVariable.findMany({
-        where: {
-          experimentId: input.id,
-        },
-      }),
-    ]);
+          include: {
+            modelResponses: {
+              include: {
+                outputEvaluations: true,
+              },
+            },
+          },
+        }),
+        prisma.evaluation.findMany({
+          where: {
+            experimentId: input.id,
+          },
+        }),
+        prisma.templateVariable.findMany({
+          where: {
+            experimentId: input.id,
+          },
+        }),
+      ]);
 
-    const newExperimentId = uuidv4();
+      const newExperimentId = uuidv4();
 
-    const existingToNewVariantIds = new Map<string, string>();
-    const variantsToCreate: Prisma.PromptVariantCreateManyInput[] = [];
-    for (const variant of existingVariants) {
-      const newVariantId = uuidv4();
-      existingToNewVariantIds.set(variant.id, newVariantId);
-      variantsToCreate.push({
-        ...variant,
-        id: newVariantId,
-        experimentId: newExperimentId,
-      });
-    }
-
-    const existingToNewScenarioIds = new Map<string, string>();
-    const scenariosToCreate: Prisma.TestScenarioCreateManyInput[] = [];
-    for (const scenario of existingScenarios) {
-      const newScenarioId = uuidv4();
-      existingToNewScenarioIds.set(scenario.id, newScenarioId);
-      scenariosToCreate.push({
-        ...scenario,
-        id: newScenarioId,
-        experimentId: newExperimentId,
-        variableValues: scenario.variableValues as Prisma.InputJsonValue,
-      });
-    }
-
-    const existingToNewEvaluationIds = new Map<string, string>();
-    const evaluationsToCreate: Prisma.EvaluationCreateManyInput[] = [];
-    for (const evaluation of evaluations) {
-      const newEvaluationId = uuidv4();
-      existingToNewEvaluationIds.set(evaluation.id, newEvaluationId);
-      evaluationsToCreate.push({
-        ...evaluation,
-        id: newEvaluationId,
-        experimentId: newExperimentId,
-      });
-    }
-
-    const cellsToCreate: Prisma.ScenarioVariantCellCreateManyInput[] = [];
-    const modelResponsesToCreate: Prisma.ModelResponseCreateManyInput[] = [];
-    const outputEvaluationsToCreate: Prisma.OutputEvaluationCreateManyInput[] = [];
-    for (const cell of existingCells) {
-      const newCellId = uuidv4();
-      const { modelResponses, ...cellData } = cell;
-      cellsToCreate.push({
-        ...cellData,
-        id: newCellId,
-        promptVariantId: existingToNewVariantIds.get(cell.promptVariantId) ?? "",
-        testScenarioId: existingToNewScenarioIds.get(cell.testScenarioId) ?? "",
-        prompt: (cell.prompt as Prisma.InputJsonValue) ?? undefined,
-      });
-      for (const modelResponse of modelResponses) {
-        const newModelResponseId = uuidv4();
-        const { outputEvaluations, ...modelResponseData } = modelResponse;
-        modelResponsesToCreate.push({
-          ...modelResponseData,
-          id: newModelResponseId,
-          scenarioVariantCellId: newCellId,
-          output: (modelResponse.output as Prisma.InputJsonValue) ?? undefined,
+      const existingToNewVariantIds = new Map<string, string>();
+      const variantsToCreate: Prisma.PromptVariantCreateManyInput[] = [];
+      for (const variant of existingVariants) {
+        const newVariantId = uuidv4();
+        existingToNewVariantIds.set(variant.id, newVariantId);
+        variantsToCreate.push({
+          ...variant,
+          id: newVariantId,
+          experimentId: newExperimentId,
         });
-        for (const evaluation of outputEvaluations) {
-          outputEvaluationsToCreate.push({
-            ...evaluation,
-            id: uuidv4(),
-            modelResponseId: newModelResponseId,
-            evaluationId: existingToNewEvaluationIds.get(evaluation.evaluationId) ?? "",
+      }
+
+      const existingToNewScenarioIds = new Map<string, string>();
+      const scenariosToCreate: Prisma.TestScenarioCreateManyInput[] = [];
+      for (const scenario of existingScenarios) {
+        const newScenarioId = uuidv4();
+        existingToNewScenarioIds.set(scenario.id, newScenarioId);
+        scenariosToCreate.push({
+          ...scenario,
+          id: newScenarioId,
+          experimentId: newExperimentId,
+          variableValues: scenario.variableValues as Prisma.InputJsonValue,
+        });
+      }
+
+      const existingToNewEvaluationIds = new Map<string, string>();
+      const evaluationsToCreate: Prisma.EvaluationCreateManyInput[] = [];
+      for (const evaluation of evaluations) {
+        const newEvaluationId = uuidv4();
+        existingToNewEvaluationIds.set(evaluation.id, newEvaluationId);
+        evaluationsToCreate.push({
+          ...evaluation,
+          id: newEvaluationId,
+          experimentId: newExperimentId,
+        });
+      }
+
+      const cellsToCreate: Prisma.ScenarioVariantCellCreateManyInput[] = [];
+      const modelResponsesToCreate: Prisma.ModelResponseCreateManyInput[] = [];
+      const outputEvaluationsToCreate: Prisma.OutputEvaluationCreateManyInput[] = [];
+      for (const cell of existingCells) {
+        const newCellId = uuidv4();
+        const { modelResponses, ...cellData } = cell;
+        cellsToCreate.push({
+          ...cellData,
+          id: newCellId,
+          promptVariantId: existingToNewVariantIds.get(cell.promptVariantId) ?? "",
+          testScenarioId: existingToNewScenarioIds.get(cell.testScenarioId) ?? "",
+          prompt: (cell.prompt as Prisma.InputJsonValue) ?? undefined,
+        });
+        for (const modelResponse of modelResponses) {
+          const newModelResponseId = uuidv4();
+          const { outputEvaluations, ...modelResponseData } = modelResponse;
+          modelResponsesToCreate.push({
+            ...modelResponseData,
+            id: newModelResponseId,
+            scenarioVariantCellId: newCellId,
+            output: (modelResponse.output as Prisma.InputJsonValue) ?? undefined,
           });
+          for (const evaluation of outputEvaluations) {
+            outputEvaluationsToCreate.push({
+              ...evaluation,
+              id: uuidv4(),
+              modelResponseId: newModelResponseId,
+              evaluationId: existingToNewEvaluationIds.get(evaluation.evaluationId) ?? "",
+            });
+          }
         }
       }
-    }
 
-    const templateVariablesToCreate: Prisma.TemplateVariableCreateManyInput[] = [];
-    for (const templateVariable of templateVariables) {
-      templateVariablesToCreate.push({
-        ...templateVariable,
-        id: uuidv4(),
-        experimentId: newExperimentId,
-      });
-    }
+      const templateVariablesToCreate: Prisma.TemplateVariableCreateManyInput[] = [];
+      for (const templateVariable of templateVariables) {
+        templateVariablesToCreate.push({
+          ...templateVariable,
+          id: uuidv4(),
+          experimentId: newExperimentId,
+        });
+      }
 
-    const maxSortIndex =
-      (
-        await prisma.experiment.aggregate({
-          _max: {
-            sortIndex: true,
+      const maxSortIndex =
+        (
+          await prisma.experiment.aggregate({
+            _max: {
+              sortIndex: true,
+            },
+          })
+        )._max?.sortIndex ?? 0;
+
+      await prisma.$transaction([
+        prisma.experiment.create({
+          data: {
+            id: newExperimentId,
+            sortIndex: maxSortIndex + 1,
+            label: `${existingExp.label} (forked)`,
+            organizationId: input.organizationId,
           },
-        })
-      )._max?.sortIndex ?? 0;
+        }),
+        prisma.promptVariant.createMany({
+          data: variantsToCreate,
+        }),
+        prisma.testScenario.createMany({
+          data: scenariosToCreate,
+        }),
+        prisma.scenarioVariantCell.createMany({
+          data: cellsToCreate,
+        }),
+        prisma.modelResponse.createMany({
+          data: modelResponsesToCreate,
+        }),
+        prisma.evaluation.createMany({
+          data: evaluationsToCreate,
+        }),
+        prisma.outputEvaluation.createMany({
+          data: outputEvaluationsToCreate,
+        }),
+        prisma.templateVariable.createMany({
+          data: templateVariablesToCreate,
+        }),
+      ]);
 
-    await prisma.$transaction([
-      prisma.experiment.create({
+      return newExperimentId;
+    }),
+
+  create: protectedProcedure
+    .input(z.object({ organizationId: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      await requireCanModifyOrganization(input.organizationId, ctx);
+
+      const maxSortIndex =
+        (
+          await prisma.experiment.aggregate({
+            _max: {
+              sortIndex: true,
+            },
+            where: { organizationId: input.organizationId },
+          })
+        )._max?.sortIndex ?? 0;
+
+      const exp = await prisma.experiment.create({
         data: {
-          id: newExperimentId,
           sortIndex: maxSortIndex + 1,
-          label: `${existingExp.label} (forked)`,
-          organizationId: (await userOrg(ctx.session.user.id)).id,
+          label: `Experiment ${maxSortIndex + 1}`,
+          organizationId: input.organizationId,
         },
-      }),
-      prisma.promptVariant.createMany({
-        data: variantsToCreate,
-      }),
-      prisma.testScenario.createMany({
-        data: scenariosToCreate,
-      }),
-      prisma.scenarioVariantCell.createMany({
-        data: cellsToCreate,
-      }),
-      prisma.modelResponse.createMany({
-        data: modelResponsesToCreate,
-      }),
-      prisma.evaluation.createMany({
-        data: evaluationsToCreate,
-      }),
-      prisma.outputEvaluation.createMany({
-        data: outputEvaluationsToCreate,
-      }),
-      prisma.templateVariable.createMany({
-        data: templateVariablesToCreate,
-      }),
-    ]);
+      });
 
-    return newExperimentId;
-  }),
-
-  create: protectedProcedure.input(z.object({})).mutation(async ({ ctx }) => {
-    // Anyone can create an experiment
-    requireNothing(ctx);
-
-    const organizationId = (await userOrg(ctx.session.user.id)).id;
-
-    const maxSortIndex =
-      (
-        await prisma.experiment.aggregate({
-          _max: {
-            sortIndex: true,
-          },
-          where: { organizationId },
-        })
-      )._max?.sortIndex ?? 0;
-
-    const exp = await prisma.experiment.create({
-      data: {
-        sortIndex: maxSortIndex + 1,
-        label: `Experiment ${maxSortIndex + 1}`,
-        organizationId,
-      },
-    });
-
-    const [variant, _, scenario1, scenario2, scenario3] = await prisma.$transaction([
-      prisma.promptVariant.create({
-        data: {
-          experimentId: exp.id,
-          label: "Prompt Variant 1",
-          sortIndex: 0,
-          // The interpolated $ is necessary until dedent incorporates
-          // https://github.com/dmnd/dedent/pull/46
-          promptConstructor: dedent`
+      const [variant, _, scenario1, scenario2, scenario3] = await prisma.$transaction([
+        prisma.promptVariant.create({
+          data: {
+            experimentId: exp.id,
+            label: "Prompt Variant 1",
+            sortIndex: 0,
+            // The interpolated $ is necessary until dedent incorporates
+            // https://github.com/dmnd/dedent/pull/46
+            promptConstructor: dedent`
           /**
            * Use Javascript to define an OpenAI chat completion
            * (https://platform.openai.com/docs/api-reference/chat/create).
@@ -341,49 +343,49 @@ export const experimentsRouter = createTRPCRouter({
               },
             ],
           });`,
-          model: "gpt-3.5-turbo-0613",
-          modelProvider: "openai/ChatCompletion",
-          promptConstructorVersion,
-        },
-      }),
-      prisma.templateVariable.create({
-        data: {
-          experimentId: exp.id,
-          label: "language",
-        },
-      }),
-      prisma.testScenario.create({
-        data: {
-          experimentId: exp.id,
-          variableValues: {
-            language: "English",
+            model: "gpt-3.5-turbo-0613",
+            modelProvider: "openai/ChatCompletion",
+            promptConstructorVersion,
           },
-        },
-      }),
-      prisma.testScenario.create({
-        data: {
-          experimentId: exp.id,
-          variableValues: {
-            language: "Spanish",
+        }),
+        prisma.templateVariable.create({
+          data: {
+            experimentId: exp.id,
+            label: "language",
           },
-        },
-      }),
-      prisma.testScenario.create({
-        data: {
-          experimentId: exp.id,
-          variableValues: {
-            language: "German",
+        }),
+        prisma.testScenario.create({
+          data: {
+            experimentId: exp.id,
+            variableValues: {
+              language: "English",
+            },
           },
-        },
-      }),
-    ]);
+        }),
+        prisma.testScenario.create({
+          data: {
+            experimentId: exp.id,
+            variableValues: {
+              language: "Spanish",
+            },
+          },
+        }),
+        prisma.testScenario.create({
+          data: {
+            experimentId: exp.id,
+            variableValues: {
+              language: "German",
+            },
+          },
+        }),
+      ]);
 
-    await generateNewCell(variant.id, scenario1.id);
-    await generateNewCell(variant.id, scenario2.id);
-    await generateNewCell(variant.id, scenario3.id);
+      await generateNewCell(variant.id, scenario1.id);
+      await generateNewCell(variant.id, scenario2.id);
+      await generateNewCell(variant.id, scenario3.id);
 
-    return exp;
-  }),
+      return exp;
+    }),
 
   update: protectedProcedure
     .input(z.object({ id: z.string(), updates: z.object({ label: z.string() }) }))

app/src/server/api/routers/externalApi.router.ts

@@ -0,0 +1,205 @@
+import { type Prisma } from "@prisma/client";
+import { type JsonValue } from "type-fest";
+import { z } from "zod";
+import { v4 as uuidv4 } from "uuid";
+import { TRPCError } from "@trpc/server";
+
+import { createTRPCRouter, publicProcedure } from "~/server/api/trpc";
+import { prisma } from "~/server/db";
+import { hashRequest } from "~/server/utils/hashObject";
+
+const reqValidator = z.object({
+  model: z.string(),
+  messages: z.array(z.any()),
+});
+
+const respValidator = z.object({
+  id: z.string(),
+  model: z.string(),
+  usage: z.object({
+    total_tokens: z.number(),
+    prompt_tokens: z.number(),
+    completion_tokens: z.number(),
+  }),
+  choices: z.array(
+    z.object({
+      finish_reason: z.string(),
+    }),
+  ),
+});
+
+export const externalApiRouter = createTRPCRouter({
+  checkCache: publicProcedure
+    .meta({
+      openapi: {
+        method: "POST",
+        path: "/v1/check-cache",
+        description: "Check if a prompt is cached",
+      },
+    })
+    .input(
+      z.object({
+        startTime: z.number().describe("Unix timestamp in milliseconds"),
+        reqPayload: z.unknown().describe("JSON-encoded request payload"),
+        tags: z
+          .record(z.string())
+          .optional()
+          .describe(
+            'Extra tags to attach to the call for filtering. Eg { "userId": "123", "promptId": "populate-title" }',
+          ),
+      }),
+    )
+    .output(
+      z.object({
+        respPayload: z.unknown().optional().describe("JSON-encoded response payload"),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      const apiKey = ctx.apiKey;
+      if (!apiKey) {
+        throw new TRPCError({ code: "UNAUTHORIZED" });
+      }
+      const key = await prisma.apiKey.findUnique({
+        where: { apiKey },
+      });
+      if (!key) {
+        throw new TRPCError({ code: "UNAUTHORIZED" });
+      }
+      const reqPayload = await reqValidator.spa(input.reqPayload);
+      const cacheKey = hashRequest(key.organizationId, reqPayload as JsonValue);
+
+      const existingResponse = await prisma.loggedCallModelResponse.findFirst({
+        where: {
+          cacheKey,
+        },
+        include: {
+          originalLoggedCall: true,
+        },
+        orderBy: {
+          startTime: "desc",
+        },
+      });
+
+      if (!existingResponse) return { respPayload: null };
+
+      await prisma.loggedCall.create({
+        data: {
+          organizationId: key.organizationId,
+          startTime: new Date(input.startTime),
+          cacheHit: true,
+          modelResponseId: existingResponse.id,
+        },
+      });
+
+      return {
+        respPayload: existingResponse.respPayload,
+      };
+    }),
+
+  report: publicProcedure
+    .meta({
+      openapi: {
+        method: "POST",
+        path: "/v1/report",
+        description: "Report an API call",
+      },
+    })
+    .input(
+      z.object({
+        startTime: z.number().describe("Unix timestamp in milliseconds"),
+        endTime: z.number().describe("Unix timestamp in milliseconds"),
+        reqPayload: z.unknown().describe("JSON-encoded request payload"),
+        respPayload: z.unknown().optional().describe("JSON-encoded response payload"),
+        respStatus: z.number().optional().describe("HTTP status code of response"),
+        error: z.string().optional().describe("User-friendly error message"),
+        tags: z
+          .record(z.string())
+          .optional()
+          .describe(
+            'Extra tags to attach to the call for filtering. Eg { "userId": "123", "promptId": "populate-title" }',
+          ),
+      }),
+    )
+    .output(z.void())
+    .mutation(async ({ input, ctx }) => {
+      const apiKey = ctx.apiKey;
+      if (!apiKey) {
+        throw new TRPCError({ code: "UNAUTHORIZED" });
+      }
+      const key = await prisma.apiKey.findUnique({
+        where: { apiKey },
+      });
+      if (!key) {
+        throw new TRPCError({ code: "UNAUTHORIZED" });
+      }
+      const reqPayload = await reqValidator.spa(input.reqPayload);
+      const respPayload = await respValidator.spa(input.respPayload);
+
+      const requestHash = hashRequest(key.organizationId, reqPayload as JsonValue);
+
+      const newLoggedCallId = uuidv4();
+      const newModelResponseId = uuidv4();
+
+      const usage = respPayload.success ? respPayload.data.usage : undefined;
+
+      await prisma.$transaction([
+        prisma.loggedCall.create({
+          data: {
+            id: newLoggedCallId,
+            organizationId: key.organizationId,
+            startTime: new Date(input.startTime),
+            cacheHit: false,
+          },
+        }),
+        prisma.loggedCallModelResponse.create({
+          data: {
+            id: newModelResponseId,
+            originalLoggedCallId: newLoggedCallId,
+            startTime: new Date(input.startTime),
+            endTime: new Date(input.endTime),
+            reqPayload: input.reqPayload as Prisma.InputJsonValue,
+            respPayload: input.respPayload as Prisma.InputJsonValue,
+            respStatus: input.respStatus,
+            error: input.error,
+            durationMs: input.endTime - input.startTime,
+            ...(respPayload.success
+              ? {
+                  cacheKey: requestHash,
+                  inputTokens: usage ? usage.prompt_tokens : undefined,
+                  outputTokens: usage ? usage.completion_tokens : undefined,
+                }
+              : null),
+          },
+        }),
+        // Avoid foreign key constraint error by updating the logged call after the model response is created
+        prisma.loggedCall.update({
+          where: {
+            id: newLoggedCallId,
+          },
+          data: {
+            modelResponseId: newModelResponseId,
+          },
+        }),
+      ]);
+
+      if (input.tags) {
+        const tagsToCreate = Object.entries(input.tags).map(([name, value]) => ({
+          loggedCallId: newLoggedCallId,
+          // sanitize tags
+          name: name.replaceAll(/[^a-zA-Z0-9_]/g, "_"),
+          value,
+        }));
+
+        if (reqPayload.success) {
+          tagsToCreate.push({
+            loggedCallId: newLoggedCallId,
+            name: "$model",
+            value: reqPayload.data.model,
+          });
+        }
+        await prisma.loggedCallTag.createMany({
+          data: tagsToCreate,
+        });
+      }
+    }),
+});

app/src/server/api/routers/organizations.router.ts

@@ -0,0 +1,128 @@
+import { TRPCError } from "@trpc/server";
+import { v4 as uuidv4 } from "uuid";
+import { z } from "zod";
+
+import { createTRPCRouter, protectedProcedure } from "~/server/api/trpc";
+import { prisma } from "~/server/db";
+import { generateApiKey } from "~/server/utils/generateApiKey";
+import userOrg from "~/server/utils/userOrg";
+import {
+  requireCanModifyOrganization,
+  requireCanViewOrganization,
+  requireIsOrgAdmin,
+  requireNothing,
+} from "~/utils/accessControl";
+
+export const organizationsRouter = createTRPCRouter({
+  list: protectedProcedure.query(async ({ ctx }) => {
+    const userId = ctx.session.user.id;
+    requireNothing(ctx);
+
+    if (!userId) {
+      return null;
+    }
+
+    const organizations = await prisma.organization.findMany({
+      where: {
+        organizationUsers: {
+          some: { userId: ctx.session.user.id },
+        },
+      },
+      orderBy: {
+        createdAt: "asc",
+      },
+    });
+
+    if (!organizations.length) {
+      // TODO: We should move this to a separate endpoint that is called on sign up
+      const personalOrg = await userOrg(userId);
+      organizations.push(personalOrg);
+    }
+
+    return organizations;
+  }),
+  get: protectedProcedure.input(z.object({ id: z.string() })).query(async ({ input, ctx }) => {
+    await requireCanViewOrganization(input.id, ctx);
+    const [org, userRole] = await prisma.$transaction([
+      prisma.organization.findUnique({
+        where: {
+          id: input.id,
+        },
+        include: {
+          apiKeys: true,
+          personalOrgUser: true,
+        },
+      }),
+      prisma.organizationUser.findFirst({
+        where: {
+          userId: ctx.session.user.id,
+          organizationId: input.id,
+          role: {
+            in: ["ADMIN", "MEMBER"],
+          },
+        },
+      }),
+    ]);
+
+    if (!org) {
+      throw new TRPCError({ code: "NOT_FOUND" });
+    }
+
+    return {
+      ...org,
+      role: userRole?.role ?? null,
+    };
+  }),
+  update: protectedProcedure
+    .input(z.object({ id: z.string(), updates: z.object({ name: z.string() }) }))
+    .mutation(async ({ input, ctx }) => {
+      await requireCanModifyOrganization(input.id, ctx);
+      return await prisma.organization.update({
+        where: {
+          id: input.id,
+        },
+        data: {
+          name: input.updates.name,
+        },
+      });
+    }),
+  create: protectedProcedure
+    .input(z.object({ name: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      requireNothing(ctx);
+      const newOrgId = uuidv4();
+      const [newOrg] = await prisma.$transaction([
+        prisma.organization.create({
+          data: {
+            id: newOrgId,
+            name: input.name,
+          },
+        }),
+        prisma.organizationUser.create({
+          data: {
+            userId: ctx.session.user.id,
+            organizationId: newOrgId,
+            role: "ADMIN",
+          },
+        }),
+        prisma.apiKey.create({
+          data: {
+            name: "Default API Key",
+            organizationId: newOrgId,
+            apiKey: generateApiKey(),
+          },
+        }),
+      ]);
+      return newOrg;
+    }),
+  delete: protectedProcedure
+    .input(z.object({ id: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      await requireIsOrgAdmin(input.id, ctx);
+      return await prisma.organization.delete({
+        where: {
+          id: input.id,
+        },
+      });
+    }),
+});

app/src/server/api/trpc.ts

@@ -11,6 +11,7 @@ import { initTRPC, TRPCError } from "@trpc/server";
 import { type CreateNextContextOptions } from "@trpc/server/adapters/next";
 import { type Session } from "next-auth";
 import superjson from "superjson";
+import { type OpenApiMeta } from "trpc-openapi";
 import { ZodError } from "zod";
 import { getServerAuthSession } from "~/server/auth";
 import { prisma } from "~/server/db";
@@ -26,6 +27,7 @@ import { capturePath } from "~/utils/analytics/serverAnalytics";
 
 type CreateContextOptions = {
   session: Session | null;
+  apiKey: string | null;
 };
 
 // eslint-disable-next-line @typescript-eslint/no-empty-function
@@ -44,6 +46,7 @@ const noOp = () => {};
 export const createInnerTRPCContext = (opts: CreateContextOptions) => {
   return {
     session: opts.session,
+    apiKey: opts.apiKey,
     prisma,
     markAccessControlRun: noOp,
   };
@@ -61,8 +64,11 @@ export const createTRPCContext = async (opts: CreateNextContextOptions) => {
   // Get the session from the server using the getServerSession wrapper function
   const session = await getServerAuthSession({ req, res });
 
+  const apiKey = req.headers["x-openpipe-api-key"] as string | null;
+
   return createInnerTRPCContext({
     session,
+    apiKey,
   });
 };
 
@@ -76,18 +82,21 @@ export const createTRPCContext = async (opts: CreateNextContextOptions) => {
 
 export type TRPCContext = Awaited<ReturnType<typeof createTRPCContext>>;
 
-const t = initTRPC.context<typeof createTRPCContext>().create({
-  transformer: superjson,
-  errorFormatter({ shape, error }) {
-    return {
-      ...shape,
-      data: {
-        ...shape.data,
-        zodError: error.cause instanceof ZodError ? error.cause.flatten() : null,
-      },
-    };
-  },
-});
+const t = initTRPC
+  .context<typeof createTRPCContext>()
+  .meta<OpenApiMeta>()
+  .create({
+    transformer: superjson,
+    errorFormatter({ shape, error }) {
+      return {
+        ...shape,
+        data: {
+          ...shape.data,
+          zodError: error.cause instanceof ZodError ? error.cause.flatten() : null,
+        },
+      };
+    },
+  });
 
 /**
  * 3. ROUTER & PROCEDURE (THE IMPORTANT BIT)

app/src/server/auth.ts

@@ -2,9 +2,16 @@ import { PrismaAdapter } from "@next-auth/prisma-adapter";
 import { type GetServerSidePropsContext } from "next";
 import { getServerSession, type NextAuthOptions, type DefaultSession } from "next-auth";
 import { prisma } from "~/server/db";
-import GitHubProvider from "next-auth/providers/github";
+import GitHubModule from "next-auth/providers/github";
 import { env } from "~/env.mjs";
 
+// The client codegen script doesn't properly read the default export
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const untypedGitHubModule = GitHubModule as unknown as any;
+const GitHubProvider: typeof GitHubModule = untypedGitHubModule.default
+  ? untypedGitHubModule.default
+  : untypedGitHubModule;
+
 /**
  * Module augmentation for `next-auth` types. Allows us to add custom properties to the `session`
  * object and keep type safety.

app/src/server/db.ts

@@ -1,6 +1,61 @@
-import { PrismaClient } from "@prisma/client";
+import {
+  type Experiment,
+  type PromptVariant,
+  type TestScenario,
+  type TemplateVariable,
+  type ScenarioVariantCell,
+  type ModelResponse,
+  type Evaluation,
+  type OutputEvaluation,
+  type Dataset,
+  type DatasetEntry,
+  type Organization,
+  type OrganizationUser,
+  type WorldChampEntrant,
+  type LoggedCall,
+  type LoggedCallModelResponse,
+  type LoggedCallTag,
+  type ApiKey,
+  type Account,
+  type Session,
+  type User,
+  type VerificationToken,
+  PrismaClient,
+} from "@prisma/client";
+import { Kysely, PostgresDialect } from "kysely";
+// TODO: Revert to normal import when our tsconfig.json is fixed
+// import { Pool } from "pg";
+import PGModule from "pg";
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const UntypedPool = PGModule.Pool as any;
+const Pool = (UntypedPool.default ? UntypedPool.default : UntypedPool) as typeof PGModule.Pool;
+
 import { env } from "~/env.mjs";
 
+interface DB {
+  Experiment: Experiment;
+  PromptVariant: PromptVariant;
+  TestScenario: TestScenario;
+  TemplateVariable: TemplateVariable;
+  ScenarioVariantCell: ScenarioVariantCell;
+  ModelResponse: ModelResponse;
+  Evaluation: Evaluation;
+  OutputEvaluation: OutputEvaluation;
+  Dataset: Dataset;
+  DatasetEntry: DatasetEntry;
+  Organization: Organization;
+  OrganizationUser: OrganizationUser;
+  WorldChampEntrant: WorldChampEntrant;
+  LoggedCall: LoggedCall;
+  LoggedCallModelResponse: LoggedCallModelResponse;
+  LoggedCallTag: LoggedCallTag;
+  ApiKey: ApiKey;
+  Account: Account;
+  Session: Session;
+  User: User;
+  VerificationToken: VerificationToken;
+}
+
 const globalForPrisma = globalThis as unknown as {
   prisma: PrismaClient | undefined;
 };
@@ -14,4 +69,12 @@ export const prisma =
         : ["error"],
   });
 
+export const kysely = new Kysely<DB>({
+  dialect: new PostgresDialect({
+    pool: new Pool({
+      connectionString: env.DATABASE_URL,
+    }),
+  }),
+});
+
 if (env.NODE_ENV !== "production") globalForPrisma.prisma = prisma;

app/src/server/scripts/backfillApiKeys.ts

@@ -0,0 +1,33 @@
+import { type Prisma } from "@prisma/client";
+import { prisma } from "~/server/db";
+import { generateApiKey } from "~/server/utils/generateApiKey";
+
+console.log("backfilling api keys");
+
+const organizations = await prisma.organization.findMany({
+  include: {
+    apiKeys: true,
+  },
+});
+
+console.log(`found ${organizations.length} organizations`);
+
+const apiKeysToCreate: Prisma.ApiKeyCreateManyInput[] = [];
+
+for (const org of organizations) {
+  if (!org.apiKeys.length) {
+    apiKeysToCreate.push({
+      name: "Default API Key",
+      organizationId: org.id,
+      apiKey: generateApiKey(),
+    });
+  }
+}
+
+console.log(`creating ${apiKeysToCreate.length} api keys`);
+
+await prisma.apiKey.createMany({
+  data: apiKeysToCreate,
+});
+
+console.log("done");

app/src/server/scripts/client-codegen.ts

@@ -0,0 +1,32 @@
+import "dotenv/config";
+import { openApiDocument } from "~/pages/api/openapi.json";
+import fs from "fs";
+import path from "path";
+import { execSync } from "child_process";
+
+console.log("Exporting public OpenAPI schema to client-libs/schema.json");
+
+const scriptPath = import.meta.url.replace("file://", "");
+const clientLibsPath = path.join(path.dirname(scriptPath), "../../../../client-libs");
+
+const schemaPath = path.join(clientLibsPath, "schema.json");
+
+console.log("Exporting schema");
+fs.writeFileSync(schemaPath, JSON.stringify(openApiDocument, null, 2), "utf-8");
+
+console.log("Generating Typescript client");
+
+const tsClientPath = path.join(clientLibsPath, "typescript/codegen");
+
+fs.rmSync(tsClientPath, { recursive: true, force: true });
+
+execSync(
+  `pnpm dlx @openapitools/openapi-generator-cli generate -i "${schemaPath}" -g typescript-axios -o "${tsClientPath}"`,
+  {
+    stdio: "inherit",
+  },
+);
+
+console.log("Done!");
+
+process.exit(0);

app/src/server/scripts/test-queries.ts

@@ -0,0 +1,63 @@
+import dayjs from "dayjs";
+import { prisma } from "../db";
+
+const projectId = "1234";
+
+// Find all calls in the last 24 hours
+const responses = await prisma.loggedCall.findMany({
+  where: {
+    organizationId: projectId,
+    startTime: {
+      gt: dayjs()
+        .subtract(24 * 3600)
+        .toDate(),
+    },
+  },
+  include: {
+    modelResponse: true,
+  },
+  orderBy: {
+    startTime: "desc",
+  },
+});
+
+// Find all calls in the last 24 hours with promptId 'hello-world'
+const helloWorld = await prisma.loggedCall.findMany({
+  where: {
+    organizationId: projectId,
+    startTime: {
+      gt: dayjs()
+        .subtract(24 * 3600)
+        .toDate(),
+    },
+    tags: {
+      some: {
+        name: "promptId",
+        value: "hello-world",
+      },
+    },
+  },
+  include: {
+    modelResponse: true,
+  },
+  orderBy: {
+    startTime: "desc",
+  },
+});
+
+// Total spent on OpenAI in the last month
+const totalSpent = await prisma.loggedCallModelResponse.aggregate({
+  _sum: {
+    totalCost: true,
+  },
+  where: {
+    originalLoggedCall: {
+      organizationId: projectId,
+    },
+    startTime: {
+      gt: dayjs()
+        .subtract(30 * 24 * 3600)
+        .toDate(),
+    },
+  },
+});

app/src/server/tasks/queryModel.task.ts

@@ -1,10 +1,10 @@
 import { type Prisma } from "@prisma/client";
-import { type JsonObject } from "type-fest";
+import { type JsonValue, type JsonObject } from "type-fest";
 import modelProviders from "~/modelProviders/modelProviders";
 import { prisma } from "~/server/db";
 import { wsConnection } from "~/utils/wsConnection";
 import { runEvalsForOutput } from "../utils/evaluations";
-import hashPrompt from "../utils/hashPrompt";
+import hashObject from "../utils/hashObject";
 import defineTask from "./defineTask";
 import parsePromptConstructor from "~/promptConstructor/parse";
 
@@ -99,7 +99,7 @@ export const queryModel = defineTask<QueryModelJob>("queryModel", async (task) =
       }
     : null;
 
-  const inputHash = hashPrompt(prompt);
+  const inputHash = hashObject(prompt as JsonValue);
 
   let modelResponse = await prisma.modelResponse.create({
     data: {

app/src/server/tasks/worker.ts

@@ -17,7 +17,7 @@ const taskList = registeredTasks.reduce((acc, task) => {
 // Run a worker to execute jobs:
 const runner = await run({
   connectionString: env.DATABASE_URL,
-  concurrency: 50,
+  concurrency: 10,
   // Install signal handlers for graceful shutdown on SIGINT, SIGTERM, etc
   noHandleSignals: false,
   pollInterval: 1000,

app/src/server/utils/generateApiKey.ts

@@ -0,0 +1,5 @@
+import cryptoRandomString from "crypto-random-string";
+
+const KEY_LENGTH = 42;
+
+export const generateApiKey = () => `opc_${cryptoRandomString({ length: KEY_LENGTH })}`;

app/src/server/utils/generateNewCell.ts

@@ -1,7 +1,7 @@
 import { Prisma } from "@prisma/client";
 import { prisma } from "../db";
 import { type JsonObject } from "type-fest";
-import hashPrompt from "./hashPrompt";
+import hashObject from "./hashObject";
 import { omit } from "lodash-es";
 import { queueQueryModel } from "../tasks/queryModel.task";
 import parsePromptConstructor from "~/promptConstructor/parse";
@@ -57,7 +57,7 @@ export const generateNewCell = async (
     return;
   }
 
-  const inputHash = hashPrompt(parsedConstructFn);
+  const inputHash = hashObject(parsedConstructFn);
 
   cell = await prisma.scenarioVariantCell.create({
     data: {

app/src/server/utils/hashObject.ts

@@ -1,6 +1,5 @@
 import crypto from "crypto";
 import { type JsonValue } from "type-fest";
-import { ParsedPromptConstructor } from "~/promptConstructor/parse";
 
 function sortKeys(obj: JsonValue): JsonValue {
   if (typeof obj !== "object" || obj === null) {
@@ -25,9 +24,17 @@ function sortKeys(obj: JsonValue): JsonValue {
   return sortedObj;
 }
 
-export default function hashPrompt(prompt: ParsedPromptConstructor<any>): string {
+export function hashRequest(organizationId: string, reqPayload: JsonValue): string {
+  const obj = {
+    organizationId,
+    reqPayload,
+  };
+  return hashObject(obj);
+}
+
+export default function hashObject(obj: JsonValue): string {
   // Sort object keys recursively
-  const sortedObj = sortKeys(prompt as unknown as JsonValue);
+  const sortedObj = sortKeys(obj);
 
   // Convert to JSON and hash it
   const str = JSON.stringify(sortedObj);

app/src/server/utils/openai.ts

@@ -1,6 +1,13 @@
 import { env } from "~/env.mjs";
 
-import OpenAI from "openai";
+import { default as OriginalOpenAI } from "openai";
+// import { OpenAI } from "openpipe";
+
+const openAIConfig = { apiKey: env.OPENAI_API_KEY ?? "dummy-key" };
 
 // Set a dummy key so it doesn't fail at build time
-export const openai = new OpenAI({ apiKey: env.OPENAI_API_KEY ?? "dummy-key" });
+// export const openai = env.OPENPIPE_API_KEY
+//   ? new OpenAI.OpenAI(openAIConfig)
+//   : new OriginalOpenAI(openAIConfig);
+
+export const openai = new OriginalOpenAI(openAIConfig);

app/src/server/utils/userOrg.ts

@@ -1,4 +1,5 @@
 import { prisma } from "~/server/db";
+import { generateApiKey } from "./generateApiKey";
 
 export default async function userOrg(userId: string) {
   return await prisma.organization.upsert({
@@ -14,6 +15,14 @@ export default async function userOrg(userId: string) {
           role: "ADMIN",
         },
       },
+      apiKeys: {
+        create: [
+          {
+            name: "Default API Key",
+            apiKey: generateApiKey(),
+          },
+        ],
+      },
     },
   });
 }

app/src/state/store.ts

@@ -14,6 +14,8 @@ export type State = {
   api: APIClient | null;
   setApi: (api: APIClient) => void;
   sharedVariantEditor: SharedVariantEditorSlice;
+  selectedOrgId: string | null;
+  setSelectedOrgId: (orgId: string) => void;
 };
 
 export type SliceCreator<T> = StateCreator<State, [["zustand/immer", never]], [], T>;
@@ -39,6 +41,11 @@ const useBaseStore = create<State, [["zustand/immer", never]]>(
         state.drawerOpen = false;
       }),
     sharedVariantEditor: createVariantEditorSlice(set, get, ...rest),
+    selectedOrgId: null,
+    setSelectedOrgId: (orgId: string) =>
+      set((state) => {
+        state.selectedOrgId = orgId;
+      }),
   })),
 );
 

app/src/theme/ChakraThemeProvider.tsx

@@ -1,6 +1,5 @@
-import { extendTheme } from "@chakra-ui/react";
+import { extendTheme, defineStyleConfig, ChakraProvider } from "@chakra-ui/react";
 import "@fontsource/inconsolata";
-import { ChakraProvider } from "@chakra-ui/react";
 import { modalAnatomy } from "@chakra-ui/anatomy";
 import { createMultiStyleConfigHelpers } from "@chakra-ui/styled-system";
 
@@ -18,6 +17,13 @@ const modalTheme = defineMultiStyleConfig({
   }),
 });
 
+const Divider = defineStyleConfig({
+  baseStyle: {
+    borderColor: "gray.300",
+    backgroundColor: "gray.300",
+  },
+});
+
 const theme = extendTheme({
   styles: {
     global: (props: { colorMode: "dark" | "light" }) => ({
@@ -53,6 +59,7 @@ const theme = extendTheme({
       },
     },
     Modal: modalTheme,
+    Divider,
   },
 });
 

app/src/utils/accessControl.ts

@@ -16,6 +16,68 @@ export const requireNothing = (ctx: TRPCContext) => {
   ctx.markAccessControlRun();
 };
 
+export const requireIsOrgAdmin = async (organizationId: string, ctx: TRPCContext) => {
+  const userId = ctx.session?.user.id;
+  if (!userId) {
+    throw new TRPCError({ code: "UNAUTHORIZED" });
+  }
+
+  const isAdmin = await prisma.organizationUser.findFirst({
+    where: {
+      userId,
+      organizationId,
+      role: "ADMIN",
+    },
+  });
+
+  if (!isAdmin) {
+    throw new TRPCError({ code: "UNAUTHORIZED" });
+  }
+
+  ctx.markAccessControlRun();
+};
+
+export const requireCanViewOrganization = async (organizationId: string, ctx: TRPCContext) => {
+  const userId = ctx.session?.user.id;
+  if (!userId) {
+    throw new TRPCError({ code: "UNAUTHORIZED" });
+  }
+
+  const canView = await prisma.organizationUser.findFirst({
+    where: {
+      userId,
+      organizationId,
+    },
+  });
+
+  if (!canView) {
+    throw new TRPCError({ code: "UNAUTHORIZED" });
+  }
+
+  ctx.markAccessControlRun();
+};
+
+export const requireCanModifyOrganization = async (organizationId: string, ctx: TRPCContext) => {
+  const userId = ctx.session?.user.id;
+  if (!userId) {
+    throw new TRPCError({ code: "UNAUTHORIZED" });
+  }
+
+  const canModify = await prisma.organizationUser.findFirst({
+    where: {
+      userId,
+      organizationId,
+      role: { in: [OrganizationUserRole.ADMIN, OrganizationUserRole.MEMBER] },
+    },
+  });
+
+  if (!canModify) {
+    throw new TRPCError({ code: "UNAUTHORIZED" });
+  }
+
+  ctx.markAccessControlRun();
+};
+
 export const requireCanViewDataset = async (datasetId: string, ctx: TRPCContext) => {
   const dataset = await prisma.dataset.findFirst({
     where: {

app/src/utils/dayjs.ts

@@ -1,9 +1,11 @@
 import dayjs from "dayjs";
 import duration from "dayjs/plugin/duration";
 import relativeTime from "dayjs/plugin/relativeTime";
+import timezone from "dayjs/plugin/timezone";
 
 dayjs.extend(duration);
 dayjs.extend(relativeTime);
+dayjs.extend(timezone);
 
 export const formatTimePast = (date: Date) =>
   dayjs.duration(dayjs(date).diff(dayjs())).humanize(true);

app/src/utils/hooks.ts

@@ -2,6 +2,15 @@ import { useRouter } from "next/router";
 import { type RefObject, useCallback, useEffect, useRef, useState } from "react";
 import { api } from "~/utils/api";
 import { NumberParam, useQueryParam, withDefault } from "use-query-params";
+import { useAppStore } from "~/state/store";
+
+export const useExperiments = () => {
+  const selectedOrgId = useAppStore((state) => state.selectedOrgId);
+  return api.experiments.list.useQuery(
+    { organizationId: selectedOrgId ?? "" },
+    { enabled: !!selectedOrgId },
+  );
+};
 
 export const useExperiment = () => {
   const router = useRouter();
@@ -17,6 +26,14 @@ export const useExperimentAccess = () => {
   return useExperiment().data?.access ?? { canView: false, canModify: false };
 };
 
+export const useDatasets = () => {
+  const selectedOrgId = useAppStore((state) => state.selectedOrgId);
+  return api.datasets.list.useQuery(
+    { organizationId: selectedOrgId ?? "" },
+    { enabled: !!selectedOrgId },
+  );
+};
+
 export const useDataset = () => {
   const router = useRouter();
   const dataset = api.datasets.get.useQuery(
@@ -132,3 +149,8 @@ export const useScenario = (scenarioId: string) => {
 };
 
 export const useVisibleScenarioIds = () => useScenarios().data?.scenarios.map((s) => s.id) ?? [];
+
+export const useSelectedOrg = () => {
+  const selectedOrgId = useAppStore((state) => state.selectedOrgId);
+  return api.organizations.get.useQuery({ id: selectedOrgId ?? "" }, { enabled: !!selectedOrgId });
+};

app/src/utils/useSocket.ts

@@ -11,7 +11,6 @@ export default function useSocket<T>(channel?: string | null) {
   useEffect(() => {
     if (!channel) return;
 
-    console.log("connecting to channel", channel);
     // Create websocket connection
     socketRef.current = io(url);