Merge pull request #123 from OpenPipe/add-openapi

Add Logged Calls and projects

app/.env.example

@@ -31,3 +31,6 @@ NEXT_PUBLIC_HOST="http://localhost:3000"
 # Next Auth Github Provider
 GITHUB_CLIENT_ID="your_client_id"
 GITHUB_CLIENT_SECRET="your_secret"
+
+OPENPIPE_BASE_URL="http://localhost:3000/api"
+OPENPIPE_API_KEY="your_key"

app/@types/nextjs-routes.d.ts

@@ -12,8 +12,10 @@ declare module "nextjs-routes" {
 
   export type Route =
     | StaticRoute<"/account/signin">
+    | DynamicRoute<"/api/[...trpc]", { "trpc": string[] }>
     | DynamicRoute<"/api/auth/[...nextauth]", { "nextauth": string[] }>
     | StaticRoute<"/api/experiments/og-image">
+    | StaticRoute<"/api/openapi">
     | StaticRoute<"/api/sentry-example-api">
     | DynamicRoute<"/api/trpc/[trpc]", { "trpc": string }>
     | DynamicRoute<"/data/[id]", { "id": string }>
@@ -21,6 +23,8 @@ declare module "nextjs-routes" {
     | DynamicRoute<"/experiments/[id]", { "id": string }>
     | StaticRoute<"/experiments">
     | StaticRoute<"/">
+    | StaticRoute<"/logged-calls">
+    | StaticRoute<"/project/settings">
     | StaticRoute<"/sentry-example-page">
     | StaticRoute<"/world-champs">
     | StaticRoute<"/world-champs/signup">;

app/Dockerfile

@@ -23,6 +23,7 @@ ARG NEXT_PUBLIC_SOCKET_URL
 ARG NEXT_PUBLIC_HOST
 ARG NEXT_PUBLIC_SENTRY_DSN
 ARG SENTRY_AUTH_TOKEN
+ARG NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS
 
 WORKDIR /app
 COPY --from=deps /app/node_modules ./node_modules

app/openapitools.json

@@ -0,0 +1,7 @@
+{
+  "$schema": "./node_modules/@openapitools/openapi-generator-cli/config.schema.json",
+  "spaces": 2,
+  "generator-cli": {
+    "version": "6.6.0"
+  }
+}

app/package.json

@@ -16,7 +16,7 @@
     "postinstall": "prisma generate",
     "lint": "next lint",
     "start": "next start",
-    "codegen": "tsx src/codegen/export-openai-types.ts",
+    "codegen": "tsx src/server/scripts/client-codegen.ts",
     "seed": "tsx prisma/seed.ts",
     "check": "concurrently 'pnpm lint' 'pnpm tsc' 'pnpm prettier . --check'",
     "test": "pnpm vitest --no-threads"
@@ -50,6 +50,7 @@
     "chroma-js": "^2.4.2",
     "concurrently": "^8.2.0",
     "cors": "^2.8.5",
+    "crypto-random-string": "^5.0.0",
     "dayjs": "^1.11.8",
     "dedent": "^1.0.1",
     "dotenv": "^16.3.1",
@@ -62,12 +63,16 @@
     "json-schema-to-typescript": "^13.0.2",
     "json-stringify-pretty-compact": "^4.0.0",
     "jsonschema": "^1.4.1",
+    "kysely": "^0.26.1",
     "lodash-es": "^4.17.21",
+    "lucide-react": "^0.265.0",
     "next": "^13.4.2",
     "next-auth": "^4.22.1",
     "next-query-params": "^4.2.3",
+    "nextjs-cors": "^2.1.2",
     "nextjs-routes": "^2.0.1",
     "openai": "4.0.0-beta.7",
+    "pg": "^8.11.2",
     "pluralize": "^8.0.0",
     "posthog-js": "^1.75.3",
     "posthog-node": "^3.1.1",
@@ -83,10 +88,12 @@
     "react-syntax-highlighter": "^15.5.0",
     "react-textarea-autosize": "^8.5.0",
     "recast": "^0.23.3",
+    "recharts": "^2.7.2",
     "replicate": "^0.12.3",
     "socket.io": "^4.7.1",
     "socket.io-client": "^4.7.1",
     "superjson": "1.12.2",
+    "trpc-openapi": "^1.2.0",
     "tsx": "^3.12.7",
     "type-fest": "^4.0.0",
     "use-query-params": "^2.2.1",
@@ -106,6 +113,7 @@
     "@types/json-schema": "^7.0.12",
     "@types/lodash-es": "^4.17.8",
     "@types/node": "^18.16.0",
+    "@types/pg": "^8.10.2",
     "@types/pluralize": "^0.0.30",
     "@types/prismjs": "^1.26.0",
     "@types/react": "^18.2.6",

app/prisma/migrations/20230806024615_add_logged_calls_and_api_keys/migration.sql

@@ -0,0 +1,90 @@
+-- CreateTable
+CREATE TABLE "LoggedCall" (
+    "id" UUID NOT NULL,
+    "startTime" TIMESTAMP(3) NOT NULL,
+    "cacheHit" BOOLEAN NOT NULL,
+    "modelResponseId" UUID NOT NULL,
+    "organizationId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "LoggedCall_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "LoggedCallModelResponse" (
+    "id" UUID NOT NULL,
+    "reqPayload" JSONB NOT NULL,
+    "respStatus" INTEGER,
+    "respPayload" JSONB,
+    "error" TEXT,
+    "startTime" TIMESTAMP(3) NOT NULL,
+    "endTime" TIMESTAMP(3) NOT NULL,
+    "cacheKey" TEXT,
+    "durationMs" INTEGER,
+    "inputTokens" INTEGER,
+    "outputTokens" INTEGER,
+    "finishReason" TEXT,
+    "completionId" TEXT,
+    "totalCost" DECIMAL(18,12),
+    "originalLoggedCallId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "LoggedCallModelResponse_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "LoggedCallTag" (
+    "id" UUID NOT NULL,
+    "name" TEXT NOT NULL,
+    "value" TEXT,
+    "loggedCallId" UUID NOT NULL,
+
+    CONSTRAINT "LoggedCallTag_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "ApiKey" (
+    "id" UUID NOT NULL,
+    "name" TEXT NOT NULL,
+    "apiKey" TEXT NOT NULL,
+    "organizationId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "ApiKey_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "LoggedCall_startTime_idx" ON "LoggedCall"("startTime");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "LoggedCallModelResponse_originalLoggedCallId_key" ON "LoggedCallModelResponse"("originalLoggedCallId");
+
+-- CreateIndex
+CREATE INDEX "LoggedCallModelResponse_cacheKey_idx" ON "LoggedCallModelResponse"("cacheKey");
+
+-- CreateIndex
+CREATE INDEX "LoggedCallTag_name_idx" ON "LoggedCallTag"("name");
+
+-- CreateIndex
+CREATE INDEX "LoggedCallTag_name_value_idx" ON "LoggedCallTag"("name", "value");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "ApiKey_apiKey_key" ON "ApiKey"("apiKey");
+
+-- AddForeignKey
+ALTER TABLE "LoggedCall" ADD CONSTRAINT "LoggedCall_modelResponseId_fkey" FOREIGN KEY ("modelResponseId") REFERENCES "LoggedCallModelResponse"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "LoggedCall" ADD CONSTRAINT "LoggedCall_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "LoggedCallModelResponse" ADD CONSTRAINT "LoggedCallModelResponse_originalLoggedCallId_fkey" FOREIGN KEY ("originalLoggedCallId") REFERENCES "LoggedCall"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "LoggedCallTag" ADD CONSTRAINT "LoggedCallTag_loggedCallId_fkey" FOREIGN KEY ("loggedCallId") REFERENCES "LoggedCall"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "ApiKey" ADD CONSTRAINT "ApiKey_organizationId_fkey" FOREIGN KEY ("organizationId") REFERENCES "Organization"("id") ON DELETE CASCADE ON UPDATE CASCADE;

app/prisma/migrations/20230807044936_add_name_to_organization/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Organization" ADD COLUMN     "name" TEXT NOT NULL DEFAULT 'Project 1';

app/prisma/migrations/20230808034313_make_model_response_optional/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LoggedCall" ALTER COLUMN "modelResponseId" DROP NOT NULL;

app/prisma/schema.prisma

@@ -200,16 +200,21 @@ model DatasetEntry {
     updatedAt DateTime @updatedAt
 }
 
+// TODO rename Organization to Project
 model Organization {
     id                String  @id @default(uuid()) @db.Uuid
+    name              String  @default("Project 1")
+
     personalOrgUserId String? @unique @db.Uuid
-    PersonalOrgUser   User?   @relation(fields: [personalOrgUserId], references: [id], onDelete: Cascade)
+    personalOrgUser   User?   @relation(fields: [personalOrgUserId], references: [id], onDelete: Cascade)
 
     createdAt         DateTime           @default(now())
     updatedAt         DateTime           @updatedAt
     organizationUsers OrganizationUser[]
     experiments       Experiment[]
     datasets          Dataset[]
+    loggedCalls       LoggedCall[]
+    apiKeys           ApiKey[]
 }
 
 enum OrganizationUserRole {
@@ -249,6 +254,99 @@ model WorldChampEntrant {
     @@unique([userId])
 }
 
+model LoggedCall {
+    id String @id @default(uuid()) @db.Uuid
+
+    startTime DateTime
+
+    // True if this call was served from the cache, false otherwise
+    cacheHit Boolean
+
+    // A LoggedCall is always associated with a LoggedCallModelResponse. If this
+    // is a cache miss, we create a new LoggedCallModelResponse.
+    // If it's a cache hit, it's a pre-existing LoggedCallModelResponse.    
+    modelResponseId String?                 @db.Uuid
+    modelResponse   LoggedCallModelResponse? @relation(fields: [modelResponseId], references: [id], onDelete: Cascade)
+
+    // The responses created by this LoggedCall. Will be empty if this LoggedCall was a cache hit.
+    createdResponses LoggedCallModelResponse[] @relation(name: "ModelResponseOriginalCall")
+
+    organizationId String        @db.Uuid
+    organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+
+    tags LoggedCallTag[]
+
+    createdAt DateTime @default(now())
+    updatedAt DateTime @updatedAt
+
+    @@index([startTime])
+}
+
+model LoggedCallModelResponse {
+    id String @id @default(uuid()) @db.Uuid
+
+    reqPayload Json
+
+    // The HTTP status returned by the model provider
+    respStatus  Int?
+    respPayload Json?
+
+    // Should be null if the request was successful, and some string if the request failed.
+    error String?
+
+    startTime DateTime
+    endTime   DateTime
+
+    // Note: the function to calculate the cacheKey should include the project
+    // ID so we don't share cached responses between projects, which could be an
+    // attack vector. Also, we should only set the cacheKey on the model if the
+    // request was successful.
+    cacheKey String?
+
+    // Derived fields
+    durationMs   Int?
+    inputTokens  Int?
+    outputTokens Int?
+    finishReason String?
+    completionId String?
+    totalCost    Decimal? @db.Decimal(18, 12)
+
+    // The LoggedCall that created this LoggedCallModelResponse
+    originalLoggedCallId String     @unique @db.Uuid
+    originalLoggedCall   LoggedCall @relation(name: "ModelResponseOriginalCall", fields: [originalLoggedCallId], references: [id], onDelete: Cascade)
+
+    createdAt   DateTime     @default(now())
+    updatedAt   DateTime     @updatedAt
+    loggedCalls LoggedCall[]
+
+    @@index([cacheKey])
+}
+
+model LoggedCallTag {
+    id    String @id @default(uuid()) @db.Uuid
+    name  String
+    value String?
+
+    loggedCallId String        @db.Uuid
+    loggedCall   LoggedCall @relation(fields: [loggedCallId], references: [id], onDelete: Cascade)
+
+    @@index([name])
+    @@index([name, value])
+}
+
+model ApiKey {
+    id String @id @default(uuid()) @db.Uuid
+
+    name   String
+    apiKey String @unique
+
+    organizationId String        @db.Uuid
+    organization   Organization? @relation(fields: [organizationId], references: [id], onDelete: Cascade)
+
+    createdAt DateTime @default(now())
+    updatedAt DateTime @updatedAt
+}
+
 model Account {
     id                       String  @id @default(uuid()) @db.Uuid
     userId                   String  @db.Uuid

app/src/components/CopiableCode.tsx

@@ -0,0 +1,40 @@
+import { HStack, Icon, IconButton, Tooltip, Text } from "@chakra-ui/react";
+import { useState } from "react";
+import { MdContentCopy } from "react-icons/md";
+import { useHandledAsyncCallback } from "~/utils/hooks";
+
+const CopiableCode = ({ code }: { code: string }) => {
+  const [copied, setCopied] = useState(false);
+
+  const [copyToClipboard] = useHandledAsyncCallback(async () => {
+    await navigator.clipboard.writeText(code);
+    setCopied(true);
+  }, [code]);
+  return (
+    <HStack
+      backgroundColor="blackAlpha.800"
+      color="white"
+      borderRadius={4}
+      padding={3}
+      w="full"
+      justifyContent="space-between"
+    >
+      <Text fontFamily="inconsolata" fontWeight="bold" letterSpacing={0.5}>
+        {code}
+      </Text>
+      <Tooltip closeOnClick={false} label={copied ? "Copied!" : "Copy to clipboard"}>
+        <IconButton
+          aria-label="Copy"
+          icon={<Icon as={MdContentCopy} boxSize={5} />}
+          size="xs"
+          colorScheme="white"
+          variant="ghost"
+          onClick={copyToClipboard}
+          onMouseLeave={() => setCopied(false)}
+        />
+      </Tooltip>
+    </HStack>
+  );
+};
+
+export default CopiableCode;

app/src/components/ExperimentSettingsDrawer/DeleteButton.tsx

@@ -14,6 +14,7 @@ import {
 import { useRouter } from "next/router";
 import { useRef } from "react";
 import { BsTrash } from "react-icons/bs";
+import { useAppStore } from "~/state/store";
 import { api } from "~/utils/api";
 import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
 
@@ -23,6 +24,8 @@ export const DeleteButton = () => {
   const utils = api.useContext();
   const router = useRouter();
 
+  const closeDrawer = useAppStore((s) => s.closeDrawer);
+
   const { isOpen, onOpen, onClose } = useDisclosure();
   const cancelRef = useRef<HTMLButtonElement>(null);
 
@@ -31,6 +34,8 @@ export const DeleteButton = () => {
     await mutation.mutateAsync({ id: experiment.data.id });
     await utils.experiments.list.invalidate();
     await router.push({ pathname: "/experiments" });
+    closeDrawer();
+
     onClose();
   }, [mutation, experiment.data?.id, router]);
 

app/src/components/StatsCard.tsx

@@ -0,0 +1,26 @@
+import { VStack, HStack, type StackProps, Text, Divider } from "@chakra-ui/react";
+import Link, { type LinkProps } from "next/link";
+
+const StatsCard = ({
+  title,
+  href,
+  children,
+  ...rest
+}: { title: string; href: string } & StackProps & LinkProps) => {
+  return (
+    <VStack flex={1} borderWidth={1} padding={4} borderRadius={4} borderColor="gray.300" {...rest}>
+      <HStack w="full" justifyContent="space-between">
+        <Text fontSize="md" fontWeight="bold">
+          {title}
+        </Text>
+        <Link href={href}>
+          <Text color="blue">View all</Text>
+        </Link>
+      </HStack>
+      <Divider />
+      {children}
+    </VStack>
+  );
+};
+
+export default StatsCard;

app/src/components/dashboard/LoggedCallTable.tsx

@@ -0,0 +1,201 @@
+import {
+  Box,
+  Card,
+  CardHeader,
+  Heading,
+  Table,
+  Tbody,
+  Td,
+  Th,
+  Thead,
+  Tr,
+  Tooltip,
+  Collapse,
+  HStack,
+  VStack,
+  IconButton,
+  useToast,
+  Icon,
+  Button,
+  ButtonGroup,
+} from "@chakra-ui/react";
+import dayjs from "dayjs";
+import relativeTime from "dayjs/plugin/relativeTime";
+import { ChevronUpIcon, ChevronDownIcon, CopyIcon } from "lucide-react";
+import { useMemo, useState } from "react";
+import { type RouterOutputs, api } from "~/utils/api";
+import SyntaxHighlighter from "react-syntax-highlighter";
+import { atelierCaveLight } from "react-syntax-highlighter/dist/cjs/styles/hljs";
+import stringify from "json-stringify-pretty-compact";
+import Link from "next/link";
+
+dayjs.extend(relativeTime);
+
+type LoggedCall = RouterOutputs["dashboard"]["loggedCalls"][0];
+
+const FormattedJson = ({ json }: { json: any }) => {
+  const jsonString = stringify(json, { maxLength: 40 });
+  const toast = useToast();
+
+  const copyToClipboard = async (text: string) => {
+    try {
+      await navigator.clipboard.writeText(text);
+      toast({
+        title: "Copied to clipboard",
+        status: "success",
+        duration: 2000,
+      });
+    } catch (err) {
+      toast({
+        title: "Failed to copy to clipboard",
+        status: "error",
+        duration: 2000,
+      });
+    }
+  };
+
+  return (
+    <Box position="relative" fontSize="sm" borderRadius="md" overflow="hidden">
+      <SyntaxHighlighter
+        customStyle={{ overflowX: "unset" }}
+        language="json"
+        style={atelierCaveLight}
+        lineProps={{
+          style: { wordBreak: "break-all", whiteSpace: "pre-wrap" },
+        }}
+        wrapLines
+      >
+        {jsonString}
+      </SyntaxHighlighter>
+      <IconButton
+        aria-label="Copy"
+        icon={<CopyIcon />}
+        position="absolute"
+        top={1}
+        right={1}
+        size="xs"
+        variant="ghost"
+        onClick={() => void copyToClipboard(jsonString)}
+      />
+    </Box>
+  );
+};
+
+function TableRow({
+  loggedCall,
+  isExpanded,
+  onToggle,
+}: {
+  loggedCall: LoggedCall;
+  isExpanded: boolean;
+  onToggle: () => void;
+}) {
+  const isError = loggedCall.modelResponse?.respStatus !== 200;
+  const timeAgo = dayjs(loggedCall.startTime).fromNow();
+  const fullTime = dayjs(loggedCall.startTime).toString();
+
+  const model = useMemo(
+    () => loggedCall.tags.find((tag) => tag.name.startsWith("$model"))?.value,
+    [loggedCall.tags],
+  );
+
+  return (
+    <>
+      <Tr
+        onClick={onToggle}
+        key={loggedCall.id}
+        _hover={{ bgColor: "gray.100", cursor: "pointer" }}
+        sx={{
+          "> td": { borderBottom: "none" },
+        }}
+      >
+        <Td>
+          <Icon boxSize={6} as={isExpanded ? ChevronUpIcon : ChevronDownIcon} />
+        </Td>
+        <Td>
+          <Tooltip label={fullTime} placement="top">
+            <Box whiteSpace="nowrap" minW="120px">
+              {timeAgo}
+            </Box>
+          </Tooltip>
+        </Td>
+        <Td width="100%">{model}</Td>
+        <Td isNumeric>{((loggedCall.modelResponse?.durationMs ?? 0) / 1000).toFixed(2)}s</Td>
+        <Td isNumeric>{loggedCall.modelResponse?.inputTokens}</Td>
+        <Td isNumeric>{loggedCall.modelResponse?.outputTokens}</Td>
+        <Td sx={{ color: isError ? "red.500" : "green.500", fontWeight: "semibold" }} isNumeric>
+          {loggedCall.modelResponse?.respStatus ?? "No response"}
+        </Td>
+      </Tr>
+      <Tr>
+        <Td colSpan={8} p={0}>
+          <Collapse in={isExpanded} unmountOnExit={true}>
+            <VStack p={4} align="stretch">
+              <HStack align="stretch">
+                <VStack flex={1} align="stretch">
+                  <Heading size="sm">Input</Heading>
+                  <FormattedJson json={loggedCall.modelResponse?.reqPayload} />
+                </VStack>
+                <VStack flex={1} align="stretch">
+                  <Heading size="sm">Output</Heading>
+                  <FormattedJson json={loggedCall.modelResponse?.respPayload} />
+                </VStack>
+              </HStack>
+              <ButtonGroup alignSelf="flex-end">
+                <Button as={Link} colorScheme="blue" href={{ pathname: "/experiments" }}>
+                  Experiments
+                </Button>
+              </ButtonGroup>
+            </VStack>
+          </Collapse>
+        </Td>
+      </Tr>
+    </>
+  );
+}
+
+export default function LoggedCallTable() {
+  const [expandedRow, setExpandedRow] = useState<string | null>(null);
+  const loggedCalls = api.dashboard.loggedCalls.useQuery({});
+
+  return (
+    <Card variant="outline" width="100%" overflow="hidden">
+      <CardHeader>
+        <Heading as="h3" size="sm">
+          Logged Calls
+        </Heading>
+      </CardHeader>
+      <Table>
+        <Thead>
+          <Tr>
+            <Th />
+            <Th>Time</Th>
+            <Th>Model</Th>
+            <Th isNumeric>Duration</Th>
+            <Th isNumeric>Input tokens</Th>
+            <Th isNumeric>Output tokens</Th>
+            <Th isNumeric>Status</Th>
+          </Tr>
+        </Thead>
+        <Tbody>
+          {loggedCalls.data?.map((loggedCall) => {
+            return (
+              <TableRow
+                key={loggedCall.id}
+                loggedCall={loggedCall}
+                isExpanded={loggedCall.id === expandedRow}
+                onToggle={() => {
+                  if (loggedCall.id === expandedRow) {
+                    setExpandedRow(null);
+                  } else {
+                    setExpandedRow(loggedCall.id);
+                  }
+                }}
+              />
+            );
+          })}
+        </Tbody>
+      </Table>
+    </Card>
+  );
+}

app/src/components/datasets/DatasetCard.tsx

@@ -15,6 +15,7 @@ import { useRouter } from "next/router";
 import { BsPlusSquare } from "react-icons/bs";
 import { api } from "~/utils/api";
 import { useHandledAsyncCallback } from "~/utils/hooks";
+import { useAppStore } from "~/state/store";
 
 type DatasetData = {
   name: string;
@@ -71,11 +72,12 @@ const CountLabel = ({ label, count }: { label: string; count: number }) => {
 
 export const NewDatasetCard = () => {
   const router = useRouter();
+  const selectedOrgId = useAppStore((s) => s.selectedOrgId);
   const createMutation = api.datasets.create.useMutation();
   const [createDataset, isLoading] = useHandledAsyncCallback(async () => {
-    const newDataset = await createMutation.mutateAsync({ label: "New Dataset" });
+    const newDataset = await createMutation.mutateAsync({ organizationId: selectedOrgId ?? "" });
     await router.push({ pathname: "/data/[id]", query: { id: newDataset.id } });
-  }, [createMutation, router]);
+  }, [createMutation, router, selectedOrgId]);
 
   return (
     <AspectRatio ratio={1.2} w="full">

app/src/components/experiments/ExperimentCard.tsx

@@ -15,6 +15,7 @@ import { useRouter } from "next/router";
 import { BsPlusSquare } from "react-icons/bs";
 import { api } from "~/utils/api";
 import { useHandledAsyncCallback } from "~/utils/hooks";
+import { useAppStore } from "~/state/store";
 
 type ExperimentData = {
   testScenarioCount: number;
@@ -75,11 +76,17 @@ const CountLabel = ({ label, count }: { label: string; count: number }) => {
 
 export const NewExperimentCard = () => {
   const router = useRouter();
+  const selectedOrgId = useAppStore((s) => s.selectedOrgId);
   const createMutation = api.experiments.create.useMutation();
   const [createExperiment, isLoading] = useHandledAsyncCallback(async () => {
-    const newExperiment = await createMutation.mutateAsync({ label: "New Experiment" });
+    const newExperiment = await createMutation.mutateAsync({
-    await router.push({ pathname: "/experiments/[id]", query: { id: newExperiment.id } });
+      organizationId: selectedOrgId ?? "",
-  }, [createMutation, router]);
+    });
+    await router.push({
+      pathname: "/experiments/[id]",
+      query: { id: newExperiment.id },
+    });
+  }, [createMutation, router, selectedOrgId]);
 
   return (
     <AspectRatio ratio={1.2} w="full">

app/src/components/experiments/ExperimentHeaderButtons/useOnForkButtonPressed.tsx

@@ -3,18 +3,23 @@ import { api } from "~/utils/api";
 import { useExperiment, useHandledAsyncCallback } from "~/utils/hooks";
 import { signIn, useSession } from "next-auth/react";
 import { useRouter } from "next/router";
+import { useAppStore } from "~/state/store";
 
 export const useOnForkButtonPressed = () => {
   const router = useRouter();
 
   const user = useSession().data;
   const experiment = useExperiment();
+  const selectedOrgId = useAppStore((state) => state.selectedOrgId);
 
   const forkMutation = api.experiments.fork.useMutation();
 
   const [onFork, isForking] = useHandledAsyncCallback(async () => {
-    if (!experiment.data?.id) return;
+    if (!experiment.data?.id || !selectedOrgId) return;
-    const forkedExperimentId = await forkMutation.mutateAsync({ id: experiment.data.id });
+    const forkedExperimentId = await forkMutation.mutateAsync({
+      id: experiment.data.id,
+      organizationId: selectedOrgId,
+    });
     await router.push({ pathname: "/experiments/[id]", query: { id: forkedExperimentId } });
   }, [forkMutation, experiment.data?.id, router]);
 

app/src/components/nav/AppShell.tsx

@@ -7,48 +7,22 @@ import {
   Image,
   Text,
   Box,
-  type BoxProps,
   Link as ChakraLink,
   Flex,
 } from "@chakra-ui/react";
 import Head from "next/head";
-import Link, { type LinkProps } from "next/link";
+import Link from "next/link";
-import { BsGithub, BsPersonCircle } from "react-icons/bs";
+import { BsGearFill, BsGithub, BsPersonCircle } from "react-icons/bs";
-import { useRouter } from "next/router";
+import { IoStatsChartOutline } from "react-icons/io5";
-import { type IconType } from "react-icons";
 import { RiDatabase2Line, RiFlaskLine } from "react-icons/ri";
 import { signIn, useSession } from "next-auth/react";
 import UserMenu from "./UserMenu";
 import { env } from "~/env.mjs";
+import ProjectMenu from "./ProjectMenu";
+import NavSidebarOption from "./NavSidebarOption";
+import IconLink from "./IconLink";
 
-type IconLinkProps = BoxProps & LinkProps & { label?: string; icon: IconType; href: string };
+const Divider = () => <Box h="1px" bgColor="gray.300" w="full" />;
-
-const IconLink = ({ icon, label, href, color, ...props }: IconLinkProps) => {
-  const router = useRouter();
-  const isActive = href && router.pathname.startsWith(href);
-  return (
-    <Link href={href} style={{ width: "100%" }}>
-      <HStack
-        w="full"
-        p={4}
-        color={color}
-        as={ChakraLink}
-        bgColor={isActive ? "gray.200" : "transparent"}
-        _hover={{ bgColor: "gray.300", textDecoration: "none" }}
-        justifyContent="start"
-        cursor="pointer"
-        {...props}
-      >
-        <Icon as={icon} boxSize={6} mr={2} />
-        <Text fontWeight="bold" fontSize="sm">
-          {label}
-        </Text>
-      </HStack>
-    </Link>
-  );
-};
-
-const Divider = () => <Box h="1px" bgColor="gray.200" />;
 
 const NavSidebar = () => {
   const user = useSession().data;
@@ -56,22 +30,31 @@ const NavSidebar = () => {
   return (
     <VStack
       align="stretch"
-      bgColor="gray.100"
+      bgColor="gray.50"
       py={2}
+      px={2}
       pb={0}
       height="100%"
-      w={{ base: "56px", md: "200px" }}
+      w={{ base: "56px", md: "240px" }}
       overflow="hidden"
+      borderRightWidth={1}
+      borderColor="gray.300"
     >
-      <HStack as={Link} href="/" _hover={{ textDecoration: "none" }} spacing={0} px={4} py={2}>
+      <HStack as={Link} href="/" _hover={{ textDecoration: "none" }} spacing={0} px={2} py={2}>
         <Image src="/logo.svg" alt="" boxSize={6} mr={4} />
         <Heading size="md" fontFamily="inconsolata, monospace">
           OpenPipe
         </Heading>
       </HStack>
-      <VStack spacing={0} align="flex-start" overflowY="auto" overflowX="hidden" flex={1}>
+      <Divider />
+      <VStack align="flex-start" overflowY="auto" overflowX="hidden" flex={1}>
         {user != null && (
           <>
+            <ProjectMenu />
+            <Divider />
+            {env.NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS && (
+              <IconLink icon={IoStatsChartOutline} label="Logged Calls" href="/logged-calls" beta />
+            )}
             <IconLink icon={RiFlaskLine} label="Experiments" href="/experiments" />
             {env.NEXT_PUBLIC_SHOW_DATA && (
               <IconLink icon={RiDatabase2Line} label="Data" href="/data" />
@@ -79,29 +62,39 @@ const NavSidebar = () => {
           </>
         )}
         {user === null && (
-          <HStack
+          <NavSidebarOption>
-            w="full"
+            <HStack
-            p={4}
+              w="full"
-            as={ChakraLink}
+              p={4}
-            _hover={{ bgColor: "gray.300", textDecoration: "none" }}
+              as={ChakraLink}
-            justifyContent="start"
+              justifyContent="start"
-            cursor="pointer"
+              onClick={() => {
-            onClick={() => {
+                signIn("github").catch(console.error);
-              signIn("github").catch(console.error);
+              }}
-            }}
+            >
-          >
+              <Icon as={BsPersonCircle} boxSize={6} mr={2} />
-            <Icon as={BsPersonCircle} boxSize={6} mr={2} />
+              <Text fontWeight="bold" fontSize="sm">
-            <Text fontWeight="bold" fontSize="sm">
+                Sign In
-              Sign In
+              </Text>
-            </Text>
+            </HStack>
-          </HStack>
+          </NavSidebarOption>
         )}
       </VStack>
-      {user ? (
+      <VStack w="full" alignItems="flex-start" spacing={0}>
-        <UserMenu user={user} borderColor={"gray.200"} borderTopWidth={1} borderBottomWidth={1} />
+        <Text
-      ) : (
+          pl={2}
-        <Divider />
+          pb={2}
-      )}
+          fontSize="xs"
+          fontWeight="bold"
+          color="gray.500"
+          display={{ base: "none", md: "flex" }}
+        >
+          CONFIGURATION
+        </Text>
+        <IconLink icon={BsGearFill} label="Project Settings" href="/project/settings" />
+      </VStack>
+      {user && <UserMenu user={user} borderColor={"gray.200"} />}
+      <Divider />
       <VStack spacing={0} align="center">
         <ChakraLink
           href="https://github.com/openpipe/openpipe"
@@ -117,7 +110,15 @@ const NavSidebar = () => {
   );
 };
 
-export default function AppShell(props: { children: React.ReactNode; title?: string }) {
+export default function AppShell({
+  children,
+  title,
+  requireAuth,
+}: {
+  children: React.ReactNode;
+  title?: string;
+  requireAuth?: boolean;
+}) {
   const [vh, setVh] = useState("100vh"); // Default height to prevent flicker on initial render
 
   useEffect(() => {
@@ -137,14 +138,23 @@ export default function AppShell(props: { children: React.ReactNode; title?: str
     };
   }, []);
 
+  const user = useSession().data;
+  const authLoading = useSession().status === "loading";
+
+  useEffect(() => {
+    if (requireAuth && user === null && !authLoading) {
+      signIn("github").catch(console.error);
+    }
+  }, [requireAuth, user, authLoading]);
+
   return (
     <Flex h={vh} w="100vw">
       <Head>
-        <title>{props.title ? `${props.title} | OpenPipe` : "OpenPipe"}</title>
+        <title>{title ? `${title} | OpenPipe` : "OpenPipe"}</title>
       </Head>
       <NavSidebar />
       <Box h="100%" flex={1} overflowY="auto">
-        {props.children}
+        {children}
       </Box>
     </Flex>
   );

app/src/components/nav/IconLink.tsx

@@ -0,0 +1,31 @@
+import { Icon, HStack, Text, type BoxProps } from "@chakra-ui/react";
+import Link, { type LinkProps } from "next/link";
+import { type IconType } from "react-icons";
+import NavSidebarOption from "./NavSidebarOption";
+
+type IconLinkProps = BoxProps &
+  LinkProps & { label?: string; icon: IconType; href: string; beta?: boolean };
+
+const IconLink = ({ icon, label, href, color, beta, ...props }: IconLinkProps) => {
+  return (
+    <Link href={href} style={{ width: "100%" }}>
+      <NavSidebarOption activeHrefPattern={href}>
+        <HStack w="full" justifyContent="space-between" p={2} color={color} {...props}>
+          <HStack w="full" justifyContent="start">
+            <Icon as={icon} boxSize={6} mr={2} />
+            <Text fontSize="sm" display={{ base: "none", md: "block" }}>
+              {label}
+            </Text>
+          </HStack>
+          {beta && (
+            <Text fontSize="xs" ml={2} fontWeight="bold" color="orange.400">
+              BETA
+            </Text>
+          )}
+        </HStack>
+      </NavSidebarOption>
+    </Link>
+  );
+};
+
+export default IconLink;

app/src/components/nav/NavSidebarOption.tsx

@@ -0,0 +1,27 @@
+import { Box, type BoxProps } from "@chakra-ui/react";
+import { useRouter } from "next/router";
+
+const NavSidebarOption = ({
+  activeHrefPattern,
+  disableHoverEffect,
+  ...props
+}: { activeHrefPattern?: string; disableHoverEffect?: boolean } & BoxProps) => {
+  const router = useRouter();
+  const isActive = activeHrefPattern && router.pathname.startsWith(activeHrefPattern);
+  return (
+    <Box
+      w="full"
+      fontWeight={isActive ? "bold" : "500"}
+      bgColor={isActive ? "gray.200" : "transparent"}
+      _hover={disableHoverEffect ? undefined : { bgColor: "gray.200", textDecoration: "none" }}
+      justifyContent="start"
+      cursor="pointer"
+      borderRadius={4}
+      {...props}
+    >
+      {props.children}
+    </Box>
+  );
+};
+
+export default NavSidebarOption;

app/src/components/nav/PageHeaderContainer.tsx

@@ -0,0 +1,19 @@
+import { Flex, type FlexProps } from "@chakra-ui/react";
+
+const PageHeaderContainer = (props: FlexProps) => {
+  return (
+    <Flex
+      px={8}
+      py={2}
+      minH={16}
+      w="full"
+      direction={{ base: "column", sm: "row" }}
+      alignItems={{ base: "flex-start", sm: "center" }}
+      justifyContent="space-between"
+      fontWeight="500"
+      {...props}
+    />
+  );
+};
+
+export default PageHeaderContainer;

app/src/components/nav/ProjectBreadcrumbContents.tsx

@@ -0,0 +1,28 @@
+import { HStack, Flex, Text } from "@chakra-ui/react";
+import { useSelectedOrg } from "~/utils/hooks";
+
+// Have to export only contents here instead of full BreadcrumbItem because Chakra doesn't
+// recognize a BreadcrumbItem exported with this component as a valid child of Breadcrumb.
+export default function ProjectBreadcrumbContents({ orgName = "" }: { orgName?: string }) {
+  const { data: selectedOrg } = useSelectedOrg();
+
+  orgName = orgName || selectedOrg?.name || "";
+
+  return (
+    <HStack w="full">
+      <Flex
+        p={1}
+        borderRadius={4}
+        backgroundColor="orange.100"
+        boxSize={6}
+        alignItems="center"
+        justifyContent="center"
+      >
+        <Text>{orgName[0]?.toUpperCase()}</Text>
+      </Flex>
+      <Text display={{ base: "none", md: "block" }} py={1}>
+        {orgName}
+      </Text>
+    </HStack>
+  );
+}

app/src/components/nav/ProjectMenu.tsx

@@ -0,0 +1,178 @@
+import {
+  HStack,
+  VStack,
+  Text,
+  Popover,
+  PopoverTrigger,
+  PopoverContent,
+  Flex,
+  IconButton,
+  Icon,
+  Divider,
+  Button,
+  useDisclosure,
+  Spinner,
+} from "@chakra-ui/react";
+import React, { useEffect, useState } from "react";
+import Link from "next/link";
+import { AiFillCaretDown } from "react-icons/ai";
+import { BsGear, BsPlus } from "react-icons/bs";
+import { type Organization } from "@prisma/client";
+
+import { useAppStore } from "~/state/store";
+import { api } from "~/utils/api";
+import NavSidebarOption from "./NavSidebarOption";
+import { useHandledAsyncCallback, useSelectedOrg } from "~/utils/hooks";
+import { useRouter } from "next/router";
+
+export default function ProjectMenu() {
+  const router = useRouter();
+  const isActive = router.pathname.startsWith("/home");
+  const utils = api.useContext();
+
+  const selectedOrgId = useAppStore((s) => s.selectedOrgId);
+  const setSelectedOrgId = useAppStore((s) => s.setSelectedOrgId);
+
+  const { data: orgs } = api.organizations.list.useQuery();
+
+  useEffect(() => {
+    if (orgs && orgs[0] && (!selectedOrgId || !orgs.find((org) => org.id === selectedOrgId))) {
+      setSelectedOrgId(orgs[0].id);
+    }
+  }, [selectedOrgId, setSelectedOrgId, orgs]);
+
+  const { data: selectedOrg } = useSelectedOrg();
+
+  const popover = useDisclosure();
+
+  const createMutation = api.organizations.create.useMutation();
+  const [createProject, isLoading] = useHandledAsyncCallback(async () => {
+    const newOrg = await createMutation.mutateAsync({ name: "New Project" });
+    await utils.organizations.list.invalidate();
+    setSelectedOrgId(newOrg.id);
+    await router.push({ pathname: "/project/settings" });
+  }, [createMutation, router]);
+
+  return (
+    <VStack w="full" alignItems="flex-start" spacing={0}>
+      <Text
+        pl={2}
+        pb={2}
+        fontSize="xs"
+        fontWeight="bold"
+        color="gray.500"
+        display={{ base: "none", md: "flex" }}
+      >
+        PROJECT
+      </Text>
+      <NavSidebarOption>
+        <Popover
+          placement="bottom-start"
+          isOpen={popover.isOpen}
+          onClose={popover.onClose}
+          closeOnBlur
+        >
+          <PopoverTrigger>
+            <HStack w="full" justifyContent="space-between" onClick={popover.onToggle}>
+              <Flex
+                p={1}
+                borderRadius={4}
+                backgroundColor="orange.100"
+                minW={{ base: 10, md: 8 }}
+                minH={{ base: 10, md: 8 }}
+                m={{ base: 0, md: 1 }}
+                alignItems="center"
+                justifyContent="center"
+                // onClick={sidebarExpanded ? undefined : openMenu}
+              >
+                <Text>{selectedOrg?.name[0]?.toUpperCase()}</Text>
+              </Flex>
+              <Text fontSize="sm" display={{ base: "none", md: "block" }} py={1}>
+                {selectedOrg?.name}
+              </Text>
+              <Icon as={AiFillCaretDown} boxSize={3} size="xs" color="gray.500" mr={2} />
+            </HStack>
+          </PopoverTrigger>
+          <PopoverContent
+            _focusVisible={{ boxShadow: "unset" }}
+            minW={0}
+            borderColor="blue.400"
+            w="full"
+          >
+            <VStack alignItems="flex-start" spacing={2} py={4} px={2}>
+              <Text color="gray.500" fontSize="xs" fontWeight="bold" pb={1}>
+                PROJECTS
+              </Text>
+              <Divider />
+              <VStack spacing={0} w="full">
+                {orgs?.map((org) => (
+                  <ProjectOption
+                    key={org.id}
+                    org={org}
+                    isActive={org.id === selectedOrgId}
+                    onClose={popover.onClose}
+                  />
+                ))}
+              </VStack>
+              <HStack
+                as={Button}
+                variant="ghost"
+                colorScheme="blue"
+                color="blue.400"
+                pr={8}
+                w="full"
+                onClick={createProject}
+              >
+                <Icon as={isLoading ? Spinner : BsPlus} boxSize={6} />
+                <Text>New project</Text>
+              </HStack>
+            </VStack>
+          </PopoverContent>
+        </Popover>
+      </NavSidebarOption>
+    </VStack>
+  );
+}
+
+const ProjectOption = ({
+  org,
+  isActive,
+  onClose,
+}: {
+  org: Organization;
+  isActive: boolean;
+  onClose: () => void;
+}) => {
+  const setSelectedOrgId = useAppStore((s) => s.setSelectedOrgId);
+  const [gearHovered, setGearHovered] = useState(false);
+  return (
+    <HStack
+      as={Link}
+      href="/experiments"
+      onClick={() => {
+        setSelectedOrgId(org.id);
+        onClose();
+      }}
+      w="full"
+      justifyContent="space-between"
+      bgColor={isActive ? "gray.100" : "transparent"}
+      _hover={gearHovered ? undefined : { bgColor: "gray.200", textDecoration: "none" }}
+      p={2}
+    >
+      <Text>{org.name}</Text>
+      <IconButton
+        as={Link}
+        href="/project/settings"
+        aria-label={`Open ${org.name} settings`}
+        icon={<Icon as={BsGear} boxSize={5} strokeWidth={0.5} color="gray.500" />}
+        variant="ghost"
+        size="xs"
+        p={0}
+        onMouseEnter={() => setGearHovered(true)}
+        onMouseLeave={() => setGearHovered(false)}
+        _hover={{ bgColor: isActive ? "gray.300" : "gray.100", transitionDelay: 0 }}
+        borderRadius={4}
+      />
+    </HStack>
+  );
+};

app/src/components/nav/UserMenu.tsx

@@ -8,16 +8,15 @@ import {
   PopoverTrigger,
   PopoverContent,
   Link,
-  useColorMode,
   type StackProps,
+  Box,
 } from "@chakra-ui/react";
 import { type Session } from "next-auth";
 import { signOut } from "next-auth/react";
 import { BsBoxArrowRight, BsChevronRight, BsPersonCircle } from "react-icons/bs";
+import NavSidebarOption from "./NavSidebarOption";
 
 export default function UserMenu({ user, ...rest }: { user: Session } & StackProps) {
-  const { colorMode } = useColorMode();
-
   const profileImage = user.user.image ? (
     <Image src={user.user.image} alt="profile picture" boxSize={8} borderRadius="50%" />
   ) : (
@@ -28,28 +27,28 @@ export default function UserMenu({ user, ...rest }: { user: Session } & StackPro
     <>
       <Popover placement="right">
         <PopoverTrigger>
-          <HStack
+          <Box>
-            // Weird values to make mobile look right; can clean up when we make the sidebar disappear on mobile
+            <NavSidebarOption>
-            px={3}
+              <HStack
-            spacing={3}
+                // Weird values to make mobile look right; can clean up when we make the sidebar disappear on mobile
-            py={2}
+                py={2}
-            {...rest}
+                px={1}
-            cursor="pointer"
+                spacing={3}
-            _hover={{
+                {...rest}
-              bgColor: colorMode === "light" ? "gray.200" : "gray.700",
+              >
-            }}
+                {profileImage}
-          >
+                <VStack spacing={0} align="start" flex={1} flexShrink={1}>
-            {profileImage}
+                  <Text fontWeight="bold" fontSize="sm">
-            <VStack spacing={0} align="start" flex={1} flexShrink={1}>
+                    {user.user.name}
-              <Text fontWeight="bold" fontSize="sm">
+                  </Text>
-                {user.user.name}
+                  <Text color="gray.500" fontSize="xs">
-              </Text>
+                    {/* {user.user.email} */}
-              <Text color="gray.500" fontSize="xs">
+                  </Text>
-                {user.user.email}
+                </VStack>
-              </Text>
+                <Icon as={BsChevronRight} boxSize={4} color="gray.500" />
-            </VStack>
+              </HStack>
-            <Icon as={BsChevronRight} boxSize={4} color="gray.500" />
+            </NavSidebarOption>
-          </HStack>
+          </Box>
         </PopoverTrigger>
         <PopoverContent _focusVisible={{ boxShadow: "unset", outline: "unset" }} maxW="200px">
           <VStack align="stretch" spacing={0}>

app/src/components/projectSettings/DeleteProjectDialog.tsx

@@ -0,0 +1,89 @@
+import {
+  Button,
+  AlertDialog,
+  AlertDialogBody,
+  AlertDialogFooter,
+  AlertDialogHeader,
+  AlertDialogContent,
+  AlertDialogOverlay,
+  Input,
+  Text,
+  VStack,
+  Box,
+  Spinner,
+} from "@chakra-ui/react";
+
+import { useRouter } from "next/router";
+import { useRef, useState } from "react";
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback, useSelectedOrg } from "~/utils/hooks";
+
+export const DeleteProjectDialog = ({
+  isOpen,
+  onClose,
+}: {
+  isOpen: boolean;
+  onClose: () => void;
+}) => {
+  const selectedOrg = useSelectedOrg();
+  const deleteMutation = api.organizations.delete.useMutation();
+  const utils = api.useContext();
+  const router = useRouter();
+
+  const cancelRef = useRef<HTMLButtonElement>(null);
+
+  const [onDeleteConfirm, isDeleting] = useHandledAsyncCallback(async () => {
+    if (!selectedOrg.data?.id) return;
+    await deleteMutation.mutateAsync({ id: selectedOrg.data.id });
+    await utils.organizations.list.invalidate();
+    await router.push({ pathname: "/experiments" });
+    onClose();
+  }, [deleteMutation, selectedOrg, router]);
+
+  const [nameToDelete, setNameToDelete] = useState("");
+
+  return (
+    <AlertDialog isOpen={isOpen} leastDestructiveRef={cancelRef} onClose={onClose}>
+      <AlertDialogOverlay>
+        <AlertDialogContent>
+          <AlertDialogHeader fontSize="lg" fontWeight="bold">
+            Delete Project
+          </AlertDialogHeader>
+
+          <AlertDialogBody>
+            <VStack spacing={4} alignItems="flex-start">
+              <Text>
+                If you delete this project all the associated data and experiments will be deleted
+                as well. If you are sure that you want to delete this project, please type the name
+                of the project below.
+              </Text>
+              <Box bgColor="orange.100" w="full" p={2} borderRadius={4}>
+                <Text fontFamily="inconsolata">{selectedOrg.data?.name}</Text>
+              </Box>
+              <Input
+                placeholder={selectedOrg.data?.name}
+                value={nameToDelete}
+                onChange={(e) => setNameToDelete(e.target.value)}
+              />
+            </VStack>
+          </AlertDialogBody>
+
+          <AlertDialogFooter>
+            <Button ref={cancelRef} onClick={onClose}>
+              Cancel
+            </Button>
+            <Button
+              colorScheme="red"
+              onClick={onDeleteConfirm}
+              ml={3}
+              isDisabled={nameToDelete !== selectedOrg.data?.name}
+              w={20}
+            >
+              {isDeleting ? <Spinner /> : "Delete"}
+            </Button>
+          </AlertDialogFooter>
+        </AlertDialogContent>
+      </AlertDialogOverlay>
+    </AlertDialog>
+  );
+};

app/src/env.mjs

@@ -20,6 +20,7 @@ export const env = createEnv({
     REPLICATE_API_TOKEN: z.string().default("placeholder"),
     ANTHROPIC_API_KEY: z.string().default("placeholder"),
     SENTRY_AUTH_TOKEN: z.string().optional(),
+    OPENPIPE_API_KEY: z.string().optional(),
   },
 
   /**
@@ -33,6 +34,7 @@ export const env = createEnv({
     NEXT_PUBLIC_HOST: z.string().url().default("http://localhost:3000"),
     NEXT_PUBLIC_SENTRY_DSN: z.string().optional(),
     NEXT_PUBLIC_SHOW_DATA: z.string().optional(),
+    NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS: z.string().optional(),
   },
 
   /**
@@ -54,6 +56,8 @@ export const env = createEnv({
     ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY,
     NEXT_PUBLIC_SENTRY_DSN: process.env.NEXT_PUBLIC_SENTRY_DSN,
     SENTRY_AUTH_TOKEN: process.env.SENTRY_AUTH_TOKEN,
+    OPENPIPE_API_KEY: process.env.OPENPIPE_API_KEY,
+    NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS: process.env.NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS,
   },
   /**
    * Run `build` or `dev` with `SKIP_ENV_VALIDATION` to skip env validation.

app/src/pages/api/[...trpc].ts

@@ -0,0 +1,22 @@
+import { type NextApiRequest, type NextApiResponse } from "next";
+import cors from "nextjs-cors";
+import { createOpenApiNextHandler } from "trpc-openapi";
+import { createProcedureCache } from "trpc-openapi/dist/adapters/node-http/procedures";
+import { appRouter } from "~/server/api/root.router";
+import { createTRPCContext } from "~/server/api/trpc";
+
+const openApiHandler = createOpenApiNextHandler({
+  router: appRouter,
+  createContext: createTRPCContext,
+});
+
+const cache = createProcedureCache(appRouter);
+
+const handler = async (req: NextApiRequest, res: NextApiResponse) => {
+  // Setup CORS
+  await cors(req, res);
+
+  return openApiHandler(req, res);
+};
+
+export default handler;

app/src/pages/api/openapi.json.ts

@@ -0,0 +1,16 @@
+import { type NextApiRequest, type NextApiResponse } from "next";
+import { generateOpenApiDocument } from "trpc-openapi";
+import { appRouter } from "~/server/api/root.router";
+
+export const openApiDocument = generateOpenApiDocument(appRouter, {
+  title: "OpenPipe API",
+  description: "The public API for reporting API calls to OpenPipe",
+  version: "0.1.0",
+  baseUrl: "https://app.openpipe.ai/api",
+});
+// Respond with our OpenAPI schema
+const hander = (req: NextApiRequest, res: NextApiResponse) => {
+  res.status(200).send(openApiDocument);
+};
+
+export default hander;

app/src/pages/data/[id].tsx

@@ -18,6 +18,8 @@ import { api } from "~/utils/api";
 import { useDataset, useHandledAsyncCallback } from "~/utils/hooks";
 import DatasetEntriesTable from "~/components/datasets/DatasetEntriesTable";
 import { DatasetHeaderButtons } from "~/components/datasets/DatasetHeaderButtons/DatasetHeaderButtons";
+import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
+import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
 
 export default function Dataset() {
   const router = useRouter();
@@ -55,15 +57,11 @@ export default function Dataset() {
   return (
     <AppShell title={dataset.data?.name}>
       <VStack h="full">
-        <Flex
+        <PageHeaderContainer>
-          pl={4}
+          <Breadcrumb>
-          pr={8}
+            <BreadcrumbItem>
-          py={2}
+              <ProjectBreadcrumbContents orgName={dataset.data?.organization?.name} />
-          w="full"
+            </BreadcrumbItem>
-          direction={{ base: "column", sm: "row" }}
-          alignItems={{ base: "flex-start", sm: "center" }}
-        >
-          <Breadcrumb flex={1} mt={1}>
             <BreadcrumbItem>
               <Link href="/data">
                 <Flex alignItems="center" _hover={{ textDecoration: "underline" }}>
@@ -89,8 +87,8 @@ export default function Dataset() {
             </BreadcrumbItem>
           </Breadcrumb>
           <DatasetHeaderButtons />
-        </Flex>
+        </PageHeaderContainer>
-        <Box w="full" overflowX="auto" flex={1} pl={4} pr={8} pt={8} pb={16}>
+        <Box w="full" overflowX="auto" flex={1} px={8} pt={8} pb={16}>
           {datasetId && <DatasetEntriesTable />}
         </Box>
       </VStack>

app/src/pages/data/index.tsx

@@ -1,83 +1,49 @@
-import {
+import { SimpleGrid, Icon, Breadcrumb, BreadcrumbItem, Flex } from "@chakra-ui/react";
-  SimpleGrid,
-  Icon,
-  VStack,
-  Breadcrumb,
-  BreadcrumbItem,
-  Flex,
-  Center,
-  Text,
-  Link,
-  HStack,
-} from "@chakra-ui/react";
 import AppShell from "~/components/nav/AppShell";
-import { api } from "~/utils/api";
-import { signIn, useSession } from "next-auth/react";
 import { RiDatabase2Line } from "react-icons/ri";
 import {
   DatasetCard,
   DatasetCardSkeleton,
   NewDatasetCard,
 } from "~/components/datasets/DatasetCard";
+import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
+import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
+import { useDatasets } from "~/utils/hooks";
 
 export default function DatasetsPage() {
-  const datasets = api.datasets.list.useQuery();
+  const datasets = useDatasets();
-
-  const user = useSession().data;
-  const authLoading = useSession().status === "loading";
-
-  if (user === null || authLoading) {
-    return (
-      <AppShell title="Data">
-        <Center h="100%">
-          {!authLoading && (
-            <Text>
-              <Link
-                onClick={() => {
-                  signIn("github").catch(console.error);
-                }}
-                textDecor="underline"
-              >
-                Sign in
-              </Link>{" "}
-              to view or create new datasets!
-            </Text>
-          )}
-        </Center>
-      </AppShell>
-    );
-  }
 
   return (
-    <AppShell title="Data">
+    <AppShell title="Data" requireAuth>
-      <VStack alignItems={"flex-start"} px={4} py={2}>
+      <PageHeaderContainer>
-        <HStack minH={8} align="center" pt={2}>
+        <Breadcrumb>
-          <Breadcrumb flex={1}>
+          <BreadcrumbItem>
-            <BreadcrumbItem>
+            <ProjectBreadcrumbContents />
-              <Flex alignItems="center">
+          </BreadcrumbItem>
-                <Icon as={RiDatabase2Line} boxSize={4} mr={2} /> Datasets
+          <BreadcrumbItem minH={8}>
-              </Flex>
+            <Flex alignItems="center">
-            </BreadcrumbItem>
+              <Icon as={RiDatabase2Line} boxSize={4} mr={2} /> Datasets
-          </Breadcrumb>
+            </Flex>
-        </HStack>
+          </BreadcrumbItem>
-        <SimpleGrid w="full" columns={{ base: 1, md: 2, lg: 3, xl: 4 }} spacing={8} p="4">
+        </Breadcrumb>
-          <NewDatasetCard />
+      </PageHeaderContainer>
-          {datasets.data && !datasets.isLoading ? (
+      <SimpleGrid w="full" columns={{ base: 1, md: 2, lg: 3, xl: 4 }} spacing={8} py={4} px={8}>
-            datasets?.data?.map((dataset) => (
+        <NewDatasetCard />
-              <DatasetCard
+        {datasets.data && !datasets.isLoading ? (
-                key={dataset.id}
+          datasets?.data?.map((dataset) => (
-                dataset={{ ...dataset, numEntries: dataset._count.datasetEntries }}
+            <DatasetCard
-              />
+              key={dataset.id}
-            ))
+              dataset={{ ...dataset, numEntries: dataset._count.datasetEntries }}
-          ) : (
+            />
-            <>
+          ))
-              <DatasetCardSkeleton />
+        ) : (
-              <DatasetCardSkeleton />
+          <>
-              <DatasetCardSkeleton />
+            <DatasetCardSkeleton />
-            </>
+            <DatasetCardSkeleton />
-          )}
+            <DatasetCardSkeleton />
-        </SimpleGrid>
+          </>
-      </VStack>
+        )}
+      </SimpleGrid>
     </AppShell>
   );
 }

app/src/pages/experiments/[id].tsx

@@ -23,6 +23,8 @@ import { useAppStore } from "~/state/store";
 import { useSyncVariantEditor } from "~/state/sync";
 import { ExperimentHeaderButtons } from "~/components/experiments/ExperimentHeaderButtons/ExperimentHeaderButtons";
 import Head from "next/head";
+import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
+import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
 
 // TODO: import less to fix deployment with server side props
 // export const getServerSideProps = async (context: GetServerSidePropsContext<{ id: string }>) => {
@@ -104,14 +106,11 @@ export default function Experiment() {
       )}
       <AppShell title={experiment.data?.label}>
         <VStack h="full">
-          <Flex
+          <PageHeaderContainer>
-            px={4}
+            <Breadcrumb>
-            py={2}
+              <BreadcrumbItem>
-            w="full"
+                <ProjectBreadcrumbContents orgName={experiment.data?.organization?.name} />
-            direction={{ base: "column", sm: "row" }}
+              </BreadcrumbItem>
-            alignItems={{ base: "flex-start", sm: "center" }}
-          >
-            <Breadcrumb flex={1}>
               <BreadcrumbItem>
                 <Link href="/experiments">
                   <Flex alignItems="center" _hover={{ textDecoration: "underline" }}>
@@ -143,7 +142,7 @@ export default function Experiment() {
               </BreadcrumbItem>
             </Breadcrumb>
             <ExperimentHeaderButtons />
-          </Flex>
+          </PageHeaderContainer>
           <ExperimentSettingsDrawer />
           <Box w="100%" overflowX="auto" flex={1}>
             <OutputsTable experimentId={router.query.id as string | undefined} />

app/src/pages/experiments/index.tsx

@@ -1,78 +1,44 @@
-import {
+import { SimpleGrid, Icon, Breadcrumb, BreadcrumbItem, Flex } from "@chakra-ui/react";
-  SimpleGrid,
-  Icon,
-  VStack,
-  Breadcrumb,
-  BreadcrumbItem,
-  Flex,
-  Center,
-  Text,
-  Link,
-  HStack,
-} from "@chakra-ui/react";
 import { RiFlaskLine } from "react-icons/ri";
 import AppShell from "~/components/nav/AppShell";
-import { api } from "~/utils/api";
 import {
   ExperimentCard,
   ExperimentCardSkeleton,
   NewExperimentCard,
 } from "~/components/experiments/ExperimentCard";
-import { signIn, useSession } from "next-auth/react";
+import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
+import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
+import { useExperiments } from "~/utils/hooks";
 
 export default function ExperimentsPage() {
-  const experiments = api.experiments.list.useQuery();
+  const experiments = useExperiments();
-
-  const user = useSession().data;
-  const authLoading = useSession().status === "loading";
-
-  if (user === null || authLoading) {
-    return (
-      <AppShell title="Experiments">
-        <Center h="100%">
-          {!authLoading && (
-            <Text>
-              <Link
-                onClick={() => {
-                  signIn("github").catch(console.error);
-                }}
-                textDecor="underline"
-              >
-                Sign in
-              </Link>{" "}
-              to view or create new experiments!
-            </Text>
-          )}
-        </Center>
-      </AppShell>
-    );
-  }
 
   return (
-    <AppShell title="Experiments">
+    <AppShell title="Experiments" requireAuth>
-      <VStack alignItems={"flex-start"} px={4} py={2}>
+      <PageHeaderContainer>
-        <HStack minH={8} align="center" pt={2}>
+        <Breadcrumb>
-          <Breadcrumb flex={1}>
+          <BreadcrumbItem>
-            <BreadcrumbItem>
+            <ProjectBreadcrumbContents />
-              <Flex alignItems="center">
+          </BreadcrumbItem>
-                <Icon as={RiFlaskLine} boxSize={4} mr={2} /> Experiments
+          <BreadcrumbItem minH={8}>
-              </Flex>
+            <Flex alignItems="center">
-            </BreadcrumbItem>
+              <Icon as={RiFlaskLine} boxSize={4} mr={2} /> Experiments
-          </Breadcrumb>
+            </Flex>
-        </HStack>
+          </BreadcrumbItem>
-        <SimpleGrid w="full" columns={{ base: 1, md: 2, lg: 3, xl: 4 }} spacing={8} p="4">
+        </Breadcrumb>
-          <NewExperimentCard />
+      </PageHeaderContainer>
-          {experiments.data && !experiments.isLoading ? (
+      <SimpleGrid w="full" columns={{ base: 1, md: 2, lg: 3, xl: 4 }} spacing={8} py="4" px={8}>
-            experiments?.data?.map((exp) => <ExperimentCard key={exp.id} exp={exp} />)
+        <NewExperimentCard />
-          ) : (
+        {experiments.data && !experiments.isLoading ? (
-            <>
+          experiments?.data?.map((exp) => <ExperimentCard key={exp.id} exp={exp} />)
-              <ExperimentCardSkeleton />
+        ) : (
-              <ExperimentCardSkeleton />
+          <>
-              <ExperimentCardSkeleton />
+            <ExperimentCardSkeleton />
-            </>
+            <ExperimentCardSkeleton />
-          )}
+            <ExperimentCardSkeleton />
-        </SimpleGrid>
+          </>
-      </VStack>
+        )}
+      </SimpleGrid>
     </AppShell>
   );
 }

app/src/pages/logged-calls/index.tsx

@@ -0,0 +1,181 @@
+import {
+  Heading,
+  Text,
+  Stat,
+  StatLabel,
+  StatNumber,
+  VStack,
+  HStack,
+  Card,
+  CardBody,
+  CardHeader,
+  Icon,
+  Table,
+  Tbody,
+  Tr,
+  Td,
+  Divider,
+  Breadcrumb,
+  BreadcrumbItem,
+} from "@chakra-ui/react";
+import {
+  LineChart,
+  Line,
+  XAxis,
+  YAxis,
+  CartesianGrid,
+  Tooltip,
+  Legend,
+  ResponsiveContainer,
+} from "recharts";
+import { Ban, DollarSign, Hash } from "lucide-react";
+import { useMemo } from "react";
+
+import AppShell from "~/components/nav/AppShell";
+import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
+import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
+import { useSelectedOrg } from "~/utils/hooks";
+import dayjs from "~/utils/dayjs";
+import { api } from "~/utils/api";
+import LoggedCallTable from "~/components/dashboard/LoggedCallTable";
+
+export default function LoggedCalls() {
+  const { data: selectedOrg } = useSelectedOrg();
+
+  const stats = api.dashboard.stats.useQuery(
+    { organizationId: selectedOrg?.id ?? "" },
+    { enabled: !!selectedOrg },
+  );
+
+  const data = useMemo(() => {
+    return (
+      stats.data?.periods.map(({ period, numQueries, totalCost }) => ({
+        period,
+        Requests: numQueries,
+        "Total Spent (USD)": parseFloat(totalCost.toString()),
+      })) || []
+    );
+  }, [stats.data]);
+
+  return (
+    <AppShell requireAuth>
+      <PageHeaderContainer>
+        <Breadcrumb>
+          <BreadcrumbItem>
+            <ProjectBreadcrumbContents />
+          </BreadcrumbItem>
+          <BreadcrumbItem isCurrentPage>
+            <Text>Logged Calls</Text>
+          </BreadcrumbItem>
+        </Breadcrumb>
+      </PageHeaderContainer>
+      <VStack px={8} pt={4} alignItems="flex-start" spacing={4}>
+        <Text fontSize="2xl" fontWeight="bold">
+          {selectedOrg?.name}
+        </Text>
+        <Divider />
+        <VStack margin="auto" spacing={4} align="stretch" w="full">
+          <HStack gap={4} align="start">
+            <Card variant="outline" flex={1}>
+              <CardHeader>
+                <Heading as="h3" size="sm">
+                  Usage Statistics
+                </Heading>
+              </CardHeader>
+              <CardBody>
+                <ResponsiveContainer width="100%" height={400}>
+                  <LineChart data={data} margin={{ top: 5, right: 20, left: 10, bottom: 5 }}>
+                    <XAxis
+                      dataKey="period"
+                      tickFormatter={(str: string) => dayjs(str).format("MMM D")}
+                    />
+                    <YAxis yAxisId="left" dataKey="Requests" orientation="left" stroke="#8884d8" />
+                    <YAxis
+                      yAxisId="right"
+                      dataKey="Total Spent (USD)"
+                      orientation="right"
+                      unit="$"
+                      stroke="#82ca9d"
+                    />
+                    <Tooltip />
+                    <Legend />
+                    <CartesianGrid stroke="#f5f5f5" />
+                    <Line
+                      dataKey="Requests"
+                      stroke="#8884d8"
+                      yAxisId="left"
+                      dot={false}
+                      strokeWidth={2}
+                    />
+                    <Line
+                      dataKey="Total Spent (USD)"
+                      stroke="#82ca9d"
+                      yAxisId="right"
+                      dot={false}
+                      strokeWidth={2}
+                    />
+                  </LineChart>
+                </ResponsiveContainer>
+              </CardBody>
+            </Card>
+            <VStack spacing="4" width="300px" align="stretch">
+              <Card variant="outline">
+                <CardBody>
+                  <Stat>
+                    <HStack>
+                      <StatLabel flex={1}>Total Spent</StatLabel>
+                      <Icon as={DollarSign} boxSize={4} color="gray.500" />
+                    </HStack>
+                    <StatNumber>
+                      ${parseFloat(stats.data?.totals?.totalCost?.toString() ?? "0").toFixed(2)}
+                    </StatNumber>
+                  </Stat>
+                </CardBody>
+              </Card>
+              <Card variant="outline">
+                <CardBody>
+                  <Stat>
+                    <HStack>
+                      <StatLabel flex={1}>Total Requests</StatLabel>
+                      <Icon as={Hash} boxSize={4} color="gray.500" />
+                    </HStack>
+                    <StatNumber>
+                      {stats.data?.totals?.numQueries
+                        ? parseInt(stats.data?.totals?.numQueries.toString())?.toLocaleString()
+                        : undefined}
+                    </StatNumber>
+                  </Stat>
+                </CardBody>
+              </Card>
+              <Card variant="outline" overflow="hidden">
+                <Stat>
+                  <CardHeader>
+                    <HStack>
+                      <StatLabel flex={1}>Errors</StatLabel>
+                      <Icon as={Ban} boxSize={4} color="gray.500" />
+                    </HStack>
+                  </CardHeader>
+                  <Table variant="simple">
+                    <Tbody>
+                      {stats.data?.errors?.map((error) => (
+                        <Tr key={error.code}>
+                          <Td>
+                            {error.name} ({error.code})
+                          </Td>
+                          <Td isNumeric color="red.600">
+                            {parseInt(error.count.toString()).toLocaleString()}
+                          </Td>
+                        </Tr>
+                      ))}
+                    </Tbody>
+                  </Table>
+                </Stat>
+              </Card>
+            </VStack>
+          </HStack>
+          <LoggedCallTable />
+        </VStack>
+      </VStack>
+    </AppShell>
+  );
+}

app/src/pages/project/settings/index.tsx

@@ -0,0 +1,152 @@
+import {
+  Breadcrumb,
+  BreadcrumbItem,
+  Text,
+  type TextProps,
+  VStack,
+  HStack,
+  Input,
+  Button,
+  Divider,
+  Icon,
+  useDisclosure,
+} from "@chakra-ui/react";
+import { useEffect, useState } from "react";
+import { BsTrash } from "react-icons/bs";
+
+import AppShell from "~/components/nav/AppShell";
+import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback, useSelectedOrg } from "~/utils/hooks";
+import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
+import CopiableCode from "~/components/CopiableCode";
+import { DeleteProjectDialog } from "~/components/projectSettings/DeleteProjectDialog";
+
+export default function Settings() {
+  const utils = api.useContext();
+  const { data: selectedOrg } = useSelectedOrg();
+
+  const apiKey =
+    selectedOrg?.apiKeys?.length && selectedOrg?.apiKeys[0] ? selectedOrg?.apiKeys[0].apiKey : "";
+
+  const updateMutation = api.organizations.update.useMutation();
+  const [onSaveName] = useHandledAsyncCallback(async () => {
+    if (name && name !== selectedOrg?.name && selectedOrg?.id) {
+      await updateMutation.mutateAsync({
+        id: selectedOrg.id,
+        updates: { name },
+      });
+      await Promise.all([utils.organizations.get.invalidate({ id: selectedOrg.id })]);
+    }
+  }, [updateMutation, selectedOrg]);
+
+  const [name, setName] = useState(selectedOrg?.name);
+  useEffect(() => {
+    setName(selectedOrg?.name);
+  }, [selectedOrg?.name]);
+
+  const deleteProjectOpen = useDisclosure();
+
+  return (
+    <>
+      <AppShell>
+        <PageHeaderContainer>
+          <Breadcrumb>
+            <BreadcrumbItem>
+              <ProjectBreadcrumbContents />
+            </BreadcrumbItem>
+            <BreadcrumbItem isCurrentPage>
+              <Text>Project Settings</Text>
+            </BreadcrumbItem>
+          </Breadcrumb>
+        </PageHeaderContainer>
+        <VStack px={8} pt={4} alignItems="flex-start" spacing={4}>
+          <VStack spacing={0} alignItems="flex-start">
+            <Text fontSize="2xl" fontWeight="bold">
+              Project Settings
+            </Text>
+            <Text fontSize="sm">
+              Configure your project settings. These settings only apply to {selectedOrg?.name}.
+            </Text>
+          </VStack>
+          <VStack
+            w="full"
+            alignItems="flex-start"
+            borderWidth={1}
+            borderRadius={4}
+            borderColor="gray.300"
+            p={6}
+            spacing={6}
+          >
+            <VStack alignItems="flex-start" w="full">
+              <Text fontWeight="bold" fontSize="xl">
+                Display Name
+              </Text>
+              <Input
+                w="full"
+                maxW={600}
+                value={name}
+                onChange={(e) => setName(e.target.value)}
+                borderColor="gray.300"
+              />
+              <Button
+                isDisabled={!name || name === selectedOrg?.name}
+                colorScheme="orange"
+                borderRadius={4}
+                mt={2}
+                _disabled={{
+                  opacity: 0.6,
+                }}
+                onClick={onSaveName}
+              >
+                Rename Project
+              </Button>
+            </VStack>
+            <Divider backgroundColor="gray.300" />
+            <VStack alignItems="flex-start">
+              <Subtitle>Project API Key</Subtitle>
+              <Text fontSize="sm">
+                Use your project API key to authenticate your requests when sending data to
+                OpenPipe. You can set this key in your environment variables, or use it directly in
+                your code.
+              </Text>
+            </VStack>
+            <CopiableCode code={apiKey} />
+            <Divider />
+            {selectedOrg?.personalOrgUserId ? (
+              <VStack alignItems="flex-start">
+                <Subtitle>Personal Project</Subtitle>
+                <Text fontSize="sm">
+                  This project is {selectedOrg?.personalOrgUser?.name}'s personal project. It cannot
+                  be deleted.
+                </Text>
+              </VStack>
+            ) : (
+              <VStack alignItems="flex-start">
+                <Subtitle color="red.600">Danger Zone</Subtitle>
+                <Text fontSize="sm">
+                  Permanently delete your project and all of its data. This action cannot be undone.
+                </Text>
+                <HStack
+                  as={Button}
+                  isDisabled={selectedOrg?.role !== "ADMIN"}
+                  colorScheme="red"
+                  variant="outline"
+                  borderRadius={4}
+                  mt={2}
+                  onClick={deleteProjectOpen.onOpen}
+                >
+                  <Icon as={BsTrash} />
+                  <Text>Delete {selectedOrg?.name}</Text>
+                </HStack>
+              </VStack>
+            )}
+          </VStack>
+        </VStack>
+      </AppShell>
+      <DeleteProjectDialog isOpen={deleteProjectOpen.isOpen} onClose={deleteProjectOpen.onClose} />
+    </>
+  );
+}
+
+const Subtitle = (props: TextProps) => <Text fontWeight="bold" fontSize="xl" {...props} />;

app/src/server/api/root.router.ts

@@ -8,6 +8,9 @@ import { evaluationsRouter } from "./routers/evaluations.router";
 import { worldChampsRouter } from "./routers/worldChamps.router";
 import { datasetsRouter } from "./routers/datasets.router";
 import { datasetEntries } from "./routers/datasetEntries.router";
+import { externalApiRouter } from "./routers/externalApi.router";
+import { organizationsRouter } from "./routers/organizations.router";
+import { dashboardRouter } from "./routers/dashboard.router";
 
 /**
  * This is the primary router for your server.
@@ -24,6 +27,9 @@ export const appRouter = createTRPCRouter({
   worldChamps: worldChampsRouter,
   datasets: datasetsRouter,
   datasetEntries: datasetEntries,
+  organizations: organizationsRouter,
+  dashboard: dashboardRouter,
+  externalApi: externalApiRouter,
 });
 
 // export type definition of API

app/src/server/api/routers/dashboard.router.ts

@@ -0,0 +1,118 @@
+import { sql } from "kysely";
+import { z } from "zod";
+import { createTRPCRouter, publicProcedure } from "~/server/api/trpc";
+import { kysely, prisma } from "~/server/db";
+import dayjs from "~/utils/dayjs";
+
+export const dashboardRouter = createTRPCRouter({
+  stats: publicProcedure
+    .input(
+      z.object({
+        // TODO: actually take startDate into account
+        startDate: z.string().optional(),
+        organizationId: z.string(),
+      }),
+    )
+    .query(async ({ input }) => {
+      // Return the stats group by hour
+      const periods = await kysely
+        .selectFrom("LoggedCall")
+        .leftJoin(
+          "LoggedCallModelResponse",
+          "LoggedCall.id",
+          "LoggedCallModelResponse.originalLoggedCallId",
+        )
+        .where("organizationId", "=", input.organizationId)
+        .select(({ fn }) => [
+          sql<Date>`date_trunc('day', "LoggedCallModelResponse"."startTime")`.as("period"),
+          sql<number>`count("LoggedCall"."id")::int`.as("numQueries"),
+          fn.sum(fn.coalesce("LoggedCallModelResponse.totalCost", sql<number>`0`)).as("totalCost"),
+        ])
+        .groupBy("period")
+        .orderBy("period")
+        .execute();
+
+      let originalDataIndex = periods.length - 1;
+      // *SLAMS DOWN GLASS OF WHISKEY* timezones, amirite?
+      let dayToMatch = dayjs(input.startDate || new Date());
+      // Ensure that the initial date we're matching against is never before the first period
+      if (
+        periods[originalDataIndex] &&
+        dayToMatch.isBefore(periods[originalDataIndex]?.period, "day")
+      ) {
+        dayToMatch = dayjs(periods[originalDataIndex]?.period);
+      }
+      const backfilledPeriods: typeof periods = [];
+
+      // Backfill from now to 14 days ago or the date of the first logged call, whichever is earlier
+      while (
+        backfilledPeriods.length < 14 ||
+        (periods[0]?.period && !dayToMatch.isBefore(periods[0]?.period, "day"))
+      ) {
+        const nextOriginalPeriod = periods[originalDataIndex];
+        if (nextOriginalPeriod && dayjs(nextOriginalPeriod?.period).isSame(dayToMatch, "day")) {
+          backfilledPeriods.unshift(nextOriginalPeriod);
+          originalDataIndex--;
+        } else {
+          backfilledPeriods.unshift({
+            period: dayjs(dayToMatch).toDate(),
+            numQueries: 0,
+            totalCost: 0,
+          });
+        }
+        dayToMatch = dayToMatch.subtract(1, "day");
+      }
+
+      const totals = await kysely
+        .selectFrom("LoggedCall")
+        .leftJoin(
+          "LoggedCallModelResponse",
+          "LoggedCall.id",
+          "LoggedCallModelResponse.originalLoggedCallId",
+        )
+        .where("organizationId", "=", input.organizationId)
+        .select(({ fn }) => [
+          fn.sum(fn.coalesce("LoggedCallModelResponse.totalCost", sql<number>`0`)).as("totalCost"),
+          fn.count("LoggedCall.id").as("numQueries"),
+        ])
+        .executeTakeFirst();
+
+      const errors = await kysely
+        .selectFrom("LoggedCall")
+        .where("organizationId", "=", input.organizationId)
+        .leftJoin(
+          "LoggedCallModelResponse",
+          "LoggedCall.id",
+          "LoggedCallModelResponse.originalLoggedCallId",
+        )
+        .select(({ fn }) => [fn.count("LoggedCall.id").as("count"), "respStatus as code"])
+        .where("respStatus", ">", 200)
+        .groupBy("code")
+        .orderBy("count", "desc")
+        .execute();
+
+      const namedErrors = errors.map((e) => {
+        if (e.code === 429) {
+          return { ...e, name: "Rate limited" };
+        } else if (e.code === 500) {
+          return { ...e, name: "Internal server error" };
+        } else {
+          return { ...e, name: "Other" };
+        }
+      });
+
+      return { periods: backfilledPeriods, totals, errors: namedErrors };
+    }),
+
+  // TODO useInfiniteQuery
+  // https://discord.com/channels/966627436387266600/1122258443886153758/1122258443886153758
+  loggedCalls: publicProcedure.input(z.object({})).query(async ({ input }) => {
+    const loggedCalls = await prisma.loggedCall.findMany({
+      orderBy: { startTime: "desc" },
+      include: { tags: true, modelResponse: true },
+      take: 20,
+    });
+
+    return loggedCalls;
+  }),
+});

app/src/server/api/routers/datasets.router.ts

@@ -3,65 +3,62 @@ import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/
 import { prisma } from "~/server/db";
 import {
   requireCanModifyDataset,
+  requireCanModifyOrganization,
   requireCanViewDataset,
-  requireNothing,
+  requireCanViewOrganization,
 } from "~/utils/accessControl";
-import userOrg from "~/server/utils/userOrg";
 
 export const datasetsRouter = createTRPCRouter({
-  list: protectedProcedure.query(async ({ ctx }) => {
+  list: protectedProcedure
-    // Anyone can list experiments
+    .input(z.object({ organizationId: z.string() }))
-    requireNothing(ctx);
+    .query(async ({ input, ctx }) => {
+      await requireCanViewOrganization(input.organizationId, ctx);
 
-    const datasets = await prisma.dataset.findMany({
+      const datasets = await prisma.dataset.findMany({
-      where: {
+        where: {
-        organization: {
+          organizationId: input.organizationId,
-          organizationUsers: {
+        },
-            some: { userId: ctx.session.user.id },
+        orderBy: {
+          createdAt: "desc",
+        },
+        include: {
+          _count: {
+            select: { datasetEntries: true },
           },
         },
-      },
+      });
-      orderBy: {
-        createdAt: "desc",
-      },
-      include: {
-        _count: {
-          select: { datasetEntries: true },
-        },
-      },
-    });
 
-    return datasets;
+      return datasets;
-  }),
+    }),
 
   get: publicProcedure.input(z.object({ id: z.string() })).query(async ({ input, ctx }) => {
     await requireCanViewDataset(input.id, ctx);
     return await prisma.dataset.findFirstOrThrow({
       where: { id: input.id },
+      include: {
+        organization: true,
+      },
     });
   }),
 
-  create: protectedProcedure.input(z.object({})).mutation(async ({ ctx }) => {
+  create: protectedProcedure
-    // Anyone can create an experiment
+    .input(z.object({ organizationId: z.string() }))
-    requireNothing(ctx);
+    .mutation(async ({ input, ctx }) => {
+      await requireCanModifyOrganization(input.organizationId, ctx);
 
-    const numDatasets = await prisma.dataset.count({
+      const numDatasets = await prisma.dataset.count({
-      where: {
+        where: {
-        organization: {
+          organizationId: input.organizationId,
-          organizationUsers: {
-            some: { userId: ctx.session.user.id },
-          },
         },
-      },
+      });
-    });
 
-    return await prisma.dataset.create({
+      return await prisma.dataset.create({
-      data: {
+        data: {
-        name: `Dataset ${numDatasets + 1}`,
+          name: `Dataset ${numDatasets + 1}`,
-        organizationId: (await userOrg(ctx.session.user.id)).id,
+          organizationId: input.organizationId,
-      },
+        },
-    });
+      });
-  }),
+    }),
 
   update: protectedProcedure
     .input(z.object({ id: z.string(), updates: z.object({ name: z.string() }) }))

app/src/server/api/routers/experiments.router.ts

@@ -8,10 +8,10 @@ import { generateNewCell } from "~/server/utils/generateNewCell";
 import {
   canModifyExperiment,
   requireCanModifyExperiment,
+  requireCanModifyOrganization,
   requireCanViewExperiment,
-  requireNothing,
+  requireCanViewOrganization,
 } from "~/utils/accessControl";
-import userOrg from "~/server/utils/userOrg";
 import generateTypes from "~/modelProviders/generateTypes";
 import { promptConstructorVersion } from "~/promptConstructor/version";
 
@@ -43,55 +43,55 @@ export const experimentsRouter = createTRPCRouter({
       testScenarioCount,
     };
   }),
-  list: protectedProcedure.query(async ({ ctx }) => {
+  list: protectedProcedure
-    // Anyone can list experiments
+    .input(z.object({ organizationId: z.string() }))
-    requireNothing(ctx);
+    .query(async ({ input, ctx }) => {
+      await requireCanViewOrganization(input.organizationId, ctx);
 
-    const experiments = await prisma.experiment.findMany({
+      const experiments = await prisma.experiment.findMany({
-      where: {
+        where: {
-        organization: {
+          organizationId: input.organizationId,
-          organizationUsers: {
-            some: { userId: ctx.session.user.id },
-          },
         },
-      },
+        orderBy: {
-      orderBy: {
+          sortIndex: "desc",
-        sortIndex: "desc",
+        },
-      },
+      });
-    });
 
-    // TODO: look for cleaner way to do this. Maybe aggregate?
+      // TODO: look for cleaner way to do this. Maybe aggregate?
-    const experimentsWithCounts = await Promise.all(
+      const experimentsWithCounts = await Promise.all(
-      experiments.map(async (experiment) => {
+        experiments.map(async (experiment) => {
-        const visibleTestScenarioCount = await prisma.testScenario.count({
+          const visibleTestScenarioCount = await prisma.testScenario.count({
-          where: {
+            where: {
-            experimentId: experiment.id,
+              experimentId: experiment.id,
-            visible: true,
+              visible: true,
-          },
+            },
-        });
+          });
 
-        const visiblePromptVariantCount = await prisma.promptVariant.count({
+          const visiblePromptVariantCount = await prisma.promptVariant.count({
-          where: {
+            where: {
-            experimentId: experiment.id,
+              experimentId: experiment.id,
-            visible: true,
+              visible: true,
-          },
+            },
-        });
+          });
 
-        return {
+          return {
-          ...experiment,
+            ...experiment,
-          testScenarioCount: visibleTestScenarioCount,
+            testScenarioCount: visibleTestScenarioCount,
-          promptVariantCount: visiblePromptVariantCount,
+            promptVariantCount: visiblePromptVariantCount,
-        };
+          };
-      }),
+        }),
-    );
+      );
 
-    return experimentsWithCounts;
+      return experimentsWithCounts;
-  }),
+    }),
 
   get: publicProcedure.input(z.object({ id: z.string() })).query(async ({ input, ctx }) => {
     await requireCanViewExperiment(input.id, ctx);
     const experiment = await prisma.experiment.findFirstOrThrow({
       where: { id: input.id },
+      include: {
+        organization: true,
+      },
     });
 
     const canModify = ctx.session?.user.id
@@ -107,222 +107,224 @@ export const experimentsRouter = createTRPCRouter({
     };
   }),
 
-  fork: protectedProcedure.input(z.object({ id: z.string() })).mutation(async ({ input, ctx }) => {
+  fork: protectedProcedure
-    await requireCanViewExperiment(input.id, ctx);
+    .input(z.object({ id: z.string(), organizationId: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      await requireCanViewExperiment(input.id, ctx);
+      await requireCanModifyOrganization(input.organizationId, ctx);
 
-    const [
+      const [
-      existingExp,
+        existingExp,
-      existingVariants,
+        existingVariants,
-      existingScenarios,
+        existingScenarios,
-      existingCells,
+        existingCells,
-      evaluations,
+        evaluations,
-      templateVariables,
+        templateVariables,
-    ] = await prisma.$transaction([
+      ] = await prisma.$transaction([
-      prisma.experiment.findUniqueOrThrow({
+        prisma.experiment.findUniqueOrThrow({
-        where: {
+          where: {
-          id: input.id,
+            id: input.id,
-        },
-      }),
-      prisma.promptVariant.findMany({
-        where: {
-          experimentId: input.id,
-          visible: true,
-        },
-      }),
-      prisma.testScenario.findMany({
-        where: {
-          experimentId: input.id,
-          visible: true,
-        },
-      }),
-      prisma.scenarioVariantCell.findMany({
-        where: {
-          testScenario: {
-            visible: true,
           },
-          promptVariant: {
+        }),
+        prisma.promptVariant.findMany({
+          where: {
             experimentId: input.id,
             visible: true,
           },
-        },
+        }),
-        include: {
+        prisma.testScenario.findMany({
-          modelResponses: {
+          where: {
-            include: {
+            experimentId: input.id,
-              outputEvaluations: true,
+            visible: true,
+          },
+        }),
+        prisma.scenarioVariantCell.findMany({
+          where: {
+            testScenario: {
+              visible: true,
+            },
+            promptVariant: {
+              experimentId: input.id,
+              visible: true,
             },
           },
-        },
+          include: {
-      }),
+            modelResponses: {
-      prisma.evaluation.findMany({
+              include: {
-        where: {
+                outputEvaluations: true,
-          experimentId: input.id,
+              },
-        },
+            },
-      }),
+          },
-      prisma.templateVariable.findMany({
+        }),
-        where: {
+        prisma.evaluation.findMany({
-          experimentId: input.id,
+          where: {
-        },
+            experimentId: input.id,
-      }),
+          },
-    ]);
+        }),
+        prisma.templateVariable.findMany({
+          where: {
+            experimentId: input.id,
+          },
+        }),
+      ]);
 
-    const newExperimentId = uuidv4();
+      const newExperimentId = uuidv4();
 
-    const existingToNewVariantIds = new Map<string, string>();
+      const existingToNewVariantIds = new Map<string, string>();
-    const variantsToCreate: Prisma.PromptVariantCreateManyInput[] = [];
+      const variantsToCreate: Prisma.PromptVariantCreateManyInput[] = [];
-    for (const variant of existingVariants) {
+      for (const variant of existingVariants) {
-      const newVariantId = uuidv4();
+        const newVariantId = uuidv4();
-      existingToNewVariantIds.set(variant.id, newVariantId);
+        existingToNewVariantIds.set(variant.id, newVariantId);
-      variantsToCreate.push({
+        variantsToCreate.push({
-        ...variant,
+          ...variant,
-        id: newVariantId,
+          id: newVariantId,
-        experimentId: newExperimentId,
+          experimentId: newExperimentId,
-      });
-    }
-
-    const existingToNewScenarioIds = new Map<string, string>();
-    const scenariosToCreate: Prisma.TestScenarioCreateManyInput[] = [];
-    for (const scenario of existingScenarios) {
-      const newScenarioId = uuidv4();
-      existingToNewScenarioIds.set(scenario.id, newScenarioId);
-      scenariosToCreate.push({
-        ...scenario,
-        id: newScenarioId,
-        experimentId: newExperimentId,
-        variableValues: scenario.variableValues as Prisma.InputJsonValue,
-      });
-    }
-
-    const existingToNewEvaluationIds = new Map<string, string>();
-    const evaluationsToCreate: Prisma.EvaluationCreateManyInput[] = [];
-    for (const evaluation of evaluations) {
-      const newEvaluationId = uuidv4();
-      existingToNewEvaluationIds.set(evaluation.id, newEvaluationId);
-      evaluationsToCreate.push({
-        ...evaluation,
-        id: newEvaluationId,
-        experimentId: newExperimentId,
-      });
-    }
-
-    const cellsToCreate: Prisma.ScenarioVariantCellCreateManyInput[] = [];
-    const modelResponsesToCreate: Prisma.ModelResponseCreateManyInput[] = [];
-    const outputEvaluationsToCreate: Prisma.OutputEvaluationCreateManyInput[] = [];
-    for (const cell of existingCells) {
-      const newCellId = uuidv4();
-      const { modelResponses, ...cellData } = cell;
-      cellsToCreate.push({
-        ...cellData,
-        id: newCellId,
-        promptVariantId: existingToNewVariantIds.get(cell.promptVariantId) ?? "",
-        testScenarioId: existingToNewScenarioIds.get(cell.testScenarioId) ?? "",
-        prompt: (cell.prompt as Prisma.InputJsonValue) ?? undefined,
-      });
-      for (const modelResponse of modelResponses) {
-        const newModelResponseId = uuidv4();
-        const { outputEvaluations, ...modelResponseData } = modelResponse;
-        modelResponsesToCreate.push({
-          ...modelResponseData,
-          id: newModelResponseId,
-          scenarioVariantCellId: newCellId,
-          output: (modelResponse.output as Prisma.InputJsonValue) ?? undefined,
         });
-        for (const evaluation of outputEvaluations) {
+      }
-          outputEvaluationsToCreate.push({
+
-            ...evaluation,
+      const existingToNewScenarioIds = new Map<string, string>();
-            id: uuidv4(),
+      const scenariosToCreate: Prisma.TestScenarioCreateManyInput[] = [];
-            modelResponseId: newModelResponseId,
+      for (const scenario of existingScenarios) {
-            evaluationId: existingToNewEvaluationIds.get(evaluation.evaluationId) ?? "",
+        const newScenarioId = uuidv4();
+        existingToNewScenarioIds.set(scenario.id, newScenarioId);
+        scenariosToCreate.push({
+          ...scenario,
+          id: newScenarioId,
+          experimentId: newExperimentId,
+          variableValues: scenario.variableValues as Prisma.InputJsonValue,
+        });
+      }
+
+      const existingToNewEvaluationIds = new Map<string, string>();
+      const evaluationsToCreate: Prisma.EvaluationCreateManyInput[] = [];
+      for (const evaluation of evaluations) {
+        const newEvaluationId = uuidv4();
+        existingToNewEvaluationIds.set(evaluation.id, newEvaluationId);
+        evaluationsToCreate.push({
+          ...evaluation,
+          id: newEvaluationId,
+          experimentId: newExperimentId,
+        });
+      }
+
+      const cellsToCreate: Prisma.ScenarioVariantCellCreateManyInput[] = [];
+      const modelResponsesToCreate: Prisma.ModelResponseCreateManyInput[] = [];
+      const outputEvaluationsToCreate: Prisma.OutputEvaluationCreateManyInput[] = [];
+      for (const cell of existingCells) {
+        const newCellId = uuidv4();
+        const { modelResponses, ...cellData } = cell;
+        cellsToCreate.push({
+          ...cellData,
+          id: newCellId,
+          promptVariantId: existingToNewVariantIds.get(cell.promptVariantId) ?? "",
+          testScenarioId: existingToNewScenarioIds.get(cell.testScenarioId) ?? "",
+          prompt: (cell.prompt as Prisma.InputJsonValue) ?? undefined,
+        });
+        for (const modelResponse of modelResponses) {
+          const newModelResponseId = uuidv4();
+          const { outputEvaluations, ...modelResponseData } = modelResponse;
+          modelResponsesToCreate.push({
+            ...modelResponseData,
+            id: newModelResponseId,
+            scenarioVariantCellId: newCellId,
+            output: (modelResponse.output as Prisma.InputJsonValue) ?? undefined,
           });
+          for (const evaluation of outputEvaluations) {
+            outputEvaluationsToCreate.push({
+              ...evaluation,
+              id: uuidv4(),
+              modelResponseId: newModelResponseId,
+              evaluationId: existingToNewEvaluationIds.get(evaluation.evaluationId) ?? "",
+            });
+          }
         }
       }
-    }
 
-    const templateVariablesToCreate: Prisma.TemplateVariableCreateManyInput[] = [];
+      const templateVariablesToCreate: Prisma.TemplateVariableCreateManyInput[] = [];
-    for (const templateVariable of templateVariables) {
+      for (const templateVariable of templateVariables) {
-      templateVariablesToCreate.push({
+        templateVariablesToCreate.push({
-        ...templateVariable,
+          ...templateVariable,
-        id: uuidv4(),
+          id: uuidv4(),
-        experimentId: newExperimentId,
+          experimentId: newExperimentId,
-      });
+        });
-    }
+      }
 
-    const maxSortIndex =
+      const maxSortIndex =
-      (
+        (
-        await prisma.experiment.aggregate({
+          await prisma.experiment.aggregate({
-          _max: {
+            _max: {
-            sortIndex: true,
+              sortIndex: true,
+            },
+          })
+        )._max?.sortIndex ?? 0;
+
+      await prisma.$transaction([
+        prisma.experiment.create({
+          data: {
+            id: newExperimentId,
+            sortIndex: maxSortIndex + 1,
+            label: `${existingExp.label} (forked)`,
+            organizationId: input.organizationId,
           },
-        })
+        }),
-      )._max?.sortIndex ?? 0;
+        prisma.promptVariant.createMany({
+          data: variantsToCreate,
+        }),
+        prisma.testScenario.createMany({
+          data: scenariosToCreate,
+        }),
+        prisma.scenarioVariantCell.createMany({
+          data: cellsToCreate,
+        }),
+        prisma.modelResponse.createMany({
+          data: modelResponsesToCreate,
+        }),
+        prisma.evaluation.createMany({
+          data: evaluationsToCreate,
+        }),
+        prisma.outputEvaluation.createMany({
+          data: outputEvaluationsToCreate,
+        }),
+        prisma.templateVariable.createMany({
+          data: templateVariablesToCreate,
+        }),
+      ]);
 
-    await prisma.$transaction([
+      return newExperimentId;
-      prisma.experiment.create({
+    }),
+
+  create: protectedProcedure
+    .input(z.object({ organizationId: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      await requireCanModifyOrganization(input.organizationId, ctx);
+
+      const maxSortIndex =
+        (
+          await prisma.experiment.aggregate({
+            _max: {
+              sortIndex: true,
+            },
+            where: { organizationId: input.organizationId },
+          })
+        )._max?.sortIndex ?? 0;
+
+      const exp = await prisma.experiment.create({
         data: {
-          id: newExperimentId,
           sortIndex: maxSortIndex + 1,
-          label: `${existingExp.label} (forked)`,
+          label: `Experiment ${maxSortIndex + 1}`,
-          organizationId: (await userOrg(ctx.session.user.id)).id,
+          organizationId: input.organizationId,
         },
-      }),
+      });
-      prisma.promptVariant.createMany({
-        data: variantsToCreate,
-      }),
-      prisma.testScenario.createMany({
-        data: scenariosToCreate,
-      }),
-      prisma.scenarioVariantCell.createMany({
-        data: cellsToCreate,
-      }),
-      prisma.modelResponse.createMany({
-        data: modelResponsesToCreate,
-      }),
-      prisma.evaluation.createMany({
-        data: evaluationsToCreate,
-      }),
-      prisma.outputEvaluation.createMany({
-        data: outputEvaluationsToCreate,
-      }),
-      prisma.templateVariable.createMany({
-        data: templateVariablesToCreate,
-      }),
-    ]);
 
-    return newExperimentId;
+      const [variant, _, scenario1, scenario2, scenario3] = await prisma.$transaction([
-  }),
+        prisma.promptVariant.create({
-
+          data: {
-  create: protectedProcedure.input(z.object({})).mutation(async ({ ctx }) => {
+            experimentId: exp.id,
-    // Anyone can create an experiment
+            label: "Prompt Variant 1",
-    requireNothing(ctx);
+            sortIndex: 0,
-
+            // The interpolated $ is necessary until dedent incorporates
-    const organizationId = (await userOrg(ctx.session.user.id)).id;
+            // https://github.com/dmnd/dedent/pull/46
-
+            promptConstructor: dedent`
-    const maxSortIndex =
-      (
-        await prisma.experiment.aggregate({
-          _max: {
-            sortIndex: true,
-          },
-          where: { organizationId },
-        })
-      )._max?.sortIndex ?? 0;
-
-    const exp = await prisma.experiment.create({
-      data: {
-        sortIndex: maxSortIndex + 1,
-        label: `Experiment ${maxSortIndex + 1}`,
-        organizationId,
-      },
-    });
-
-    const [variant, _, scenario1, scenario2, scenario3] = await prisma.$transaction([
-      prisma.promptVariant.create({
-        data: {
-          experimentId: exp.id,
-          label: "Prompt Variant 1",
-          sortIndex: 0,
-          // The interpolated $ is necessary until dedent incorporates
-          // https://github.com/dmnd/dedent/pull/46
-          promptConstructor: dedent`
           /**
            * Use Javascript to define an OpenAI chat completion
            * (https://platform.openai.com/docs/api-reference/chat/create).
@@ -341,49 +343,49 @@ export const experimentsRouter = createTRPCRouter({
               },
             ],
           });`,
-          model: "gpt-3.5-turbo-0613",
+            model: "gpt-3.5-turbo-0613",
-          modelProvider: "openai/ChatCompletion",
+            modelProvider: "openai/ChatCompletion",
-          promptConstructorVersion,
+            promptConstructorVersion,
-        },
-      }),
-      prisma.templateVariable.create({
-        data: {
-          experimentId: exp.id,
-          label: "language",
-        },
-      }),
-      prisma.testScenario.create({
-        data: {
-          experimentId: exp.id,
-          variableValues: {
-            language: "English",
           },
-        },
+        }),
-      }),
+        prisma.templateVariable.create({
-      prisma.testScenario.create({
+          data: {
-        data: {
+            experimentId: exp.id,
-          experimentId: exp.id,
+            label: "language",
-          variableValues: {
-            language: "Spanish",
           },
-        },
+        }),
-      }),
+        prisma.testScenario.create({
-      prisma.testScenario.create({
+          data: {
-        data: {
+            experimentId: exp.id,
-          experimentId: exp.id,
+            variableValues: {
-          variableValues: {
+              language: "English",
-            language: "German",
+            },
           },
-        },
+        }),
-      }),
+        prisma.testScenario.create({
-    ]);
+          data: {
+            experimentId: exp.id,
+            variableValues: {
+              language: "Spanish",
+            },
+          },
+        }),
+        prisma.testScenario.create({
+          data: {
+            experimentId: exp.id,
+            variableValues: {
+              language: "German",
+            },
+          },
+        }),
+      ]);
 
-    await generateNewCell(variant.id, scenario1.id);
+      await generateNewCell(variant.id, scenario1.id);
-    await generateNewCell(variant.id, scenario2.id);
+      await generateNewCell(variant.id, scenario2.id);
-    await generateNewCell(variant.id, scenario3.id);
+      await generateNewCell(variant.id, scenario3.id);
 
-    return exp;
+      return exp;
-  }),
+    }),
 
   update: protectedProcedure
     .input(z.object({ id: z.string(), updates: z.object({ label: z.string() }) }))

app/src/server/api/routers/externalApi.router.ts

@@ -0,0 +1,205 @@
+import { type Prisma } from "@prisma/client";
+import { type JsonValue } from "type-fest";
+import { z } from "zod";
+import { v4 as uuidv4 } from "uuid";
+import { TRPCError } from "@trpc/server";
+
+import { createTRPCRouter, publicProcedure } from "~/server/api/trpc";
+import { prisma } from "~/server/db";
+import { hashRequest } from "~/server/utils/hashObject";
+
+const reqValidator = z.object({
+  model: z.string(),
+  messages: z.array(z.any()),
+});
+
+const respValidator = z.object({
+  id: z.string(),
+  model: z.string(),
+  usage: z.object({
+    total_tokens: z.number(),
+    prompt_tokens: z.number(),
+    completion_tokens: z.number(),
+  }),
+  choices: z.array(
+    z.object({
+      finish_reason: z.string(),
+    }),
+  ),
+});
+
+export const externalApiRouter = createTRPCRouter({
+  checkCache: publicProcedure
+    .meta({
+      openapi: {
+        method: "POST",
+        path: "/v1/check-cache",
+        description: "Check if a prompt is cached",
+      },
+    })
+    .input(
+      z.object({
+        startTime: z.number().describe("Unix timestamp in milliseconds"),
+        reqPayload: z.unknown().describe("JSON-encoded request payload"),
+        tags: z
+          .record(z.string())
+          .optional()
+          .describe(
+            'Extra tags to attach to the call for filtering. Eg { "userId": "123", "promptId": "populate-title" }',
+          ),
+      }),
+    )
+    .output(
+      z.object({
+        respPayload: z.unknown().optional().describe("JSON-encoded response payload"),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      const apiKey = ctx.apiKey;
+      if (!apiKey) {
+        throw new TRPCError({ code: "UNAUTHORIZED" });
+      }
+      const key = await prisma.apiKey.findUnique({
+        where: { apiKey },
+      });
+      if (!key) {
+        throw new TRPCError({ code: "UNAUTHORIZED" });
+      }
+      const reqPayload = await reqValidator.spa(input.reqPayload);
+      const cacheKey = hashRequest(key.organizationId, reqPayload as JsonValue);
+
+      const existingResponse = await prisma.loggedCallModelResponse.findFirst({
+        where: {
+          cacheKey,
+        },
+        include: {
+          originalLoggedCall: true,
+        },
+        orderBy: {
+          startTime: "desc",
+        },
+      });
+
+      if (!existingResponse) return { respPayload: null };
+
+      await prisma.loggedCall.create({
+        data: {
+          organizationId: key.organizationId,
+          startTime: new Date(input.startTime),
+          cacheHit: true,
+          modelResponseId: existingResponse.id,
+        },
+      });
+
+      return {
+        respPayload: existingResponse.respPayload,
+      };
+    }),
+
+  report: publicProcedure
+    .meta({
+      openapi: {
+        method: "POST",
+        path: "/v1/report",
+        description: "Report an API call",
+      },
+    })
+    .input(
+      z.object({
+        startTime: z.number().describe("Unix timestamp in milliseconds"),
+        endTime: z.number().describe("Unix timestamp in milliseconds"),
+        reqPayload: z.unknown().describe("JSON-encoded request payload"),
+        respPayload: z.unknown().optional().describe("JSON-encoded response payload"),
+        respStatus: z.number().optional().describe("HTTP status code of response"),
+        error: z.string().optional().describe("User-friendly error message"),
+        tags: z
+          .record(z.string())
+          .optional()
+          .describe(
+            'Extra tags to attach to the call for filtering. Eg { "userId": "123", "promptId": "populate-title" }',
+          ),
+      }),
+    )
+    .output(z.void())
+    .mutation(async ({ input, ctx }) => {
+      const apiKey = ctx.apiKey;
+      if (!apiKey) {
+        throw new TRPCError({ code: "UNAUTHORIZED" });
+      }
+      const key = await prisma.apiKey.findUnique({
+        where: { apiKey },
+      });
+      if (!key) {
+        throw new TRPCError({ code: "UNAUTHORIZED" });
+      }
+      const reqPayload = await reqValidator.spa(input.reqPayload);
+      const respPayload = await respValidator.spa(input.respPayload);
+
+      const requestHash = hashRequest(key.organizationId, reqPayload as JsonValue);
+
+      const newLoggedCallId = uuidv4();
+      const newModelResponseId = uuidv4();
+
+      const usage = respPayload.success ? respPayload.data.usage : undefined;
+
+      await prisma.$transaction([
+        prisma.loggedCall.create({
+          data: {
+            id: newLoggedCallId,
+            organizationId: key.organizationId,
+            startTime: new Date(input.startTime),
+            cacheHit: false,
+          },
+        }),
+        prisma.loggedCallModelResponse.create({
+          data: {
+            id: newModelResponseId,
+            originalLoggedCallId: newLoggedCallId,
+            startTime: new Date(input.startTime),
+            endTime: new Date(input.endTime),
+            reqPayload: input.reqPayload as Prisma.InputJsonValue,
+            respPayload: input.respPayload as Prisma.InputJsonValue,
+            respStatus: input.respStatus,
+            error: input.error,
+            durationMs: input.endTime - input.startTime,
+            ...(respPayload.success
+              ? {
+                  cacheKey: requestHash,
+                  inputTokens: usage ? usage.prompt_tokens : undefined,
+                  outputTokens: usage ? usage.completion_tokens : undefined,
+                }
+              : null),
+          },
+        }),
+        // Avoid foreign key constraint error by updating the logged call after the model response is created
+        prisma.loggedCall.update({
+          where: {
+            id: newLoggedCallId,
+          },
+          data: {
+            modelResponseId: newModelResponseId,
+          },
+        }),
+      ]);
+
+      if (input.tags) {
+        const tagsToCreate = Object.entries(input.tags).map(([name, value]) => ({
+          loggedCallId: newLoggedCallId,
+          // sanitize tags
+          name: name.replaceAll(/[^a-zA-Z0-9_]/g, "_"),
+          value,
+        }));
+
+        if (reqPayload.success) {
+          tagsToCreate.push({
+            loggedCallId: newLoggedCallId,
+            name: "$model",
+            value: reqPayload.data.model,
+          });
+        }
+        await prisma.loggedCallTag.createMany({
+          data: tagsToCreate,
+        });
+      }
+    }),
+});

app/src/server/api/routers/organizations.router.ts

@@ -0,0 +1,128 @@
+import { TRPCError } from "@trpc/server";
+import { v4 as uuidv4 } from "uuid";
+import { z } from "zod";
+
+import { createTRPCRouter, protectedProcedure } from "~/server/api/trpc";
+import { prisma } from "~/server/db";
+import { generateApiKey } from "~/server/utils/generateApiKey";
+import userOrg from "~/server/utils/userOrg";
+import {
+  requireCanModifyOrganization,
+  requireCanViewOrganization,
+  requireIsOrgAdmin,
+  requireNothing,
+} from "~/utils/accessControl";
+
+export const organizationsRouter = createTRPCRouter({
+  list: protectedProcedure.query(async ({ ctx }) => {
+    const userId = ctx.session.user.id;
+    requireNothing(ctx);
+
+    if (!userId) {
+      return null;
+    }
+
+    const organizations = await prisma.organization.findMany({
+      where: {
+        organizationUsers: {
+          some: { userId: ctx.session.user.id },
+        },
+      },
+      orderBy: {
+        createdAt: "asc",
+      },
+    });
+
+    if (!organizations.length) {
+      // TODO: We should move this to a separate endpoint that is called on sign up
+      const personalOrg = await userOrg(userId);
+      organizations.push(personalOrg);
+    }
+
+    return organizations;
+  }),
+  get: protectedProcedure.input(z.object({ id: z.string() })).query(async ({ input, ctx }) => {
+    await requireCanViewOrganization(input.id, ctx);
+    const [org, userRole] = await prisma.$transaction([
+      prisma.organization.findUnique({
+        where: {
+          id: input.id,
+        },
+        include: {
+          apiKeys: true,
+          personalOrgUser: true,
+        },
+      }),
+      prisma.organizationUser.findFirst({
+        where: {
+          userId: ctx.session.user.id,
+          organizationId: input.id,
+          role: {
+            in: ["ADMIN", "MEMBER"],
+          },
+        },
+      }),
+    ]);
+
+    if (!org) {
+      throw new TRPCError({ code: "NOT_FOUND" });
+    }
+
+    return {
+      ...org,
+      role: userRole?.role ?? null,
+    };
+  }),
+  update: protectedProcedure
+    .input(z.object({ id: z.string(), updates: z.object({ name: z.string() }) }))
+    .mutation(async ({ input, ctx }) => {
+      await requireCanModifyOrganization(input.id, ctx);
+      return await prisma.organization.update({
+        where: {
+          id: input.id,
+        },
+        data: {
+          name: input.updates.name,
+        },
+      });
+    }),
+  create: protectedProcedure
+    .input(z.object({ name: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      requireNothing(ctx);
+      const newOrgId = uuidv4();
+      const [newOrg] = await prisma.$transaction([
+        prisma.organization.create({
+          data: {
+            id: newOrgId,
+            name: input.name,
+          },
+        }),
+        prisma.organizationUser.create({
+          data: {
+            userId: ctx.session.user.id,
+            organizationId: newOrgId,
+            role: "ADMIN",
+          },
+        }),
+        prisma.apiKey.create({
+          data: {
+            name: "Default API Key",
+            organizationId: newOrgId,
+            apiKey: generateApiKey(),
+          },
+        }),
+      ]);
+      return newOrg;
+    }),
+  delete: protectedProcedure
+    .input(z.object({ id: z.string() }))
+    .mutation(async ({ input, ctx }) => {
+      await requireIsOrgAdmin(input.id, ctx);
+      return await prisma.organization.delete({
+        where: {
+          id: input.id,
+        },
+      });
+    }),
+});

app/src/server/api/trpc.ts

@@ -11,6 +11,7 @@ import { initTRPC, TRPCError } from "@trpc/server";
 import { type CreateNextContextOptions } from "@trpc/server/adapters/next";
 import { type Session } from "next-auth";
 import superjson from "superjson";
+import { type OpenApiMeta } from "trpc-openapi";
 import { ZodError } from "zod";
 import { getServerAuthSession } from "~/server/auth";
 import { prisma } from "~/server/db";
@@ -26,6 +27,7 @@ import { capturePath } from "~/utils/analytics/serverAnalytics";
 
 type CreateContextOptions = {
   session: Session | null;
+  apiKey: string | null;
 };
 
 // eslint-disable-next-line @typescript-eslint/no-empty-function
@@ -44,6 +46,7 @@ const noOp = () => {};
 export const createInnerTRPCContext = (opts: CreateContextOptions) => {
   return {
     session: opts.session,
+    apiKey: opts.apiKey,
     prisma,
     markAccessControlRun: noOp,
   };
@@ -61,8 +64,11 @@ export const createTRPCContext = async (opts: CreateNextContextOptions) => {
   // Get the session from the server using the getServerSession wrapper function
   const session = await getServerAuthSession({ req, res });
 
+  const apiKey = req.headers["x-openpipe-api-key"] as string | null;
+
   return createInnerTRPCContext({
     session,
+    apiKey,
   });
 };
 
@@ -76,18 +82,21 @@ export const createTRPCContext = async (opts: CreateNextContextOptions) => {
 
 export type TRPCContext = Awaited<ReturnType<typeof createTRPCContext>>;
 
-const t = initTRPC.context<typeof createTRPCContext>().create({
+const t = initTRPC
-  transformer: superjson,
+  .context<typeof createTRPCContext>()
-  errorFormatter({ shape, error }) {
+  .meta<OpenApiMeta>()
-    return {
+  .create({
-      ...shape,
+    transformer: superjson,
-      data: {
+    errorFormatter({ shape, error }) {
-        ...shape.data,
+      return {
-        zodError: error.cause instanceof ZodError ? error.cause.flatten() : null,
+        ...shape,
-      },
+        data: {
-    };
+          ...shape.data,
-  },
+          zodError: error.cause instanceof ZodError ? error.cause.flatten() : null,
-});
+        },
+      };
+    },
+  });
 
 /**
  * 3. ROUTER & PROCEDURE (THE IMPORTANT BIT)

app/src/server/auth.ts

@@ -2,9 +2,16 @@ import { PrismaAdapter } from "@next-auth/prisma-adapter";
 import { type GetServerSidePropsContext } from "next";
 import { getServerSession, type NextAuthOptions, type DefaultSession } from "next-auth";
 import { prisma } from "~/server/db";
-import GitHubProvider from "next-auth/providers/github";
+import GitHubModule from "next-auth/providers/github";
 import { env } from "~/env.mjs";
 
+// The client codegen script doesn't properly read the default export
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const untypedGitHubModule = GitHubModule as unknown as any;
+const GitHubProvider: typeof GitHubModule = untypedGitHubModule.default
+  ? untypedGitHubModule.default
+  : untypedGitHubModule;
+
 /**
  * Module augmentation for `next-auth` types. Allows us to add custom properties to the `session`
  * object and keep type safety.

app/src/server/db.ts

@@ -1,6 +1,61 @@
-import { PrismaClient } from "@prisma/client";
+import {
+  type Experiment,
+  type PromptVariant,
+  type TestScenario,
+  type TemplateVariable,
+  type ScenarioVariantCell,
+  type ModelResponse,
+  type Evaluation,
+  type OutputEvaluation,
+  type Dataset,
+  type DatasetEntry,
+  type Organization,
+  type OrganizationUser,
+  type WorldChampEntrant,
+  type LoggedCall,
+  type LoggedCallModelResponse,
+  type LoggedCallTag,
+  type ApiKey,
+  type Account,
+  type Session,
+  type User,
+  type VerificationToken,
+  PrismaClient,
+} from "@prisma/client";
+import { Kysely, PostgresDialect } from "kysely";
+// TODO: Revert to normal import when our tsconfig.json is fixed
+// import { Pool } from "pg";
+import PGModule from "pg";
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const UntypedPool = PGModule.Pool as any;
+const Pool = (UntypedPool.default ? UntypedPool.default : UntypedPool) as typeof PGModule.Pool;
+
 import { env } from "~/env.mjs";
 
+interface DB {
+  Experiment: Experiment;
+  PromptVariant: PromptVariant;
+  TestScenario: TestScenario;
+  TemplateVariable: TemplateVariable;
+  ScenarioVariantCell: ScenarioVariantCell;
+  ModelResponse: ModelResponse;
+  Evaluation: Evaluation;
+  OutputEvaluation: OutputEvaluation;
+  Dataset: Dataset;
+  DatasetEntry: DatasetEntry;
+  Organization: Organization;
+  OrganizationUser: OrganizationUser;
+  WorldChampEntrant: WorldChampEntrant;
+  LoggedCall: LoggedCall;
+  LoggedCallModelResponse: LoggedCallModelResponse;
+  LoggedCallTag: LoggedCallTag;
+  ApiKey: ApiKey;
+  Account: Account;
+  Session: Session;
+  User: User;
+  VerificationToken: VerificationToken;
+}
+
 const globalForPrisma = globalThis as unknown as {
   prisma: PrismaClient | undefined;
 };
@@ -14,4 +69,12 @@ export const prisma =
         : ["error"],
   });
 
+export const kysely = new Kysely<DB>({
+  dialect: new PostgresDialect({
+    pool: new Pool({
+      connectionString: env.DATABASE_URL,
+    }),
+  }),
+});
+
 if (env.NODE_ENV !== "production") globalForPrisma.prisma = prisma;

app/src/server/scripts/backfillApiKeys.ts

@@ -0,0 +1,33 @@
+import { type Prisma } from "@prisma/client";
+import { prisma } from "~/server/db";
+import { generateApiKey } from "~/server/utils/generateApiKey";
+
+console.log("backfilling api keys");
+
+const organizations = await prisma.organization.findMany({
+  include: {
+    apiKeys: true,
+  },
+});
+
+console.log(`found ${organizations.length} organizations`);
+
+const apiKeysToCreate: Prisma.ApiKeyCreateManyInput[] = [];
+
+for (const org of organizations) {
+  if (!org.apiKeys.length) {
+    apiKeysToCreate.push({
+      name: "Default API Key",
+      organizationId: org.id,
+      apiKey: generateApiKey(),
+    });
+  }
+}
+
+console.log(`creating ${apiKeysToCreate.length} api keys`);
+
+await prisma.apiKey.createMany({
+  data: apiKeysToCreate,
+});
+
+console.log("done");

app/src/server/scripts/client-codegen.ts

@@ -0,0 +1,32 @@
+import "dotenv/config";
+import { openApiDocument } from "~/pages/api/openapi.json";
+import fs from "fs";
+import path from "path";
+import { execSync } from "child_process";
+
+console.log("Exporting public OpenAPI schema to client-libs/schema.json");
+
+const scriptPath = import.meta.url.replace("file://", "");
+const clientLibsPath = path.join(path.dirname(scriptPath), "../../../../client-libs");
+
+const schemaPath = path.join(clientLibsPath, "schema.json");
+
+console.log("Exporting schema");
+fs.writeFileSync(schemaPath, JSON.stringify(openApiDocument, null, 2), "utf-8");
+
+console.log("Generating Typescript client");
+
+const tsClientPath = path.join(clientLibsPath, "typescript/codegen");
+
+fs.rmSync(tsClientPath, { recursive: true, force: true });
+
+execSync(
+  `pnpm dlx @openapitools/openapi-generator-cli generate -i "${schemaPath}" -g typescript-axios -o "${tsClientPath}"`,
+  {
+    stdio: "inherit",
+  },
+);
+
+console.log("Done!");
+
+process.exit(0);

app/src/server/scripts/test-queries.ts

@@ -0,0 +1,63 @@
+import dayjs from "dayjs";
+import { prisma } from "../db";
+
+const projectId = "1234";
+
+// Find all calls in the last 24 hours
+const responses = await prisma.loggedCall.findMany({
+  where: {
+    organizationId: projectId,
+    startTime: {
+      gt: dayjs()
+        .subtract(24 * 3600)
+        .toDate(),
+    },
+  },
+  include: {
+    modelResponse: true,
+  },
+  orderBy: {
+    startTime: "desc",
+  },
+});
+
+// Find all calls in the last 24 hours with promptId 'hello-world'
+const helloWorld = await prisma.loggedCall.findMany({
+  where: {
+    organizationId: projectId,
+    startTime: {
+      gt: dayjs()
+        .subtract(24 * 3600)
+        .toDate(),
+    },
+    tags: {
+      some: {
+        name: "promptId",
+        value: "hello-world",
+      },
+    },
+  },
+  include: {
+    modelResponse: true,
+  },
+  orderBy: {
+    startTime: "desc",
+  },
+});
+
+// Total spent on OpenAI in the last month
+const totalSpent = await prisma.loggedCallModelResponse.aggregate({
+  _sum: {
+    totalCost: true,
+  },
+  where: {
+    originalLoggedCall: {
+      organizationId: projectId,
+    },
+    startTime: {
+      gt: dayjs()
+        .subtract(30 * 24 * 3600)
+        .toDate(),
+    },
+  },
+});

app/src/server/tasks/queryModel.task.ts

@@ -1,10 +1,10 @@
 import { type Prisma } from "@prisma/client";
-import { type JsonObject } from "type-fest";
+import { type JsonValue, type JsonObject } from "type-fest";
 import modelProviders from "~/modelProviders/modelProviders";
 import { prisma } from "~/server/db";
 import { wsConnection } from "~/utils/wsConnection";
 import { runEvalsForOutput } from "../utils/evaluations";
-import hashPrompt from "../utils/hashPrompt";
+import hashObject from "../utils/hashObject";
 import defineTask from "./defineTask";
 import parsePromptConstructor from "~/promptConstructor/parse";
 
@@ -99,7 +99,7 @@ export const queryModel = defineTask<QueryModelJob>("queryModel", async (task) =
       }
     : null;
 
-  const inputHash = hashPrompt(prompt);
+  const inputHash = hashObject(prompt as JsonValue);
 
   let modelResponse = await prisma.modelResponse.create({
     data: {

app/src/server/tasks/worker.ts

@@ -17,7 +17,7 @@ const taskList = registeredTasks.reduce((acc, task) => {
 // Run a worker to execute jobs:
 const runner = await run({
   connectionString: env.DATABASE_URL,
-  concurrency: 50,
+  concurrency: 10,
   // Install signal handlers for graceful shutdown on SIGINT, SIGTERM, etc
   noHandleSignals: false,
   pollInterval: 1000,

app/src/server/utils/generateApiKey.ts

@@ -0,0 +1,5 @@
+import cryptoRandomString from "crypto-random-string";
+
+const KEY_LENGTH = 42;
+
+export const generateApiKey = () => `opc_${cryptoRandomString({ length: KEY_LENGTH })}`;

app/src/server/utils/generateNewCell.ts

@@ -1,7 +1,7 @@
 import { Prisma } from "@prisma/client";
 import { prisma } from "../db";
 import { type JsonObject } from "type-fest";
-import hashPrompt from "./hashPrompt";
+import hashObject from "./hashObject";
 import { omit } from "lodash-es";
 import { queueQueryModel } from "../tasks/queryModel.task";
 import parsePromptConstructor from "~/promptConstructor/parse";
@@ -57,7 +57,7 @@ export const generateNewCell = async (
     return;
   }
 
-  const inputHash = hashPrompt(parsedConstructFn);
+  const inputHash = hashObject(parsedConstructFn);
 
   cell = await prisma.scenarioVariantCell.create({
     data: {

app/src/server/utils/hashObject.ts

@@ -1,6 +1,5 @@
 import crypto from "crypto";
 import { type JsonValue } from "type-fest";
-import { ParsedPromptConstructor } from "~/promptConstructor/parse";
 
 function sortKeys(obj: JsonValue): JsonValue {
   if (typeof obj !== "object" || obj === null) {
@@ -25,9 +24,17 @@ function sortKeys(obj: JsonValue): JsonValue {
   return sortedObj;
 }
 
-export default function hashPrompt(prompt: ParsedPromptConstructor<any>): string {
+export function hashRequest(organizationId: string, reqPayload: JsonValue): string {
+  const obj = {
+    organizationId,
+    reqPayload,
+  };
+  return hashObject(obj);
+}
+
+export default function hashObject(obj: JsonValue): string {
   // Sort object keys recursively
-  const sortedObj = sortKeys(prompt as unknown as JsonValue);
+  const sortedObj = sortKeys(obj);
 
   // Convert to JSON and hash it
   const str = JSON.stringify(sortedObj);

app/src/server/utils/openai.ts

@@ -1,6 +1,13 @@
 import { env } from "~/env.mjs";
 
-import OpenAI from "openai";
+import { default as OriginalOpenAI } from "openai";
+// import { OpenAI } from "openpipe";
+
+const openAIConfig = { apiKey: env.OPENAI_API_KEY ?? "dummy-key" };
 
 // Set a dummy key so it doesn't fail at build time
-export const openai = new OpenAI({ apiKey: env.OPENAI_API_KEY ?? "dummy-key" });
+// export const openai = env.OPENPIPE_API_KEY
+//   ? new OpenAI.OpenAI(openAIConfig)
+//   : new OriginalOpenAI(openAIConfig);
+
+export const openai = new OriginalOpenAI(openAIConfig);

app/src/server/utils/userOrg.ts

@@ -1,4 +1,5 @@
 import { prisma } from "~/server/db";
+import { generateApiKey } from "./generateApiKey";
 
 export default async function userOrg(userId: string) {
   return await prisma.organization.upsert({
@@ -14,6 +15,14 @@ export default async function userOrg(userId: string) {
           role: "ADMIN",
         },
       },
+      apiKeys: {
+        create: [
+          {
+            name: "Default API Key",
+            apiKey: generateApiKey(),
+          },
+        ],
+      },
     },
   });
 }

app/src/state/store.ts

@@ -14,6 +14,8 @@ export type State = {
   api: APIClient | null;
   setApi: (api: APIClient) => void;
   sharedVariantEditor: SharedVariantEditorSlice;
+  selectedOrgId: string | null;
+  setSelectedOrgId: (orgId: string) => void;
 };
 
 export type SliceCreator<T> = StateCreator<State, [["zustand/immer", never]], [], T>;
@@ -39,6 +41,11 @@ const useBaseStore = create<State, [["zustand/immer", never]]>(
         state.drawerOpen = false;
       }),
     sharedVariantEditor: createVariantEditorSlice(set, get, ...rest),
+    selectedOrgId: null,
+    setSelectedOrgId: (orgId: string) =>
+      set((state) => {
+        state.selectedOrgId = orgId;
+      }),
   })),
 );
 

app/src/theme/ChakraThemeProvider.tsx

@@ -1,6 +1,5 @@
-import { extendTheme } from "@chakra-ui/react";
+import { extendTheme, defineStyleConfig, ChakraProvider } from "@chakra-ui/react";
 import "@fontsource/inconsolata";
-import { ChakraProvider } from "@chakra-ui/react";
 import { modalAnatomy } from "@chakra-ui/anatomy";
 import { createMultiStyleConfigHelpers } from "@chakra-ui/styled-system";
 
@@ -18,6 +17,13 @@ const modalTheme = defineMultiStyleConfig({
   }),
 });
 
+const Divider = defineStyleConfig({
+  baseStyle: {
+    borderColor: "gray.300",
+    backgroundColor: "gray.300",
+  },
+});
+
 const theme = extendTheme({
   styles: {
     global: (props: { colorMode: "dark" | "light" }) => ({
@@ -53,6 +59,7 @@ const theme = extendTheme({
       },
     },
     Modal: modalTheme,
+    Divider,
   },
 });
 

app/src/utils/accessControl.ts

@@ -16,6 +16,68 @@ export const requireNothing = (ctx: TRPCContext) => {
   ctx.markAccessControlRun();
 };
 
+export const requireIsOrgAdmin = async (organizationId: string, ctx: TRPCContext) => {
+  const userId = ctx.session?.user.id;
+  if (!userId) {
+    throw new TRPCError({ code: "UNAUTHORIZED" });
+  }
+
+  const isAdmin = await prisma.organizationUser.findFirst({
+    where: {
+      userId,
+      organizationId,
+      role: "ADMIN",
+    },
+  });
+
+  if (!isAdmin) {
+    throw new TRPCError({ code: "UNAUTHORIZED" });
+  }
+
+  ctx.markAccessControlRun();
+};
+
+export const requireCanViewOrganization = async (organizationId: string, ctx: TRPCContext) => {
+  const userId = ctx.session?.user.id;
+  if (!userId) {
+    throw new TRPCError({ code: "UNAUTHORIZED" });
+  }
+
+  const canView = await prisma.organizationUser.findFirst({
+    where: {
+      userId,
+      organizationId,
+    },
+  });
+
+  if (!canView) {
+    throw new TRPCError({ code: "UNAUTHORIZED" });
+  }
+
+  ctx.markAccessControlRun();
+};
+
+export const requireCanModifyOrganization = async (organizationId: string, ctx: TRPCContext) => {
+  const userId = ctx.session?.user.id;
+  if (!userId) {
+    throw new TRPCError({ code: "UNAUTHORIZED" });
+  }
+
+  const canModify = await prisma.organizationUser.findFirst({
+    where: {
+      userId,
+      organizationId,
+      role: { in: [OrganizationUserRole.ADMIN, OrganizationUserRole.MEMBER] },
+    },
+  });
+
+  if (!canModify) {
+    throw new TRPCError({ code: "UNAUTHORIZED" });
+  }
+
+  ctx.markAccessControlRun();
+};
+
 export const requireCanViewDataset = async (datasetId: string, ctx: TRPCContext) => {
   const dataset = await prisma.dataset.findFirst({
     where: {

app/src/utils/dayjs.ts

@@ -1,9 +1,11 @@
 import dayjs from "dayjs";
 import duration from "dayjs/plugin/duration";
 import relativeTime from "dayjs/plugin/relativeTime";
+import timezone from "dayjs/plugin/timezone";
 
 dayjs.extend(duration);
 dayjs.extend(relativeTime);
+dayjs.extend(timezone);
 
 export const formatTimePast = (date: Date) =>
   dayjs.duration(dayjs(date).diff(dayjs())).humanize(true);

app/src/utils/hooks.ts

@@ -2,6 +2,15 @@ import { useRouter } from "next/router";
 import { type RefObject, useCallback, useEffect, useRef, useState } from "react";
 import { api } from "~/utils/api";
 import { NumberParam, useQueryParam, withDefault } from "use-query-params";
+import { useAppStore } from "~/state/store";
+
+export const useExperiments = () => {
+  const selectedOrgId = useAppStore((state) => state.selectedOrgId);
+  return api.experiments.list.useQuery(
+    { organizationId: selectedOrgId ?? "" },
+    { enabled: !!selectedOrgId },
+  );
+};
 
 export const useExperiment = () => {
   const router = useRouter();
@@ -17,6 +26,14 @@ export const useExperimentAccess = () => {
   return useExperiment().data?.access ?? { canView: false, canModify: false };
 };
 
+export const useDatasets = () => {
+  const selectedOrgId = useAppStore((state) => state.selectedOrgId);
+  return api.datasets.list.useQuery(
+    { organizationId: selectedOrgId ?? "" },
+    { enabled: !!selectedOrgId },
+  );
+};
+
 export const useDataset = () => {
   const router = useRouter();
   const dataset = api.datasets.get.useQuery(
@@ -132,3 +149,8 @@ export const useScenario = (scenarioId: string) => {
 };
 
 export const useVisibleScenarioIds = () => useScenarios().data?.scenarios.map((s) => s.id) ?? [];
+
+export const useSelectedOrg = () => {
+  const selectedOrgId = useAppStore((state) => state.selectedOrgId);
+  return api.organizations.get.useQuery({ id: selectedOrgId ?? "" }, { enabled: !!selectedOrgId });
+};

app/src/utils/useSocket.ts

@@ -11,7 +11,6 @@ export default function useSocket<T>(channel?: string | null) {
   useEffect(() => {
     if (!channel) return;
 
-    console.log("connecting to channel", channel);
     // Create websocket connection
     socketRef.current = io(url);
 

client-libs/schema.json

@@ -0,0 +1,187 @@
+{
+  "openapi": "3.0.3",
+  "info": {
+    "title": "OpenPipe API",
+    "description": "The public API for reporting API calls to OpenPipe",
+    "version": "0.1.0"
+  },
+  "servers": [
+    {
+      "url": "https://app.openpipe.ai/api"
+    }
+  ],
+  "paths": {
+    "/v1/check-cache": {
+      "post": {
+        "operationId": "externalApi-checkCache",
+        "description": "Check if a prompt is cached",
+        "requestBody": {
+          "required": true,
+          "content": {
+            "application/json": {
+              "schema": {
+                "type": "object",
+                "properties": {
+                  "startTime": {
+                    "type": "number",
+                    "description": "Unix timestamp in milliseconds"
+                  },
+                  "reqPayload": {
+                    "description": "JSON-encoded request payload"
+                  },
+                  "tags": {
+                    "type": "object",
+                    "additionalProperties": {
+                      "type": "string"
+                    },
+                    "description": "Extra tags to attach to the call for filtering. Eg { \"userId\": \"123\", \"promptId\": \"populate-title\" }"
+                  }
+                },
+                "required": [
+                  "startTime"
+                ],
+                "additionalProperties": false
+              }
+            }
+          }
+        },
+        "parameters": [],
+        "responses": {
+          "200": {
+            "description": "Successful response",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "respPayload": {
+                      "description": "JSON-encoded response payload"
+                    }
+                  },
+                  "additionalProperties": false
+                }
+              }
+            }
+          },
+          "default": {
+            "$ref": "#/components/responses/error"
+          }
+        }
+      }
+    },
+    "/v1/report": {
+      "post": {
+        "operationId": "externalApi-report",
+        "description": "Report an API call",
+        "requestBody": {
+          "required": true,
+          "content": {
+            "application/json": {
+              "schema": {
+                "type": "object",
+                "properties": {
+                  "startTime": {
+                    "type": "number",
+                    "description": "Unix timestamp in milliseconds"
+                  },
+                  "endTime": {
+                    "type": "number",
+                    "description": "Unix timestamp in milliseconds"
+                  },
+                  "reqPayload": {
+                    "description": "JSON-encoded request payload"
+                  },
+                  "respPayload": {
+                    "description": "JSON-encoded response payload"
+                  },
+                  "respStatus": {
+                    "type": "number",
+                    "description": "HTTP status code of response"
+                  },
+                  "error": {
+                    "type": "string",
+                    "description": "User-friendly error message"
+                  },
+                  "tags": {
+                    "type": "object",
+                    "additionalProperties": {
+                      "type": "string"
+                    },
+                    "description": "Extra tags to attach to the call for filtering. Eg { \"userId\": \"123\", \"promptId\": \"populate-title\" }"
+                  }
+                },
+                "required": [
+                  "startTime",
+                  "endTime"
+                ],
+                "additionalProperties": false
+              }
+            }
+          }
+        },
+        "parameters": [],
+        "responses": {
+          "200": {
+            "description": "Successful response",
+            "content": {
+              "application/json": {
+                "schema": {}
+              }
+            }
+          },
+          "default": {
+            "$ref": "#/components/responses/error"
+          }
+        }
+      }
+    }
+  },
+  "components": {
+    "securitySchemes": {
+      "Authorization": {
+        "type": "http",
+        "scheme": "bearer"
+      }
+    },
+    "responses": {
+      "error": {
+        "description": "Error response",
+        "content": {
+          "application/json": {
+            "schema": {
+              "type": "object",
+              "properties": {
+                "message": {
+                  "type": "string"
+                },
+                "code": {
+                  "type": "string"
+                },
+                "issues": {
+                  "type": "array",
+                  "items": {
+                    "type": "object",
+                    "properties": {
+                      "message": {
+                        "type": "string"
+                      }
+                    },
+                    "required": [
+                      "message"
+                    ],
+                    "additionalProperties": false
+                  }
+                }
+              },
+              "required": [
+                "message",
+                "code"
+              ],
+              "additionalProperties": false
+            }
+          }
+        }
+      }
+    }
+  }
+}

client-libs/typescript/.gitignore

@@ -0,0 +1,2 @@
+node_modules
+dist

client-libs/typescript/codegen/.gitignore

@@ -0,0 +1,4 @@
+wwwroot/*.js
+node_modules
+typings
+dist

client-libs/typescript/codegen/.npmignore

@@ -0,0 +1 @@
+# empty npmignore to ensure all required files (e.g., in the dist folder) are published by npm

client-libs/typescript/codegen/.openapi-generator-ignore

@@ -0,0 +1,23 @@
+# OpenAPI Generator Ignore
+# Generated by openapi-generator https://github.com/openapitools/openapi-generator
+
+# Use this file to prevent files from being overwritten by the generator.
+# The patterns follow closely to .gitignore or .dockerignore.
+
+# As an example, the C# client generator defines ApiClient.cs.
+# You can make changes and tell OpenAPI Generator to ignore just this file by uncommenting the following line:
+#ApiClient.cs
+
+# You can match any string of characters against a directory, file or extension with a single asterisk (*):
+#foo/*/qux
+# The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux
+
+# You can recursively match patterns against a directory, file or extension with a double asterisk (**):
+#foo/**/qux
+# This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux
+
+# You can also negate patterns with an exclamation (!).
+# For example, you can ignore all files in a docs folder with the file extension .md:
+#docs/*.md
+# Then explicitly reverse the ignore rule for a single file:
+#!docs/README.md

client-libs/typescript/codegen/.openapi-generator/FILES

@@ -0,0 +1,9 @@
+.gitignore
+.npmignore
+.openapi-generator-ignore
+api.ts
+base.ts
+common.ts
+configuration.ts
+git_push.sh
+index.ts

client-libs/typescript/codegen/.openapi-generator/VERSION

@@ -0,0 +1 @@
+6.6.0

client-libs/typescript/codegen/api.js

@@ -0,0 +1,272 @@
+"use strict";
+/* tslint:disable */
+/* eslint-disable */
+/**
+ * OpenPipe API
+ * The public API for reporting API calls to OpenPipe
+ *
+ * The version of the OpenAPI document: 0.1.0
+ *
+ *
+ * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+ * https://openapi-generator.tech
+ * Do not edit the class manually.
+ */
+var __extends = (this && this.__extends) || (function () {
+    var extendStatics = function (d, b) {
+        extendStatics = Object.setPrototypeOf ||
+            ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
+            function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
+        return extendStatics(d, b);
+    };
+    return function (d, b) {
+        if (typeof b !== "function" && b !== null)
+            throw new TypeError("Class extends value " + String(b) + " is not a constructor or null");
+        extendStatics(d, b);
+        function __() { this.constructor = d; }
+        d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
+    };
+})();
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DefaultApi = exports.DefaultApiFactory = exports.DefaultApiFp = exports.DefaultApiAxiosParamCreator = void 0;
+var axios_1 = require("axios");
+// Some imports not used depending on template conditions
+// @ts-ignore
+var common_1 = require("./common");
+// @ts-ignore
+var base_1 = require("./base");
+/**
+ * DefaultApi - axios parameter creator
+ * @export
+ */
+var DefaultApiAxiosParamCreator = function (configuration) {
+    var _this = this;
+    return {
+        /**
+         * Check if a prompt is cached
+         * @param {ExternalApiCheckCacheRequest} externalApiCheckCacheRequest
+         * @param {*} [options] Override http request option.
+         * @throws {RequiredError}
+         */
+        externalApiCheckCache: function (externalApiCheckCacheRequest, options) {
+            if (options === void 0) { options = {}; }
+            return __awaiter(_this, void 0, void 0, function () {
+                var localVarPath, localVarUrlObj, baseOptions, localVarRequestOptions, localVarHeaderParameter, localVarQueryParameter, headersFromBaseOptions;
+                return __generator(this, function (_a) {
+                    // verify required parameter 'externalApiCheckCacheRequest' is not null or undefined
+                    (0, common_1.assertParamExists)('externalApiCheckCache', 'externalApiCheckCacheRequest', externalApiCheckCacheRequest);
+                    localVarPath = "/v1/check-cache";
+                    localVarUrlObj = new URL(localVarPath, common_1.DUMMY_BASE_URL);
+                    if (configuration) {
+                        baseOptions = configuration.baseOptions;
+                    }
+                    localVarRequestOptions = __assign(__assign({ method: 'POST' }, baseOptions), options);
+                    localVarHeaderParameter = {};
+                    localVarQueryParameter = {};
+                    localVarHeaderParameter['Content-Type'] = 'application/json';
+                    (0, common_1.setSearchParams)(localVarUrlObj, localVarQueryParameter);
+                    headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
+                    localVarRequestOptions.headers = __assign(__assign(__assign({}, localVarHeaderParameter), headersFromBaseOptions), options.headers);
+                    localVarRequestOptions.data = (0, common_1.serializeDataIfNeeded)(externalApiCheckCacheRequest, localVarRequestOptions, configuration);
+                    return [2 /*return*/, {
+                            url: (0, common_1.toPathString)(localVarUrlObj),
+                            options: localVarRequestOptions,
+                        }];
+                });
+            });
+        },
+        /**
+         * Report an API call
+         * @param {ExternalApiReportRequest} externalApiReportRequest
+         * @param {*} [options] Override http request option.
+         * @throws {RequiredError}
+         */
+        externalApiReport: function (externalApiReportRequest, options) {
+            if (options === void 0) { options = {}; }
+            return __awaiter(_this, void 0, void 0, function () {
+                var localVarPath, localVarUrlObj, baseOptions, localVarRequestOptions, localVarHeaderParameter, localVarQueryParameter, headersFromBaseOptions;
+                return __generator(this, function (_a) {
+                    // verify required parameter 'externalApiReportRequest' is not null or undefined
+                    (0, common_1.assertParamExists)('externalApiReport', 'externalApiReportRequest', externalApiReportRequest);
+                    localVarPath = "/v1/report";
+                    localVarUrlObj = new URL(localVarPath, common_1.DUMMY_BASE_URL);
+                    if (configuration) {
+                        baseOptions = configuration.baseOptions;
+                    }
+                    localVarRequestOptions = __assign(__assign({ method: 'POST' }, baseOptions), options);
+                    localVarHeaderParameter = {};
+                    localVarQueryParameter = {};
+                    localVarHeaderParameter['Content-Type'] = 'application/json';
+                    (0, common_1.setSearchParams)(localVarUrlObj, localVarQueryParameter);
+                    headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
+                    localVarRequestOptions.headers = __assign(__assign(__assign({}, localVarHeaderParameter), headersFromBaseOptions), options.headers);
+                    localVarRequestOptions.data = (0, common_1.serializeDataIfNeeded)(externalApiReportRequest, localVarRequestOptions, configuration);
+                    return [2 /*return*/, {
+                            url: (0, common_1.toPathString)(localVarUrlObj),
+                            options: localVarRequestOptions,
+                        }];
+                });
+            });
+        },
+    };
+};
+exports.DefaultApiAxiosParamCreator = DefaultApiAxiosParamCreator;
+/**
+ * DefaultApi - functional programming interface
+ * @export
+ */
+var DefaultApiFp = function (configuration) {
+    var localVarAxiosParamCreator = (0, exports.DefaultApiAxiosParamCreator)(configuration);
+    return {
+        /**
+         * Check if a prompt is cached
+         * @param {ExternalApiCheckCacheRequest} externalApiCheckCacheRequest
+         * @param {*} [options] Override http request option.
+         * @throws {RequiredError}
+         */
+        externalApiCheckCache: function (externalApiCheckCacheRequest, options) {
+            return __awaiter(this, void 0, void 0, function () {
+                var localVarAxiosArgs;
+                return __generator(this, function (_a) {
+                    switch (_a.label) {
+                        case 0: return [4 /*yield*/, localVarAxiosParamCreator.externalApiCheckCache(externalApiCheckCacheRequest, options)];
+                        case 1:
+                            localVarAxiosArgs = _a.sent();
+                            return [2 /*return*/, (0, common_1.createRequestFunction)(localVarAxiosArgs, axios_1.default, base_1.BASE_PATH, configuration)];
+                    }
+                });
+            });
+        },
+        /**
+         * Report an API call
+         * @param {ExternalApiReportRequest} externalApiReportRequest
+         * @param {*} [options] Override http request option.
+         * @throws {RequiredError}
+         */
+        externalApiReport: function (externalApiReportRequest, options) {
+            return __awaiter(this, void 0, void 0, function () {
+                var localVarAxiosArgs;
+                return __generator(this, function (_a) {
+                    switch (_a.label) {
+                        case 0: return [4 /*yield*/, localVarAxiosParamCreator.externalApiReport(externalApiReportRequest, options)];
+                        case 1:
+                            localVarAxiosArgs = _a.sent();
+                            return [2 /*return*/, (0, common_1.createRequestFunction)(localVarAxiosArgs, axios_1.default, base_1.BASE_PATH, configuration)];
+                    }
+                });
+            });
+        },
+    };
+};
+exports.DefaultApiFp = DefaultApiFp;
+/**
+ * DefaultApi - factory interface
+ * @export
+ */
+var DefaultApiFactory = function (configuration, basePath, axios) {
+    var localVarFp = (0, exports.DefaultApiFp)(configuration);
+    return {
+        /**
+         * Check if a prompt is cached
+         * @param {ExternalApiCheckCacheRequest} externalApiCheckCacheRequest
+         * @param {*} [options] Override http request option.
+         * @throws {RequiredError}
+         */
+        externalApiCheckCache: function (externalApiCheckCacheRequest, options) {
+            return localVarFp.externalApiCheckCache(externalApiCheckCacheRequest, options).then(function (request) { return request(axios, basePath); });
+        },
+        /**
+         * Report an API call
+         * @param {ExternalApiReportRequest} externalApiReportRequest
+         * @param {*} [options] Override http request option.
+         * @throws {RequiredError}
+         */
+        externalApiReport: function (externalApiReportRequest, options) {
+            return localVarFp.externalApiReport(externalApiReportRequest, options).then(function (request) { return request(axios, basePath); });
+        },
+    };
+};
+exports.DefaultApiFactory = DefaultApiFactory;
+/**
+ * DefaultApi - object-oriented interface
+ * @export
+ * @class DefaultApi
+ * @extends {BaseAPI}
+ */
+var DefaultApi = /** @class */ (function (_super) {
+    __extends(DefaultApi, _super);
+    function DefaultApi() {
+        return _super !== null && _super.apply(this, arguments) || this;
+    }
+    /**
+     * Check if a prompt is cached
+     * @param {ExternalApiCheckCacheRequest} externalApiCheckCacheRequest
+     * @param {*} [options] Override http request option.
+     * @throws {RequiredError}
+     * @memberof DefaultApi
+     */
+    DefaultApi.prototype.externalApiCheckCache = function (externalApiCheckCacheRequest, options) {
+        var _this = this;
+        return (0, exports.DefaultApiFp)(this.configuration).externalApiCheckCache(externalApiCheckCacheRequest, options).then(function (request) { return request(_this.axios, _this.basePath); });
+    };
+    /**
+     * Report an API call
+     * @param {ExternalApiReportRequest} externalApiReportRequest
+     * @param {*} [options] Override http request option.
+     * @throws {RequiredError}
+     * @memberof DefaultApi
+     */
+    DefaultApi.prototype.externalApiReport = function (externalApiReportRequest, options) {
+        var _this = this;
+        return (0, exports.DefaultApiFp)(this.configuration).externalApiReport(externalApiReportRequest, options).then(function (request) { return request(_this.axios, _this.basePath); });
+    };
+    return DefaultApi;
+}(base_1.BaseAPI));
+exports.DefaultApi = DefaultApi;

client-libs/typescript/codegen/api.ts

@@ -0,0 +1,319 @@
+/* tslint:disable */
+/* eslint-disable */
+/**
+ * OpenPipe API
+ * The public API for reporting API calls to OpenPipe
+ *
+ * The version of the OpenAPI document: 0.1.0
+ * 
+ *
+ * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+ * https://openapi-generator.tech
+ * Do not edit the class manually.
+ */
+
+
+import type { Configuration } from './configuration';
+import type { AxiosPromise, AxiosInstance, AxiosRequestConfig } from 'axios';
+import globalAxios from 'axios';
+// Some imports not used depending on template conditions
+// @ts-ignore
+import { DUMMY_BASE_URL, assertParamExists, setApiKeyToObject, setBasicAuthToObject, setBearerAuthToObject, setOAuthToObject, setSearchParams, serializeDataIfNeeded, toPathString, createRequestFunction } from './common';
+import type { RequestArgs } from './base';
+// @ts-ignore
+import { BASE_PATH, COLLECTION_FORMATS, BaseAPI, RequiredError } from './base';
+
+/**
+ * 
+ * @export
+ * @interface ExternalApiCheckCache200Response
+ */
+export interface ExternalApiCheckCache200Response {
+    /**
+     * JSON-encoded response payload
+     * @type {any}
+     * @memberof ExternalApiCheckCache200Response
+     */
+    'respPayload'?: any;
+}
+/**
+ * 
+ * @export
+ * @interface ExternalApiCheckCacheDefaultResponse
+ */
+export interface ExternalApiCheckCacheDefaultResponse {
+    /**
+     * 
+     * @type {string}
+     * @memberof ExternalApiCheckCacheDefaultResponse
+     */
+    'message': string;
+    /**
+     * 
+     * @type {string}
+     * @memberof ExternalApiCheckCacheDefaultResponse
+     */
+    'code': string;
+    /**
+     * 
+     * @type {Array<ExternalApiCheckCacheDefaultResponseIssuesInner>}
+     * @memberof ExternalApiCheckCacheDefaultResponse
+     */
+    'issues'?: Array<ExternalApiCheckCacheDefaultResponseIssuesInner>;
+}
+/**
+ * 
+ * @export
+ * @interface ExternalApiCheckCacheDefaultResponseIssuesInner
+ */
+export interface ExternalApiCheckCacheDefaultResponseIssuesInner {
+    /**
+     * 
+     * @type {string}
+     * @memberof ExternalApiCheckCacheDefaultResponseIssuesInner
+     */
+    'message': string;
+}
+/**
+ * 
+ * @export
+ * @interface ExternalApiCheckCacheRequest
+ */
+export interface ExternalApiCheckCacheRequest {
+    /**
+     * Unix timestamp in milliseconds
+     * @type {number}
+     * @memberof ExternalApiCheckCacheRequest
+     */
+    'startTime': number;
+    /**
+     * JSON-encoded request payload
+     * @type {any}
+     * @memberof ExternalApiCheckCacheRequest
+     */
+    'reqPayload'?: any;
+    /**
+     * Extra tags to attach to the call for filtering. Eg { \"userId\": \"123\", \"promptId\": \"populate-title\" }
+     * @type {{ [key: string]: string; }}
+     * @memberof ExternalApiCheckCacheRequest
+     */
+    'tags'?: { [key: string]: string; };
+}
+/**
+ * 
+ * @export
+ * @interface ExternalApiReportRequest
+ */
+export interface ExternalApiReportRequest {
+    /**
+     * Unix timestamp in milliseconds
+     * @type {number}
+     * @memberof ExternalApiReportRequest
+     */
+    'startTime': number;
+    /**
+     * Unix timestamp in milliseconds
+     * @type {number}
+     * @memberof ExternalApiReportRequest
+     */
+    'endTime': number;
+    /**
+     * JSON-encoded request payload
+     * @type {any}
+     * @memberof ExternalApiReportRequest
+     */
+    'reqPayload'?: any;
+    /**
+     * JSON-encoded response payload
+     * @type {any}
+     * @memberof ExternalApiReportRequest
+     */
+    'respPayload'?: any;
+    /**
+     * HTTP status code of response
+     * @type {number}
+     * @memberof ExternalApiReportRequest
+     */
+    'respStatus'?: number;
+    /**
+     * User-friendly error message
+     * @type {string}
+     * @memberof ExternalApiReportRequest
+     */
+    'error'?: string;
+    /**
+     * Extra tags to attach to the call for filtering. Eg { \"userId\": \"123\", \"promptId\": \"populate-title\" }
+     * @type {{ [key: string]: string; }}
+     * @memberof ExternalApiReportRequest
+     */
+    'tags'?: { [key: string]: string; };
+}
+
+/**
+ * DefaultApi - axios parameter creator
+ * @export
+ */
+export const DefaultApiAxiosParamCreator = function (configuration?: Configuration) {
+    return {
+        /**
+         * Check if a prompt is cached
+         * @param {ExternalApiCheckCacheRequest} externalApiCheckCacheRequest 
+         * @param {*} [options] Override http request option.
+         * @throws {RequiredError}
+         */
+        externalApiCheckCache: async (externalApiCheckCacheRequest: ExternalApiCheckCacheRequest, options: AxiosRequestConfig = {}): Promise<RequestArgs> => {
+            // verify required parameter 'externalApiCheckCacheRequest' is not null or undefined
+            assertParamExists('externalApiCheckCache', 'externalApiCheckCacheRequest', externalApiCheckCacheRequest)
+            const localVarPath = `/v1/check-cache`;
+            // use dummy base URL string because the URL constructor only accepts absolute URLs.
+            const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL);
+            let baseOptions;
+            if (configuration) {
+                baseOptions = configuration.baseOptions;
+            }
+
+            const localVarRequestOptions = { method: 'POST', ...baseOptions, ...options};
+            const localVarHeaderParameter = {} as any;
+            const localVarQueryParameter = {} as any;
+
+
+    
+            localVarHeaderParameter['Content-Type'] = 'application/json';
+
+            setSearchParams(localVarUrlObj, localVarQueryParameter);
+            let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
+            localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers};
+            localVarRequestOptions.data = serializeDataIfNeeded(externalApiCheckCacheRequest, localVarRequestOptions, configuration)
+
+            return {
+                url: toPathString(localVarUrlObj),
+                options: localVarRequestOptions,
+            };
+        },
+        /**
+         * Report an API call
+         * @param {ExternalApiReportRequest} externalApiReportRequest 
+         * @param {*} [options] Override http request option.
+         * @throws {RequiredError}
+         */
+        externalApiReport: async (externalApiReportRequest: ExternalApiReportRequest, options: AxiosRequestConfig = {}): Promise<RequestArgs> => {
+            // verify required parameter 'externalApiReportRequest' is not null or undefined
+            assertParamExists('externalApiReport', 'externalApiReportRequest', externalApiReportRequest)
+            const localVarPath = `/v1/report`;
+            // use dummy base URL string because the URL constructor only accepts absolute URLs.
+            const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL);
+            let baseOptions;
+            if (configuration) {
+                baseOptions = configuration.baseOptions;
+            }
+
+            const localVarRequestOptions = { method: 'POST', ...baseOptions, ...options};
+            const localVarHeaderParameter = {} as any;
+            const localVarQueryParameter = {} as any;
+
+
+    
+            localVarHeaderParameter['Content-Type'] = 'application/json';
+
+            setSearchParams(localVarUrlObj, localVarQueryParameter);
+            let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {};
+            localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers};
+            localVarRequestOptions.data = serializeDataIfNeeded(externalApiReportRequest, localVarRequestOptions, configuration)
+
+            return {
+                url: toPathString(localVarUrlObj),
+                options: localVarRequestOptions,
+            };
+        },
+    }
+};
+
+/**
+ * DefaultApi - functional programming interface
+ * @export
+ */
+export const DefaultApiFp = function(configuration?: Configuration) {
+    const localVarAxiosParamCreator = DefaultApiAxiosParamCreator(configuration)
+    return {
+        /**
+         * Check if a prompt is cached
+         * @param {ExternalApiCheckCacheRequest} externalApiCheckCacheRequest 
+         * @param {*} [options] Override http request option.
+         * @throws {RequiredError}
+         */
+        async externalApiCheckCache(externalApiCheckCacheRequest: ExternalApiCheckCacheRequest, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<ExternalApiCheckCache200Response>> {
+            const localVarAxiosArgs = await localVarAxiosParamCreator.externalApiCheckCache(externalApiCheckCacheRequest, options);
+            return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
+        },
+        /**
+         * Report an API call
+         * @param {ExternalApiReportRequest} externalApiReportRequest 
+         * @param {*} [options] Override http request option.
+         * @throws {RequiredError}
+         */
+        async externalApiReport(externalApiReportRequest: ExternalApiReportRequest, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<any>> {
+            const localVarAxiosArgs = await localVarAxiosParamCreator.externalApiReport(externalApiReportRequest, options);
+            return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
+        },
+    }
+};
+
+/**
+ * DefaultApi - factory interface
+ * @export
+ */
+export const DefaultApiFactory = function (configuration?: Configuration, basePath?: string, axios?: AxiosInstance) {
+    const localVarFp = DefaultApiFp(configuration)
+    return {
+        /**
+         * Check if a prompt is cached
+         * @param {ExternalApiCheckCacheRequest} externalApiCheckCacheRequest 
+         * @param {*} [options] Override http request option.
+         * @throws {RequiredError}
+         */
+        externalApiCheckCache(externalApiCheckCacheRequest: ExternalApiCheckCacheRequest, options?: any): AxiosPromise<ExternalApiCheckCache200Response> {
+            return localVarFp.externalApiCheckCache(externalApiCheckCacheRequest, options).then((request) => request(axios, basePath));
+        },
+        /**
+         * Report an API call
+         * @param {ExternalApiReportRequest} externalApiReportRequest 
+         * @param {*} [options] Override http request option.
+         * @throws {RequiredError}
+         */
+        externalApiReport(externalApiReportRequest: ExternalApiReportRequest, options?: any): AxiosPromise<any> {
+            return localVarFp.externalApiReport(externalApiReportRequest, options).then((request) => request(axios, basePath));
+        },
+    };
+};
+
+/**
+ * DefaultApi - object-oriented interface
+ * @export
+ * @class DefaultApi
+ * @extends {BaseAPI}
+ */
+export class DefaultApi extends BaseAPI {
+    /**
+     * Check if a prompt is cached
+     * @param {ExternalApiCheckCacheRequest} externalApiCheckCacheRequest 
+     * @param {*} [options] Override http request option.
+     * @throws {RequiredError}
+     * @memberof DefaultApi
+     */
+    public externalApiCheckCache(externalApiCheckCacheRequest: ExternalApiCheckCacheRequest, options?: AxiosRequestConfig) {
+        return DefaultApiFp(this.configuration).externalApiCheckCache(externalApiCheckCacheRequest, options).then((request) => request(this.axios, this.basePath));
+    }
+
+    /**
+     * Report an API call
+     * @param {ExternalApiReportRequest} externalApiReportRequest 
+     * @param {*} [options] Override http request option.
+     * @throws {RequiredError}
+     * @memberof DefaultApi
+     */
+    public externalApiReport(externalApiReportRequest: ExternalApiReportRequest, options?: AxiosRequestConfig) {
+        return DefaultApiFp(this.configuration).externalApiReport(externalApiReportRequest, options).then((request) => request(this.axios, this.basePath));
+    }
+}
+
+

client-libs/typescript/codegen/base.js

@@ -0,0 +1,80 @@
+"use strict";
+/* tslint:disable */
+/* eslint-disable */
+/**
+ * OpenPipe API
+ * The public API for reporting API calls to OpenPipe
+ *
+ * The version of the OpenAPI document: 0.1.0
+ *
+ *
+ * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+ * https://openapi-generator.tech
+ * Do not edit the class manually.
+ */
+var __extends = (this && this.__extends) || (function () {
+    var extendStatics = function (d, b) {
+        extendStatics = Object.setPrototypeOf ||
+            ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
+            function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
+        return extendStatics(d, b);
+    };
+    return function (d, b) {
+        if (typeof b !== "function" && b !== null)
+            throw new TypeError("Class extends value " + String(b) + " is not a constructor or null");
+        extendStatics(d, b);
+        function __() { this.constructor = d; }
+        d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RequiredError = exports.BaseAPI = exports.COLLECTION_FORMATS = exports.BASE_PATH = void 0;
+var axios_1 = require("axios");
+exports.BASE_PATH = "https://app.openpipe.ai/api".replace(/\/+$/, "");
+/**
+ *
+ * @export
+ */
+exports.COLLECTION_FORMATS = {
+    csv: ",",
+    ssv: " ",
+    tsv: "\t",
+    pipes: "|",
+};
+/**
+ *
+ * @export
+ * @class BaseAPI
+ */
+var BaseAPI = /** @class */ (function () {
+    function BaseAPI(configuration, basePath, axios) {
+        if (basePath === void 0) { basePath = exports.BASE_PATH; }
+        if (axios === void 0) { axios = axios_1.default; }
+        this.basePath = basePath;
+        this.axios = axios;
+        if (configuration) {
+            this.configuration = configuration;
+            this.basePath = configuration.basePath || this.basePath;
+        }
+    }
+    return BaseAPI;
+}());
+exports.BaseAPI = BaseAPI;
+;
+/**
+ *
+ * @export
+ * @class RequiredError
+ * @extends {Error}
+ */
+var RequiredError = /** @class */ (function (_super) {
+    __extends(RequiredError, _super);
+    function RequiredError(field, msg) {
+        var _this = _super.call(this, msg) || this;
+        _this.field = field;
+        _this.name = "RequiredError";
+        return _this;
+    }
+    return RequiredError;
+}(Error));
+exports.RequiredError = RequiredError;

client-libs/typescript/codegen/base.ts

@@ -0,0 +1,72 @@
+/* tslint:disable */
+/* eslint-disable */
+/**
+ * OpenPipe API
+ * The public API for reporting API calls to OpenPipe
+ *
+ * The version of the OpenAPI document: 0.1.0
+ * 
+ *
+ * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+ * https://openapi-generator.tech
+ * Do not edit the class manually.
+ */
+
+
+import type { Configuration } from './configuration';
+// Some imports not used depending on template conditions
+// @ts-ignore
+import type { AxiosPromise, AxiosInstance, AxiosRequestConfig } from 'axios';
+import globalAxios from 'axios';
+
+export const BASE_PATH = "https://app.openpipe.ai/api".replace(/\/+$/, "");
+
+/**
+ *
+ * @export
+ */
+export const COLLECTION_FORMATS = {
+    csv: ",",
+    ssv: " ",
+    tsv: "\t",
+    pipes: "|",
+};
+
+/**
+ *
+ * @export
+ * @interface RequestArgs
+ */
+export interface RequestArgs {
+    url: string;
+    options: AxiosRequestConfig;
+}
+
+/**
+ *
+ * @export
+ * @class BaseAPI
+ */
+export class BaseAPI {
+    protected configuration: Configuration | undefined;
+
+    constructor(configuration?: Configuration, protected basePath: string = BASE_PATH, protected axios: AxiosInstance = globalAxios) {
+        if (configuration) {
+            this.configuration = configuration;
+            this.basePath = configuration.basePath || this.basePath;
+        }
+    }
+};
+
+/**
+ *
+ * @export
+ * @class RequiredError
+ * @extends {Error}
+ */
+export class RequiredError extends Error {
+    constructor(public field: string, msg?: string) {
+        super(msg);
+        this.name = "RequiredError"
+    }
+}

client-libs/typescript/codegen/common.js

@@ -0,0 +1,252 @@
+"use strict";
+/* tslint:disable */
+/* eslint-disable */
+/**
+ * OpenPipe API
+ * The public API for reporting API calls to OpenPipe
+ *
+ * The version of the OpenAPI document: 0.1.0
+ *
+ *
+ * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+ * https://openapi-generator.tech
+ * Do not edit the class manually.
+ */
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
+    return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createRequestFunction = exports.toPathString = exports.serializeDataIfNeeded = exports.setSearchParams = exports.setOAuthToObject = exports.setBearerAuthToObject = exports.setBasicAuthToObject = exports.setApiKeyToObject = exports.assertParamExists = exports.DUMMY_BASE_URL = void 0;
+var base_1 = require("./base");
+/**
+ *
+ * @export
+ */
+exports.DUMMY_BASE_URL = 'https://example.com';
+/**
+ *
+ * @throws {RequiredError}
+ * @export
+ */
+var assertParamExists = function (functionName, paramName, paramValue) {
+    if (paramValue === null || paramValue === undefined) {
+        throw new base_1.RequiredError(paramName, "Required parameter ".concat(paramName, " was null or undefined when calling ").concat(functionName, "."));
+    }
+};
+exports.assertParamExists = assertParamExists;
+/**
+ *
+ * @export
+ */
+var setApiKeyToObject = function (object, keyParamName, configuration) {
+    return __awaiter(this, void 0, void 0, function () {
+        var localVarApiKeyValue, _a;
+        return __generator(this, function (_b) {
+            switch (_b.label) {
+                case 0:
+                    if (!(configuration && configuration.apiKey)) return [3 /*break*/, 5];
+                    if (!(typeof configuration.apiKey === 'function')) return [3 /*break*/, 2];
+                    return [4 /*yield*/, configuration.apiKey(keyParamName)];
+                case 1:
+                    _a = _b.sent();
+                    return [3 /*break*/, 4];
+                case 2: return [4 /*yield*/, configuration.apiKey];
+                case 3:
+                    _a = _b.sent();
+                    _b.label = 4;
+                case 4:
+                    localVarApiKeyValue = _a;
+                    object[keyParamName] = localVarApiKeyValue;
+                    _b.label = 5;
+                case 5: return [2 /*return*/];
+            }
+        });
+    });
+};
+exports.setApiKeyToObject = setApiKeyToObject;
+/**
+ *
+ * @export
+ */
+var setBasicAuthToObject = function (object, configuration) {
+    if (configuration && (configuration.username || configuration.password)) {
+        object["auth"] = { username: configuration.username, password: configuration.password };
+    }
+};
+exports.setBasicAuthToObject = setBasicAuthToObject;
+/**
+ *
+ * @export
+ */
+var setBearerAuthToObject = function (object, configuration) {
+    return __awaiter(this, void 0, void 0, function () {
+        var accessToken, _a;
+        return __generator(this, function (_b) {
+            switch (_b.label) {
+                case 0:
+                    if (!(configuration && configuration.accessToken)) return [3 /*break*/, 5];
+                    if (!(typeof configuration.accessToken === 'function')) return [3 /*break*/, 2];
+                    return [4 /*yield*/, configuration.accessToken()];
+                case 1:
+                    _a = _b.sent();
+                    return [3 /*break*/, 4];
+                case 2: return [4 /*yield*/, configuration.accessToken];
+                case 3:
+                    _a = _b.sent();
+                    _b.label = 4;
+                case 4:
+                    accessToken = _a;
+                    object["Authorization"] = "Bearer " + accessToken;
+                    _b.label = 5;
+                case 5: return [2 /*return*/];
+            }
+        });
+    });
+};
+exports.setBearerAuthToObject = setBearerAuthToObject;
+/**
+ *
+ * @export
+ */
+var setOAuthToObject = function (object, name, scopes, configuration) {
+    return __awaiter(this, void 0, void 0, function () {
+        var localVarAccessTokenValue, _a;
+        return __generator(this, function (_b) {
+            switch (_b.label) {
+                case 0:
+                    if (!(configuration && configuration.accessToken)) return [3 /*break*/, 5];
+                    if (!(typeof configuration.accessToken === 'function')) return [3 /*break*/, 2];
+                    return [4 /*yield*/, configuration.accessToken(name, scopes)];
+                case 1:
+                    _a = _b.sent();
+                    return [3 /*break*/, 4];
+                case 2: return [4 /*yield*/, configuration.accessToken];
+                case 3:
+                    _a = _b.sent();
+                    _b.label = 4;
+                case 4:
+                    localVarAccessTokenValue = _a;
+                    object["Authorization"] = "Bearer " + localVarAccessTokenValue;
+                    _b.label = 5;
+                case 5: return [2 /*return*/];
+            }
+        });
+    });
+};
+exports.setOAuthToObject = setOAuthToObject;
+function setFlattenedQueryParams(urlSearchParams, parameter, key) {
+    if (key === void 0) { key = ""; }
+    if (parameter == null)
+        return;
+    if (typeof parameter === "object") {
+        if (Array.isArray(parameter)) {
+            parameter.forEach(function (item) { return setFlattenedQueryParams(urlSearchParams, item, key); });
+        }
+        else {
+            Object.keys(parameter).forEach(function (currentKey) {
+                return setFlattenedQueryParams(urlSearchParams, parameter[currentKey], "".concat(key).concat(key !== '' ? '.' : '').concat(currentKey));
+            });
+        }
+    }
+    else {
+        if (urlSearchParams.has(key)) {
+            urlSearchParams.append(key, parameter);
+        }
+        else {
+            urlSearchParams.set(key, parameter);
+        }
+    }
+}
+/**
+ *
+ * @export
+ */
+var setSearchParams = function (url) {
+    var objects = [];
+    for (var _i = 1; _i < arguments.length; _i++) {
+        objects[_i - 1] = arguments[_i];
+    }
+    var searchParams = new URLSearchParams(url.search);
+    setFlattenedQueryParams(searchParams, objects);
+    url.search = searchParams.toString();
+};
+exports.setSearchParams = setSearchParams;
+/**
+ *
+ * @export
+ */
+var serializeDataIfNeeded = function (value, requestOptions, configuration) {
+    var nonString = typeof value !== 'string';
+    var needsSerialization = nonString && configuration && configuration.isJsonMime
+        ? configuration.isJsonMime(requestOptions.headers['Content-Type'])
+        : nonString;
+    return needsSerialization
+        ? JSON.stringify(value !== undefined ? value : {})
+        : (value || "");
+};
+exports.serializeDataIfNeeded = serializeDataIfNeeded;
+/**
+ *
+ * @export
+ */
+var toPathString = function (url) {
+    return url.pathname + url.search + url.hash;
+};
+exports.toPathString = toPathString;
+/**
+ *
+ * @export
+ */
+var createRequestFunction = function (axiosArgs, globalAxios, BASE_PATH, configuration) {
+    return function (axios, basePath) {
+        if (axios === void 0) { axios = globalAxios; }
+        if (basePath === void 0) { basePath = BASE_PATH; }
+        var axiosRequestArgs = __assign(__assign({}, axiosArgs.options), { url: ((configuration === null || configuration === void 0 ? void 0 : configuration.basePath) || basePath) + axiosArgs.url });
+        return axios.request(axiosRequestArgs);
+    };
+};
+exports.createRequestFunction = createRequestFunction;

client-libs/typescript/codegen/common.ts

@@ -0,0 +1,150 @@
+/* tslint:disable */
+/* eslint-disable */
+/**
+ * OpenPipe API
+ * The public API for reporting API calls to OpenPipe
+ *
+ * The version of the OpenAPI document: 0.1.0
+ * 
+ *
+ * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+ * https://openapi-generator.tech
+ * Do not edit the class manually.
+ */
+
+
+import type { Configuration } from "./configuration";
+import type { RequestArgs } from "./base";
+import type { AxiosInstance, AxiosResponse } from 'axios';
+import { RequiredError } from "./base";
+
+/**
+ *
+ * @export
+ */
+export const DUMMY_BASE_URL = 'https://example.com'
+
+/**
+ *
+ * @throws {RequiredError}
+ * @export
+ */
+export const assertParamExists = function (functionName: string, paramName: string, paramValue: unknown) {
+    if (paramValue === null || paramValue === undefined) {
+        throw new RequiredError(paramName, `Required parameter ${paramName} was null or undefined when calling ${functionName}.`);
+    }
+}
+
+/**
+ *
+ * @export
+ */
+export const setApiKeyToObject = async function (object: any, keyParamName: string, configuration?: Configuration) {
+    if (configuration && configuration.apiKey) {
+        const localVarApiKeyValue = typeof configuration.apiKey === 'function'
+            ? await configuration.apiKey(keyParamName)
+            : await configuration.apiKey;
+        object[keyParamName] = localVarApiKeyValue;
+    }
+}
+
+/**
+ *
+ * @export
+ */
+export const setBasicAuthToObject = function (object: any, configuration?: Configuration) {
+    if (configuration && (configuration.username || configuration.password)) {
+        object["auth"] = { username: configuration.username, password: configuration.password };
+    }
+}
+
+/**
+ *
+ * @export
+ */
+export const setBearerAuthToObject = async function (object: any, configuration?: Configuration) {
+    if (configuration && configuration.accessToken) {
+        const accessToken = typeof configuration.accessToken === 'function'
+            ? await configuration.accessToken()
+            : await configuration.accessToken;
+        object["Authorization"] = "Bearer " + accessToken;
+    }
+}
+
+/**
+ *
+ * @export
+ */
+export const setOAuthToObject = async function (object: any, name: string, scopes: string[], configuration?: Configuration) {
+    if (configuration && configuration.accessToken) {
+        const localVarAccessTokenValue = typeof configuration.accessToken === 'function'
+            ? await configuration.accessToken(name, scopes)
+            : await configuration.accessToken;
+        object["Authorization"] = "Bearer " + localVarAccessTokenValue;
+    }
+}
+
+function setFlattenedQueryParams(urlSearchParams: URLSearchParams, parameter: any, key: string = ""): void {
+    if (parameter == null) return;
+    if (typeof parameter === "object") {
+        if (Array.isArray(parameter)) {
+            (parameter as any[]).forEach(item => setFlattenedQueryParams(urlSearchParams, item, key));
+        } 
+        else {
+            Object.keys(parameter).forEach(currentKey => 
+                setFlattenedQueryParams(urlSearchParams, parameter[currentKey], `${key}${key !== '' ? '.' : ''}${currentKey}`)
+            );
+        }
+    } 
+    else {
+        if (urlSearchParams.has(key)) {
+            urlSearchParams.append(key, parameter);
+        } 
+        else {
+            urlSearchParams.set(key, parameter);
+        }
+    }
+}
+
+/**
+ *
+ * @export
+ */
+export const setSearchParams = function (url: URL, ...objects: any[]) {
+    const searchParams = new URLSearchParams(url.search);
+    setFlattenedQueryParams(searchParams, objects);
+    url.search = searchParams.toString();
+}
+
+/**
+ *
+ * @export
+ */
+export const serializeDataIfNeeded = function (value: any, requestOptions: any, configuration?: Configuration) {
+    const nonString = typeof value !== 'string';
+    const needsSerialization = nonString && configuration && configuration.isJsonMime
+        ? configuration.isJsonMime(requestOptions.headers['Content-Type'])
+        : nonString;
+    return needsSerialization
+        ? JSON.stringify(value !== undefined ? value : {})
+        : (value || "");
+}
+
+/**
+ *
+ * @export
+ */
+export const toPathString = function (url: URL) {
+    return url.pathname + url.search + url.hash
+}
+
+/**
+ *
+ * @export
+ */
+export const createRequestFunction = function (axiosArgs: RequestArgs, globalAxios: AxiosInstance, BASE_PATH: string, configuration?: Configuration) {
+    return <T = unknown, R = AxiosResponse<T>>(axios: AxiosInstance = globalAxios, basePath: string = BASE_PATH) => {
+        const axiosRequestArgs = {...axiosArgs.options, url: (configuration?.basePath || basePath) + axiosArgs.url};
+        return axios.request<T, R>(axiosRequestArgs);
+    };
+}

client-libs/typescript/codegen/configuration.js

@@ -0,0 +1,44 @@
+"use strict";
+/* tslint:disable */
+/* eslint-disable */
+/**
+ * OpenPipe API
+ * The public API for reporting API calls to OpenPipe
+ *
+ * The version of the OpenAPI document: 0.1.0
+ *
+ *
+ * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+ * https://openapi-generator.tech
+ * Do not edit the class manually.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Configuration = void 0;
+var Configuration = /** @class */ (function () {
+    function Configuration(param) {
+        if (param === void 0) { param = {}; }
+        this.apiKey = param.apiKey;
+        this.username = param.username;
+        this.password = param.password;
+        this.accessToken = param.accessToken;
+        this.basePath = param.basePath;
+        this.baseOptions = param.baseOptions;
+        this.formDataCtor = param.formDataCtor;
+    }
+    /**
+     * Check if the given MIME is a JSON MIME.
+     * JSON MIME examples:
+     *   application/json
+     *   application/json; charset=UTF8
+     *   APPLICATION/JSON
+     *   application/vnd.company+json
+     * @param mime - MIME (Multipurpose Internet Mail Extensions)
+     * @return True if the given MIME is JSON, false otherwise.
+     */
+    Configuration.prototype.isJsonMime = function (mime) {
+        var jsonMime = new RegExp('^(application\/json|[^;/ \t]+\/[^;/ \t]+[+]json)[ \t]*(;.*)?$', 'i');
+        return mime !== null && (jsonMime.test(mime) || mime.toLowerCase() === 'application/json-patch+json');
+    };
+    return Configuration;
+}());
+exports.Configuration = Configuration;

client-libs/typescript/codegen/configuration.ts

@@ -0,0 +1,101 @@
+/* tslint:disable */
+/* eslint-disable */
+/**
+ * OpenPipe API
+ * The public API for reporting API calls to OpenPipe
+ *
+ * The version of the OpenAPI document: 0.1.0
+ * 
+ *
+ * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+ * https://openapi-generator.tech
+ * Do not edit the class manually.
+ */
+
+
+export interface ConfigurationParameters {
+    apiKey?: string | Promise<string> | ((name: string) => string) | ((name: string) => Promise<string>);
+    username?: string;
+    password?: string;
+    accessToken?: string | Promise<string> | ((name?: string, scopes?: string[]) => string) | ((name?: string, scopes?: string[]) => Promise<string>);
+    basePath?: string;
+    baseOptions?: any;
+    formDataCtor?: new () => any;
+}
+
+export class Configuration {
+    /**
+     * parameter for apiKey security
+     * @param name security name
+     * @memberof Configuration
+     */
+    apiKey?: string | Promise<string> | ((name: string) => string) | ((name: string) => Promise<string>);
+    /**
+     * parameter for basic security
+     *
+     * @type {string}
+     * @memberof Configuration
+     */
+    username?: string;
+    /**
+     * parameter for basic security
+     *
+     * @type {string}
+     * @memberof Configuration
+     */
+    password?: string;
+    /**
+     * parameter for oauth2 security
+     * @param name security name
+     * @param scopes oauth2 scope
+     * @memberof Configuration
+     */
+    accessToken?: string | Promise<string> | ((name?: string, scopes?: string[]) => string) | ((name?: string, scopes?: string[]) => Promise<string>);
+    /**
+     * override base path
+     *
+     * @type {string}
+     * @memberof Configuration
+     */
+    basePath?: string;
+    /**
+     * base options for axios calls
+     *
+     * @type {any}
+     * @memberof Configuration
+     */
+    baseOptions?: any;
+    /**
+     * The FormData constructor that will be used to create multipart form data
+     * requests. You can inject this here so that execution environments that
+     * do not support the FormData class can still run the generated client.
+     *
+     * @type {new () => FormData}
+     */
+    formDataCtor?: new () => any;
+
+    constructor(param: ConfigurationParameters = {}) {
+        this.apiKey = param.apiKey;
+        this.username = param.username;
+        this.password = param.password;
+        this.accessToken = param.accessToken;
+        this.basePath = param.basePath;
+        this.baseOptions = param.baseOptions;
+        this.formDataCtor = param.formDataCtor;
+    }
+
+    /**
+     * Check if the given MIME is a JSON MIME.
+     * JSON MIME examples:
+     *   application/json
+     *   application/json; charset=UTF8
+     *   APPLICATION/JSON
+     *   application/vnd.company+json
+     * @param mime - MIME (Multipurpose Internet Mail Extensions)
+     * @return True if the given MIME is JSON, false otherwise.
+     */
+    public isJsonMime(mime: string): boolean {
+        const jsonMime: RegExp = new RegExp('^(application\/json|[^;/ \t]+\/[^;/ \t]+[+]json)[ \t]*(;.*)?$', 'i');
+        return mime !== null && (jsonMime.test(mime) || mime.toLowerCase() === 'application/json-patch+json');
+    }
+}

client-libs/typescript/codegen/git_push.sh

@@ -0,0 +1,57 @@
+#!/bin/sh
+# ref: https://help.github.com/articles/adding-an-existing-project-to-github-using-the-command-line/
+#
+# Usage example: /bin/sh ./git_push.sh wing328 openapi-petstore-perl "minor update" "gitlab.com"
+
+git_user_id=$1
+git_repo_id=$2
+release_note=$3
+git_host=$4
+
+if [ "$git_host" = "" ]; then
+    git_host="github.com"
+    echo "[INFO] No command line input provided. Set \$git_host to $git_host"
+fi
+
+if [ "$git_user_id" = "" ]; then
+    git_user_id="GIT_USER_ID"
+    echo "[INFO] No command line input provided. Set \$git_user_id to $git_user_id"
+fi
+
+if [ "$git_repo_id" = "" ]; then
+    git_repo_id="GIT_REPO_ID"
+    echo "[INFO] No command line input provided. Set \$git_repo_id to $git_repo_id"
+fi
+
+if [ "$release_note" = "" ]; then
+    release_note="Minor update"
+    echo "[INFO] No command line input provided. Set \$release_note to $release_note"
+fi
+
+# Initialize the local directory as a Git repository
+git init
+
+# Adds the files in the local repository and stages them for commit.
+git add .
+
+# Commits the tracked changes and prepares them to be pushed to a remote repository.
+git commit -m "$release_note"
+
+# Sets the new remote
+git_remote=$(git remote)
+if [ "$git_remote" = "" ]; then # git remote not defined
+
+    if [ "$GIT_TOKEN" = "" ]; then
+        echo "[INFO] \$GIT_TOKEN (environment variable) is not set. Using the git credential in your environment."
+        git remote add origin https://${git_host}/${git_user_id}/${git_repo_id}.git
+    else
+        git remote add origin https://${git_user_id}:"${GIT_TOKEN}"@${git_host}/${git_user_id}/${git_repo_id}.git
+    fi
+
+fi
+
+git pull origin master
+
+# Pushes (Forces) the changes in the local repository up to the remote repository
+echo "Git pushing to https://${git_host}/${git_user_id}/${git_repo_id}.git"
+git push origin master 2>&1 | grep -v 'To https'

client-libs/typescript/codegen/index.js

@@ -0,0 +1,31 @@
+"use strict";
+/* tslint:disable */
+/* eslint-disable */
+/**
+ * OpenPipe API
+ * The public API for reporting API calls to OpenPipe
+ *
+ * The version of the OpenAPI document: 0.1.0
+ *
+ *
+ * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+ * https://openapi-generator.tech
+ * Do not edit the class manually.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./api"), exports);
+__exportStar(require("./configuration"), exports);

client-libs/typescript/codegen/index.ts

@@ -0,0 +1,18 @@
+/* tslint:disable */
+/* eslint-disable */
+/**
+ * OpenPipe API
+ * The public API for reporting API calls to OpenPipe
+ *
+ * The version of the OpenAPI document: 0.1.0
+ * 
+ *
+ * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+ * https://openapi-generator.tech
+ * Do not edit the class manually.
+ */
+
+
+export * from "./api";
+export * from "./configuration";
+

client-libs/typescript/index.ts

@@ -0,0 +1,3 @@
+// main.ts or index.ts at the root level
+export * as OpenAI from './openai';
+export * as OpenAILegacy from './openai-legacy';

client-libs/typescript/openai-legacy/index.ts

@@ -0,0 +1,81 @@
+import * as openPipeClient from "../codegen";
+import * as openai from "openai-legacy";
+import { version } from "../package.json";
+
+// Anything we don't override we want to pass through to openai directly
+export * as openAILegacy from "openai-legacy";
+
+type OPConfigurationParameters = {
+  apiKey?: string;
+  basePath?: string;
+};
+
+export class Configuration extends openai.Configuration {
+  public qkConfig?: openPipeClient.Configuration;
+
+  constructor(config: openai.ConfigurationParameters & { opParameters?: OPConfigurationParameters }) {
+    super(config);
+    if (config.opParameters) {
+      this.qkConfig = new openPipeClient.Configuration(config.opParameters);
+    }
+  }
+}
+
+type CreateChatCompletion = InstanceType<typeof openai.OpenAIApi>["createChatCompletion"];
+
+export class OpenAIApi extends openai.OpenAIApi {
+  public openPipeApi?: openPipeClient.DefaultApi;
+
+  constructor(config: Configuration) {
+    super(config);
+    if (config.qkConfig) {
+      this.openPipeApi = new openPipeClient.DefaultApi(config.qkConfig);
+    }
+  }
+
+  public async createChatCompletion(
+    createChatCompletionRequest: Parameters<CreateChatCompletion>[0],
+    options?: Parameters<CreateChatCompletion>[1]
+  ): ReturnType<CreateChatCompletion> {
+    const startTime = Date.now();
+    let resp: Awaited<ReturnType<CreateChatCompletion>> | null = null;
+    let respPayload: openai.CreateChatCompletionResponse | null = null;
+    let respStatus: number | undefined = undefined;
+    let error: string | undefined;
+    try {
+      resp = await super.createChatCompletion(createChatCompletionRequest, options);
+      respPayload = resp.data;
+      respStatus = resp.status;
+    } catch (err) {
+      console.error("Error in createChatCompletion");
+      if ("isAxiosError" in err && err.isAxiosError) {
+        error = err.response?.data?.error?.message;
+        respPayload = err.response?.data;
+        respStatus = err.response?.status;
+      } else if ("message" in err) {
+        error = err.message.toString();
+      }
+      throw err;
+    } finally {
+      this.openPipeApi
+        ?.externalApiReport({
+          startTime,
+          endTime: Date.now(),
+          reqPayload: createChatCompletionRequest,
+          respPayload: respPayload,
+          respStatus: respStatus,
+          error,
+          tags: {
+            client: "openai-js",
+            clientVersion: version,
+          },
+        })
+        .catch((err) => {
+          console.error("Error reporting to QK", err);
+        });
+    }
+
+    console.log("done");
+    return resp;
+  }
+}

client-libs/typescript/openai/index.ts

@@ -0,0 +1,111 @@
+import * as openai from "openai-beta";
+import { readEnv, type RequestOptions } from "openai-beta/core";
+import { CompletionCreateParams } from "openai-beta/resources/chat/completions";
+import axios from "axios";
+
+export * as openai from "openai-beta";
+import * as openPipeClient from "../codegen";
+
+interface ClientOptions extends openai.ClientOptions {
+  openPipeApiKey?: string;
+  openPipeBaseUrl?: string;
+}
+
+export class OpenAI extends openai.OpenAI {
+  public openPipeApi?: openPipeClient.DefaultApi;
+
+  constructor({
+    openPipeApiKey = readEnv("OPENPIPE_API_KEY"),
+    openPipeBaseUrl = readEnv("OPENPIPE_BASE_URL") ??
+      `https://app.openpipe.ai/v1`,
+    ...opts
+  }: ClientOptions = {}) {
+    super({ ...opts });
+
+    if (openPipeApiKey) {
+      const axiosInstance = axios.create({
+        baseURL: openPipeBaseUrl,
+        headers: {
+          'x-openpipe-api-key': openPipeApiKey,
+        },
+      });
+      this.openPipeApi = new openPipeClient.DefaultApi(
+        new openPipeClient.Configuration({
+          apiKey: openPipeApiKey,
+          basePath: openPipeBaseUrl,
+        }),
+        undefined,
+        axiosInstance
+      );
+    }
+
+    // Override the chat property
+    this.chat = new ExtendedChat(this);
+
+    if (openPipeApiKey === undefined) {
+      console.error(
+        "The OPENPIPE_API_KEY environment variable is missing or empty; either provide it, or instantiate the OpenPipe client with an openPipeApiKey option, like new OpenPipe({ openPipeApiKey: undefined })."
+      );
+    }
+  }
+}
+
+class ExtendedChat extends openai.OpenAI.Chat {
+  completions: ExtendedCompletions;
+
+  constructor(openaiInstance: OpenAI) {
+    super(openaiInstance);
+    // Initialize the new completions instance
+    this.completions = new ExtendedCompletions(openaiInstance);
+  }
+}
+
+class ExtendedCompletions extends openai.OpenAI.Chat.Completions {
+  private openaiInstance: OpenAI;
+
+  constructor(openaiInstance: OpenAI) {
+    super(openaiInstance);
+    this.openaiInstance = openaiInstance;
+  }
+
+  async create(
+    params:
+      | CompletionCreateParams.CreateChatCompletionRequestNonStreaming
+      | CompletionCreateParams.CreateChatCompletionRequestStreaming,
+    options?: RequestOptions,
+    tags?: Record<string, string>
+  ): Promise<any> {
+    // Your pre API call logic here
+    console.log("Doing pre API call...");
+
+    // Determine the type of request
+    if (params.hasOwnProperty("stream") && params.stream === true) {
+      const result = await super.create(
+        params as CompletionCreateParams.CreateChatCompletionRequestStreaming,
+        options
+      );
+      // Your post API call logic here
+      console.log("Doing post API call for Streaming...");
+      return result;
+    } else {
+      const startTime = Date.now();
+      const result = await super.create(
+        params as CompletionCreateParams.CreateChatCompletionRequestNonStreaming,
+        options
+      );
+      await this.openaiInstance.openPipeApi?.externalApiReport({
+        startTime,
+        endTime: Date.now(),
+        reqPayload: params,
+        respPayload: result,
+        respStatus: 200,
+        error: undefined,
+        tags,
+      });
+
+      // Your post API call logic here
+      console.log("Doing post API call for NonStreaming...");
+      return result;
+    }
+  }
+}

client-libs/typescript/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "openpipe",
+  "version": "0.1.0",
+  "description": "Metrics and auto-evaluation for LLM calls",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "axios": "^0.26.0",
+    "openai-beta": "npm:openai@4.0.0-beta.7",
+    "openai-legacy": "npm:openai@3.3.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.4.8",
+    "dotenv": "^16.3.1",
+    "tsx": "^3.12.7",
+    "typescript": "^5.0.4"
+  }
+}

client-libs/typescript/pnpm-lock.yaml

@@ -0,0 +1,548 @@
+lockfileVersion: '6.0'
+
+settings:
+  autoInstallPeers: true
+  excludeLinksFromLockfile: false
+
+dependencies:
+  axios:
+    specifier: ^0.26.0
+    version: 0.26.0
+  openai-beta:
+    specifier: npm:openai@4.0.0-beta.7
+    version: /openai@4.0.0-beta.7
+  openai-legacy:
+    specifier: npm:openai@3.3.0
+    version: /openai@3.3.0
+
+devDependencies:
+  '@types/node':
+    specifier: ^20.4.8
+    version: 20.4.8
+  dotenv:
+    specifier: ^16.3.1
+    version: 16.3.1
+  tsx:
+    specifier: ^3.12.7
+    version: 3.12.7
+  typescript:
+    specifier: ^5.0.4
+    version: 5.0.4
+
+packages:
+
+  /@esbuild-kit/cjs-loader@2.4.2:
+    resolution: {integrity: sha512-BDXFbYOJzT/NBEtp71cvsrGPwGAMGRB/349rwKuoxNSiKjPraNNnlK6MIIabViCjqZugu6j+xeMDlEkWdHHJSg==}
+    dependencies:
+      '@esbuild-kit/core-utils': 3.1.0
+      get-tsconfig: 4.6.2
+    dev: true
+
+  /@esbuild-kit/core-utils@3.1.0:
+    resolution: {integrity: sha512-Uuk8RpCg/7fdHSceR1M6XbSZFSuMrxcePFuGgyvsBn+u339dk5OeL4jv2EojwTN2st/unJGsVm4qHWjWNmJ/tw==}
+    dependencies:
+      esbuild: 0.17.19
+      source-map-support: 0.5.21
+    dev: true
+
+  /@esbuild-kit/esm-loader@2.5.5:
+    resolution: {integrity: sha512-Qwfvj/qoPbClxCRNuac1Du01r9gvNOT+pMYtJDapfB1eoGN1YlJ1BixLyL9WVENRx5RXgNLdfYdx/CuswlGhMw==}
+    dependencies:
+      '@esbuild-kit/core-utils': 3.1.0
+      get-tsconfig: 4.6.2
+    dev: true
+
+  /@esbuild/android-arm64@0.17.19:
+    resolution: {integrity: sha512-KBMWvEZooR7+kzY0BtbTQn0OAYY7CsiydT63pVEaPtVYF0hXbUaOyZog37DKxK7NF3XacBJOpYT4adIJh+avxA==}
+    engines: {node: '>=12'}
+    cpu: [arm64]
+    os: [android]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/android-arm@0.17.19:
+    resolution: {integrity: sha512-rIKddzqhmav7MSmoFCmDIb6e2W57geRsM94gV2l38fzhXMwq7hZoClug9USI2pFRGL06f4IOPHHpFNOkWieR8A==}
+    engines: {node: '>=12'}
+    cpu: [arm]
+    os: [android]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/android-x64@0.17.19:
+    resolution: {integrity: sha512-uUTTc4xGNDT7YSArp/zbtmbhO0uEEK9/ETW29Wk1thYUJBz3IVnvgEiEwEa9IeLyvnpKrWK64Utw2bgUmDveww==}
+    engines: {node: '>=12'}
+    cpu: [x64]
+    os: [android]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/darwin-arm64@0.17.19:
+    resolution: {integrity: sha512-80wEoCfF/hFKM6WE1FyBHc9SfUblloAWx6FJkFWTWiCoht9Mc0ARGEM47e67W9rI09YoUxJL68WHfDRYEAvOhg==}
+    engines: {node: '>=12'}
+    cpu: [arm64]
+    os: [darwin]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/darwin-x64@0.17.19:
+    resolution: {integrity: sha512-IJM4JJsLhRYr9xdtLytPLSH9k/oxR3boaUIYiHkAawtwNOXKE8KoU8tMvryogdcT8AU+Bflmh81Xn6Q0vTZbQw==}
+    engines: {node: '>=12'}
+    cpu: [x64]
+    os: [darwin]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/freebsd-arm64@0.17.19:
+    resolution: {integrity: sha512-pBwbc7DufluUeGdjSU5Si+P3SoMF5DQ/F/UmTSb8HXO80ZEAJmrykPyzo1IfNbAoaqw48YRpv8shwd1NoI0jcQ==}
+    engines: {node: '>=12'}
+    cpu: [arm64]
+    os: [freebsd]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/freebsd-x64@0.17.19:
+    resolution: {integrity: sha512-4lu+n8Wk0XlajEhbEffdy2xy53dpR06SlzvhGByyg36qJw6Kpfk7cp45DR/62aPH9mtJRmIyrXAS5UWBrJT6TQ==}
+    engines: {node: '>=12'}
+    cpu: [x64]
+    os: [freebsd]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/linux-arm64@0.17.19:
+    resolution: {integrity: sha512-ct1Tg3WGwd3P+oZYqic+YZF4snNl2bsnMKRkb3ozHmnM0dGWuxcPTTntAF6bOP0Sp4x0PjSF+4uHQ1xvxfRKqg==}
+    engines: {node: '>=12'}
+    cpu: [arm64]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/linux-arm@0.17.19:
+    resolution: {integrity: sha512-cdmT3KxjlOQ/gZ2cjfrQOtmhG4HJs6hhvm3mWSRDPtZ/lP5oe8FWceS10JaSJC13GBd4eH/haHnqf7hhGNLerA==}
+    engines: {node: '>=12'}
+    cpu: [arm]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/linux-ia32@0.17.19:
+    resolution: {integrity: sha512-w4IRhSy1VbsNxHRQpeGCHEmibqdTUx61Vc38APcsRbuVgK0OPEnQ0YD39Brymn96mOx48Y2laBQGqgZ0j9w6SQ==}
+    engines: {node: '>=12'}
+    cpu: [ia32]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/linux-loong64@0.17.19:
+    resolution: {integrity: sha512-2iAngUbBPMq439a+z//gE+9WBldoMp1s5GWsUSgqHLzLJ9WoZLZhpwWuym0u0u/4XmZ3gpHmzV84PonE+9IIdQ==}
+    engines: {node: '>=12'}
+    cpu: [loong64]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/linux-mips64el@0.17.19:
+    resolution: {integrity: sha512-LKJltc4LVdMKHsrFe4MGNPp0hqDFA1Wpt3jE1gEyM3nKUvOiO//9PheZZHfYRfYl6AwdTH4aTcXSqBerX0ml4A==}
+    engines: {node: '>=12'}
+    cpu: [mips64el]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/linux-ppc64@0.17.19:
+    resolution: {integrity: sha512-/c/DGybs95WXNS8y3Ti/ytqETiW7EU44MEKuCAcpPto3YjQbyK3IQVKfF6nbghD7EcLUGl0NbiL5Rt5DMhn5tg==}
+    engines: {node: '>=12'}
+    cpu: [ppc64]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/linux-riscv64@0.17.19:
+    resolution: {integrity: sha512-FC3nUAWhvFoutlhAkgHf8f5HwFWUL6bYdvLc/TTuxKlvLi3+pPzdZiFKSWz/PF30TB1K19SuCxDTI5KcqASJqA==}
+    engines: {node: '>=12'}
+    cpu: [riscv64]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/linux-s390x@0.17.19:
+    resolution: {integrity: sha512-IbFsFbxMWLuKEbH+7sTkKzL6NJmG2vRyy6K7JJo55w+8xDk7RElYn6xvXtDW8HCfoKBFK69f3pgBJSUSQPr+4Q==}
+    engines: {node: '>=12'}
+    cpu: [s390x]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/linux-x64@0.17.19:
+    resolution: {integrity: sha512-68ngA9lg2H6zkZcyp22tsVt38mlhWde8l3eJLWkyLrp4HwMUr3c1s/M2t7+kHIhvMjglIBrFpncX1SzMckomGw==}
+    engines: {node: '>=12'}
+    cpu: [x64]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/netbsd-x64@0.17.19:
+    resolution: {integrity: sha512-CwFq42rXCR8TYIjIfpXCbRX0rp1jo6cPIUPSaWwzbVI4aOfX96OXY8M6KNmtPcg7QjYeDmN+DD0Wp3LaBOLf4Q==}
+    engines: {node: '>=12'}
+    cpu: [x64]
+    os: [netbsd]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/openbsd-x64@0.17.19:
+    resolution: {integrity: sha512-cnq5brJYrSZ2CF6c35eCmviIN3k3RczmHz8eYaVlNasVqsNY+JKohZU5MKmaOI+KkllCdzOKKdPs762VCPC20g==}
+    engines: {node: '>=12'}
+    cpu: [x64]
+    os: [openbsd]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/sunos-x64@0.17.19:
+    resolution: {integrity: sha512-vCRT7yP3zX+bKWFeP/zdS6SqdWB8OIpaRq/mbXQxTGHnIxspRtigpkUcDMlSCOejlHowLqII7K2JKevwyRP2rg==}
+    engines: {node: '>=12'}
+    cpu: [x64]
+    os: [sunos]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/win32-arm64@0.17.19:
+    resolution: {integrity: sha512-yYx+8jwowUstVdorcMdNlzklLYhPxjniHWFKgRqH7IFlUEa0Umu3KuYplf1HUZZ422e3NU9F4LGb+4O0Kdcaag==}
+    engines: {node: '>=12'}
+    cpu: [arm64]
+    os: [win32]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/win32-ia32@0.17.19:
+    resolution: {integrity: sha512-eggDKanJszUtCdlVs0RB+h35wNlb5v4TWEkq4vZcmVt5u/HiDZrTXe2bWFQUez3RgNHwx/x4sk5++4NSSicKkw==}
+    engines: {node: '>=12'}
+    cpu: [ia32]
+    os: [win32]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/win32-x64@0.17.19:
+    resolution: {integrity: sha512-lAhycmKnVOuRYNtRtatQR1LPQf2oYCkRGkSFnseDAKPl8lu5SOsK/e1sXe5a0Pc5kHIHe6P2I/ilntNv2xf3cA==}
+    engines: {node: '>=12'}
+    cpu: [x64]
+    os: [win32]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@types/node-fetch@2.6.4:
+    resolution: {integrity: sha512-1ZX9fcN4Rvkvgv4E6PAY5WXUFWFcRWxZa3EW83UjycOB9ljJCedb2CupIP4RZMEwF/M3eTcCihbBRgwtGbg5Rg==}
+    dependencies:
+      '@types/node': 20.4.8
+      form-data: 3.0.1
+    dev: false
+
+  /@types/node@18.17.3:
+    resolution: {integrity: sha512-2x8HWtFk0S99zqVQABU9wTpr8wPoaDHZUcAkoTKH+nL7kPv3WUI9cRi/Kk5Mz4xdqXSqTkKP7IWNoQQYCnDsTA==}
+    dev: false
+
+  /@types/node@20.4.8:
+    resolution: {integrity: sha512-0mHckf6D2DiIAzh8fM8f3HQCvMKDpK94YQ0DSVkfWTG9BZleYIWudw9cJxX8oCk9bM+vAkDyujDV6dmKHbvQpg==}
+
+  /abort-controller@3.0.0:
+    resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==}
+    engines: {node: '>=6.5'}
+    dependencies:
+      event-target-shim: 5.0.1
+    dev: false
+
+  /agentkeepalive@4.5.0:
+    resolution: {integrity: sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==}
+    engines: {node: '>= 8.0.0'}
+    dependencies:
+      humanize-ms: 1.2.1
+    dev: false
+
+  /asynckit@0.4.0:
+    resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==}
+    dev: false
+
+  /axios@0.26.0:
+    resolution: {integrity: sha512-lKoGLMYtHvFrPVt3r+RBMp9nh34N0M8zEfCWqdWZx6phynIEhQqAdydpyBAAG211zlhX9Rgu08cOamy6XjE5Og==}
+    dependencies:
+      follow-redirects: 1.15.2
+    transitivePeerDependencies:
+      - debug
+    dev: false
+
+  /base-64@0.1.0:
+    resolution: {integrity: sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==}
+    dev: false
+
+  /buffer-from@1.1.2:
+    resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==}
+    dev: true
+
+  /charenc@0.0.2:
+    resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==}
+    dev: false
+
+  /combined-stream@1.0.8:
+    resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==}
+    engines: {node: '>= 0.8'}
+    dependencies:
+      delayed-stream: 1.0.0
+    dev: false
+
+  /crypt@0.0.2:
+    resolution: {integrity: sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==}
+    dev: false
+
+  /delayed-stream@1.0.0:
+    resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
+    engines: {node: '>=0.4.0'}
+    dev: false
+
+  /digest-fetch@1.3.0:
+    resolution: {integrity: sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==}
+    dependencies:
+      base-64: 0.1.0
+      md5: 2.3.0
+    dev: false
+
+  /dotenv@16.3.1:
+    resolution: {integrity: sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==}
+    engines: {node: '>=12'}
+    dev: true
+
+  /esbuild@0.17.19:
+    resolution: {integrity: sha512-XQ0jAPFkK/u3LcVRcvVHQcTIqD6E2H1fvZMA5dQPSOWb3suUbWbfbRf94pjc0bNzRYLfIrDRQXr7X+LHIm5oHw==}
+    engines: {node: '>=12'}
+    hasBin: true
+    requiresBuild: true
+    optionalDependencies:
+      '@esbuild/android-arm': 0.17.19
+      '@esbuild/android-arm64': 0.17.19
+      '@esbuild/android-x64': 0.17.19
+      '@esbuild/darwin-arm64': 0.17.19
+      '@esbuild/darwin-x64': 0.17.19
+      '@esbuild/freebsd-arm64': 0.17.19
+      '@esbuild/freebsd-x64': 0.17.19
+      '@esbuild/linux-arm': 0.17.19
+      '@esbuild/linux-arm64': 0.17.19
+      '@esbuild/linux-ia32': 0.17.19
+      '@esbuild/linux-loong64': 0.17.19
+      '@esbuild/linux-mips64el': 0.17.19
+      '@esbuild/linux-ppc64': 0.17.19
+      '@esbuild/linux-riscv64': 0.17.19
+      '@esbuild/linux-s390x': 0.17.19
+      '@esbuild/linux-x64': 0.17.19
+      '@esbuild/netbsd-x64': 0.17.19
+      '@esbuild/openbsd-x64': 0.17.19
+      '@esbuild/sunos-x64': 0.17.19
+      '@esbuild/win32-arm64': 0.17.19
+      '@esbuild/win32-ia32': 0.17.19
+      '@esbuild/win32-x64': 0.17.19
+    dev: true
+
+  /event-target-shim@5.0.1:
+    resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==}
+    engines: {node: '>=6'}
+    dev: false
+
+  /follow-redirects@1.15.2:
+    resolution: {integrity: sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==}
+    engines: {node: '>=4.0'}
+    peerDependencies:
+      debug: '*'
+    peerDependenciesMeta:
+      debug:
+        optional: true
+    dev: false
+
+  /form-data-encoder@1.7.2:
+    resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==}
+    dev: false
+
+  /form-data@3.0.1:
+    resolution: {integrity: sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==}
+    engines: {node: '>= 6'}
+    dependencies:
+      asynckit: 0.4.0
+      combined-stream: 1.0.8
+      mime-types: 2.1.35
+    dev: false
+
+  /form-data@4.0.0:
+    resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==}
+    engines: {node: '>= 6'}
+    dependencies:
+      asynckit: 0.4.0
+      combined-stream: 1.0.8
+      mime-types: 2.1.35
+    dev: false
+
+  /formdata-node@4.4.1:
+    resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==}
+    engines: {node: '>= 12.20'}
+    dependencies:
+      node-domexception: 1.0.0
+      web-streams-polyfill: 4.0.0-beta.3
+    dev: false
+
+  /fsevents@2.3.2:
+    resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==}
+    engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
+    os: [darwin]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /get-tsconfig@4.6.2:
+    resolution: {integrity: sha512-E5XrT4CbbXcXWy+1jChlZmrmCwd5KGx502kDCXJJ7y898TtWW9FwoG5HfOLVRKmlmDGkWN2HM9Ho+/Y8F0sJDg==}
+    dependencies:
+      resolve-pkg-maps: 1.0.0
+    dev: true
+
+  /humanize-ms@1.2.1:
+    resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==}
+    dependencies:
+      ms: 2.1.3
+    dev: false
+
+  /is-buffer@1.1.6:
+    resolution: {integrity: sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==}
+    dev: false
+
+  /md5@2.3.0:
+    resolution: {integrity: sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==}
+    dependencies:
+      charenc: 0.0.2
+      crypt: 0.0.2
+      is-buffer: 1.1.6
+    dev: false
+
+  /mime-db@1.52.0:
+    resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==}
+    engines: {node: '>= 0.6'}
+    dev: false
+
+  /mime-types@2.1.35:
+    resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==}
+    engines: {node: '>= 0.6'}
+    dependencies:
+      mime-db: 1.52.0
+    dev: false
+
+  /ms@2.1.3:
+    resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
+    dev: false
+
+  /node-domexception@1.0.0:
+    resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==}
+    engines: {node: '>=10.5.0'}
+    dev: false
+
+  /node-fetch@2.6.12:
+    resolution: {integrity: sha512-C/fGU2E8ToujUivIO0H+tpQ6HWo4eEmchoPIoXtxCrVghxdKq+QOHqEZW7tuP3KlV3bC8FRMO5nMCC7Zm1VP6g==}
+    engines: {node: 4.x || >=6.0.0}
+    peerDependencies:
+      encoding: ^0.1.0
+    peerDependenciesMeta:
+      encoding:
+        optional: true
+    dependencies:
+      whatwg-url: 5.0.0
+    dev: false
+
+  /openai@3.3.0:
+    resolution: {integrity: sha512-uqxI/Au+aPRnsaQRe8CojU0eCR7I0mBiKjD3sNMzY6DaC1ZVrc85u98mtJW6voDug8fgGN+DIZmTDxTthxb7dQ==}
+    dependencies:
+      axios: 0.26.0
+      form-data: 4.0.0
+    transitivePeerDependencies:
+      - debug
+    dev: false
+
+  /openai@4.0.0-beta.7:
+    resolution: {integrity: sha512-jHjwvpMuGkNxiQ3erwLZsOvPEhcVrMtwtfNeYmGCjhbdB+oStVw/7pIhIPkualu8rlhLwgMR7awknIaN3IQcOA==}
+    dependencies:
+      '@types/node': 18.17.3
+      '@types/node-fetch': 2.6.4
+      abort-controller: 3.0.0
+      agentkeepalive: 4.5.0
+      digest-fetch: 1.3.0
+      form-data-encoder: 1.7.2
+      formdata-node: 4.4.1
+      node-fetch: 2.6.12
+    transitivePeerDependencies:
+      - encoding
+    dev: false
+
+  /resolve-pkg-maps@1.0.0:
+    resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==}
+    dev: true
+
+  /source-map-support@0.5.21:
+    resolution: {integrity: sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==}
+    dependencies:
+      buffer-from: 1.1.2
+      source-map: 0.6.1
+    dev: true
+
+  /source-map@0.6.1:
+    resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==}
+    engines: {node: '>=0.10.0'}
+    dev: true
+
+  /tr46@0.0.3:
+    resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==}
+    dev: false
+
+  /tsx@3.12.7:
+    resolution: {integrity: sha512-C2Ip+jPmqKd1GWVQDvz/Eyc6QJbGfE7NrR3fx5BpEHMZsEHoIxHL1j+lKdGobr8ovEyqeNkPLSKp6SCSOt7gmw==}
+    hasBin: true
+    dependencies:
+      '@esbuild-kit/cjs-loader': 2.4.2
+      '@esbuild-kit/core-utils': 3.1.0
+      '@esbuild-kit/esm-loader': 2.5.5
+    optionalDependencies:
+      fsevents: 2.3.2
+    dev: true
+
+  /typescript@5.0.4:
+    resolution: {integrity: sha512-cW9T5W9xY37cc+jfEnaUvX91foxtHkza3Nw3wkoF4sSlKn0MONdkdEndig/qPBWXNkmplh3NzayQzCiHM4/hqw==}
+    engines: {node: '>=12.20'}
+    hasBin: true
+    dev: true
+
+  /web-streams-polyfill@4.0.0-beta.3:
+    resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==}
+    engines: {node: '>= 14'}
+    dev: false
+
+  /webidl-conversions@3.0.1:
+    resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==}
+    dev: false
+
+  /whatwg-url@5.0.0:
+    resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==}
+    dependencies:
+      tr46: 0.0.3
+      webidl-conversions: 3.0.1
+    dev: false

client-libs/typescript/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "declaration": true,
+    "target": "es6",
+    "module": "commonjs",
+    "noImplicitAny": true,
+    "resolveJsonModule": true,
+    "outDir": "dist",
+    "rootDir": ".",
+    "skipLibCheck": true
+  },
+  "exclude": ["dist", "node_modules"],
+}