Don't define CellWrapper inline

This way we don't re-render the entire cell every time a variable changes. Better performance and handles modals correctly.

OutputCell is still a pretty messy component, which we'll have to address at some point, but the complexity is still manageable for now.

.prettierignore

@@ -0,0 +1,2 @@
+*.schema.json
+app/pnpm-lock.yaml

app/.env.example

@@ -32,5 +32,11 @@ NEXT_PUBLIC_HOST="http://localhost:3000"
 GITHUB_CLIENT_ID="your_client_id"
 GITHUB_CLIENT_SECRET="your_secret"
 
-OPENPIPE_BASE_URL="http://localhost:3000/api"
+OPENPIPE_BASE_URL="http://localhost:3000/api/v1"
 OPENPIPE_API_KEY="your_key"
+
+SENDER_EMAIL="placeholder"
+SMTP_HOST="placeholder"
+SMTP_PORT="placeholder"
+SMTP_LOGIN="placeholder"
+SMTP_PASSWORD="placeholder"

app/.prettierignore

@@ -1,2 +0,0 @@
-*.schema.json
-pnpm-lock.yaml

app/@types/nextjs-routes.d.ts

@@ -12,18 +12,18 @@ declare module "nextjs-routes" {
 
   export type Route =
     | StaticRoute<"/account/signin">
-    | DynamicRoute<"/api/[...trpc]", { "trpc": string[] }>
     | DynamicRoute<"/api/auth/[...nextauth]", { "nextauth": string[] }>
     | StaticRoute<"/api/experiments/og-image">
-    | StaticRoute<"/api/openapi">
-    | StaticRoute<"/api/sentry-example-api">
     | DynamicRoute<"/api/trpc/[trpc]", { "trpc": string }>
+    | DynamicRoute<"/api/v1/[...trpc]", { "trpc": string[] }>
+    | StaticRoute<"/api/v1/openapi">
     | StaticRoute<"/dashboard">
     | DynamicRoute<"/data/[id]", { "id": string }>
     | StaticRoute<"/data">
     | DynamicRoute<"/experiments/[id]", { "id": string }>
     | StaticRoute<"/experiments">
     | StaticRoute<"/">
+    | DynamicRoute<"/invitations/[invitationToken]", { "invitationToken": string }>
     | StaticRoute<"/project/settings">
     | StaticRoute<"/request-logs">
     | StaticRoute<"/sentry-example-page">

app/package.json

@@ -10,13 +10,13 @@
   },
   "scripts": {
     "build": "next build",
-    "dev:next": "next dev",
+    "dev:next": "TZ=UTC next dev",
     "dev:wss": "pnpm tsx --watch src/wss-server.ts",
     "dev:worker": "NODE_ENV='development' pnpm tsx --watch src/server/tasks/worker.ts",
     "dev": "concurrently --kill-others 'pnpm dev:next' 'pnpm dev:wss' 'pnpm dev:worker'",
     "postinstall": "prisma generate",
     "lint": "next lint",
-    "start": "next start",
+    "start": "TZ=UTC next start",
     "codegen:clients": "tsx src/server/scripts/client-codegen.ts",
     "seed": "tsx prisma/seed.ts",
     "check": "concurrently 'pnpm lint' 'pnpm tsc' 'pnpm prettier . --check'",
@@ -37,6 +37,7 @@
     "@monaco-editor/loader": "^1.3.3",
     "@next-auth/prisma-adapter": "^1.0.5",
     "@prisma/client": "^4.14.0",
+    "@sendinblue/client": "^3.3.1",
     "@sentry/nextjs": "^7.61.0",
     "@t3-oss/env-nextjs": "^0.3.1",
     "@tabler/icons-react": "^2.22.0",
@@ -66,12 +67,15 @@
     "kysely": "^0.26.1",
     "lodash-es": "^4.17.21",
     "lucide-react": "^0.265.0",
+    "marked": "^7.0.3",
     "next": "^13.4.2",
     "next-auth": "^4.22.1",
     "next-query-params": "^4.2.3",
     "nextjs-cors": "^2.1.2",
     "nextjs-routes": "^2.0.1",
+    "nodemailer": "^6.9.4",
     "openai": "4.0.0-beta.7",
+    "openpipe": "workspace:*",
     "pg": "^8.11.2",
     "pluralize": "^8.0.0",
     "posthog-js": "^1.75.3",
@@ -100,8 +104,7 @@
     "uuid": "^9.0.0",
     "vite-tsconfig-paths": "^4.2.0",
     "zod": "^3.21.4",
-    "zustand": "^4.3.9",
-    "openpipe": "workspace:*"
+    "zustand": "^4.3.9"
   },
   "devDependencies": {
     "@openapi-contrib/openapi-schema-to-json-schema": "^4.0.5",
@@ -114,6 +117,7 @@
     "@types/json-schema": "^7.0.12",
     "@types/lodash-es": "^4.17.8",
     "@types/node": "^18.16.0",
+    "@types/nodemailer": "^6.4.9",
     "@types/pg": "^8.10.2",
     "@types/pluralize": "^0.0.30",
     "@types/prismjs": "^1.26.0",
@@ -129,6 +133,7 @@
     "eslint-plugin-unused-imports": "^2.0.0",
     "monaco-editor": "^0.40.0",
     "openapi-typescript": "^6.3.4",
+    "openapi-typescript-codegen": "^0.25.0",
     "prisma": "^4.14.0",
     "raw-loader": "^4.0.2",
     "typescript": "^5.0.4",

app/prisma/migrations/20230814233722_index_tags_by_project/migration.sql

@@ -0,0 +1,22 @@
+-- DropIndex
+DROP INDEX "LoggedCallTag_name_idx";
+DROP INDEX "LoggedCallTag_name_value_idx";
+
+-- AlterTable: Add projectId column without NOT NULL constraint for now
+ALTER TABLE "LoggedCallTag" ADD COLUMN "projectId" UUID;
+
+-- Set the default value
+UPDATE "LoggedCallTag" lct
+SET "projectId" = lc."projectId"
+FROM "LoggedCall" lc
+WHERE lct."loggedCallId" = lc.id;
+
+-- Now set the NOT NULL constraint
+ALTER TABLE "LoggedCallTag" ALTER COLUMN "projectId" SET NOT NULL;
+
+-- CreateIndex
+CREATE INDEX "LoggedCallTag_projectId_name_idx" ON "LoggedCallTag"("projectId", "name");
+CREATE INDEX "LoggedCallTag_projectId_name_value_idx" ON "LoggedCallTag"("projectId", "name", "value");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "LoggedCallTag_loggedCallId_name_key" ON "LoggedCallTag"("loggedCallId", "name");

app/prisma/migrations/20230815234811_add_user_invitations/migration.sql

@@ -0,0 +1,25 @@
+-- CreateTable
+CREATE TABLE "UserInvitation" (
+    "id" UUID NOT NULL,
+    "projectId" UUID NOT NULL,
+    "email" TEXT NOT NULL,
+    "role" "ProjectUserRole" NOT NULL,
+    "invitationToken" TEXT NOT NULL,
+    "senderId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "UserInvitation_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "UserInvitation_invitationToken_key" ON "UserInvitation"("invitationToken");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "UserInvitation_projectId_email_key" ON "UserInvitation"("projectId", "email");
+
+-- AddForeignKey
+ALTER TABLE "UserInvitation" ADD CONSTRAINT "UserInvitation_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "UserInvitation" ADD CONSTRAINT "UserInvitation_senderId_fkey" FOREIGN KEY ("senderId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE;

app/prisma/schema.prisma

@@ -112,17 +112,17 @@ model ScenarioVariantCell {
 model ModelResponse {
     id String @id @default(uuid()) @db.Uuid
 
-    cacheKey        String
-    requestedAt      DateTime?
-    receivedAt       DateTime?
-    respPayload      Json?
-    cost             Float?
-    inputTokens      Int?
-    outputTokens     Int?
-    statusCode       Int?
-    errorMessage     String?
-    retryTime        DateTime?
-    outdated         Boolean   @default(false)
+    cacheKey     String
+    requestedAt  DateTime?
+    receivedAt   DateTime?
+    respPayload  Json?
+    cost         Float?
+    inputTokens  Int?
+    outputTokens Int?
+    statusCode   Int?
+    errorMessage String?
+    retryTime    DateTime?
+    outdated     Boolean   @default(false)
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
@@ -207,13 +207,14 @@ model Project {
     personalProjectUserId String? @unique @db.Uuid
     personalProjectUser   User?   @relation(fields: [personalProjectUserId], references: [id], onDelete: Cascade)
 
-    createdAt    DateTime      @default(now())
-    updatedAt    DateTime      @updatedAt
-    projectUsers ProjectUser[]
-    experiments  Experiment[]
-    datasets     Dataset[]
-    loggedCalls  LoggedCall[]
-    apiKeys      ApiKey[]
+    createdAt               DateTime      @default(now())
+    updatedAt               DateTime      @updatedAt
+    projectUsers            ProjectUser[]
+    projectUserInvitations  UserInvitation[]
+    experiments             Experiment[]
+    datasets                Dataset[]
+    loggedCalls             LoggedCall[]
+    apiKeys                 ApiKey[]
 }
 
 enum ProjectUserRole {
@@ -273,8 +274,8 @@ model LoggedCall {
     projectId String   @db.Uuid
     project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
-    model        String?
-    tags         LoggedCallTag[]
+    model String?
+    tags  LoggedCallTag[]
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
@@ -295,7 +296,7 @@ model LoggedCallModelResponse {
     errorMessage String?
 
     requestedAt DateTime
-    receivedAt   DateTime
+    receivedAt  DateTime
 
     // Note: the function to calculate the cacheKey should include the project
     // ID so we don't share cached responses between projects, which could be an
@@ -326,12 +327,14 @@ model LoggedCallTag {
     id    String  @id @default(uuid()) @db.Uuid
     name  String
     value String?
+    projectId String   @db.Uuid
 
     loggedCallId String     @db.Uuid
     loggedCall   LoggedCall @relation(fields: [loggedCallId], references: [id], onDelete: Cascade)
 
-    @@index([name])
-    @@index([name, value])
+    @@unique([loggedCallId, name])
+    @@index([projectId, name])
+    @@index([projectId, name, value])
 }
 
 model ApiKey {
@@ -340,8 +343,8 @@ model ApiKey {
     name   String
     apiKey String @unique
 
-    projectId String   @db.Uuid
-    project   Project? @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    projectId String  @db.Uuid
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
 
     createdAt DateTime @default(now())
     updatedAt DateTime @updatedAt
@@ -388,16 +391,33 @@ model User {
 
     role UserRole @default(USER)
 
-    accounts          Account[]
-    sessions          Session[]
-    projectUsers      ProjectUser[]
-    projects          Project[]
-    worldChampEntrant WorldChampEntrant?
+    accounts                Account[]
+    sessions                Session[]
+    projectUsers            ProjectUser[]
+    projects                Project[]
+    worldChampEntrant       WorldChampEntrant?
+    sentUserInvitations     UserInvitation[]
 
     createdAt DateTime @default(now())
     updatedAt DateTime @default(now()) @updatedAt
 }
 
+model UserInvitation {
+    id String @id @default(uuid()) @db.Uuid
+
+    projectId String @db.Uuid
+    project   Project @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    email String
+    role ProjectUserRole
+    invitationToken String @unique
+    senderId String @db.Uuid
+    sender User @relation(fields: [senderId], references: [id], onDelete: Cascade)
+
+    @@unique([projectId, email])
+    createdAt DateTime @default(now())
+    updatedAt DateTime @updatedAt
+}
+
 model VerificationToken {
     identifier String
     token      String   @unique

app/prisma/seed.ts

@@ -2,6 +2,7 @@ import { prisma } from "~/server/db";
 import dedent from "dedent";
 import { generateNewCell } from "~/server/utils/generateNewCell";
 import { promptConstructorVersion } from "~/promptConstructor/version";
+import { env } from "~/env.mjs";
 
 const defaultId = "11111111-1111-1111-1111-111111111111";
 
@@ -16,6 +17,16 @@ const project =
     data: { id: defaultId },
   }));
 
+if (env.OPENPIPE_API_KEY) {
+  await prisma.apiKey.create({
+    data: {
+      projectId: project.id,
+      name: "Default API Key",
+      apiKey: env.OPENPIPE_API_KEY,
+    },
+  });
+}
+
 await prisma.experiment.deleteMany({
   where: {
     id: defaultId,

app/prisma/seedDashboard.ts

@@ -13,6 +13,7 @@ const MODEL_RESPONSE_TEMPLATES: {
   inputTokens: number;
   outputTokens: number;
   finishReason: string;
+  tags: { name: string; value: string }[];
 }[] = [
   {
     reqPayload: {
@@ -107,6 +108,7 @@ const MODEL_RESPONSE_TEMPLATES: {
     inputTokens: 236,
     outputTokens: 5,
     finishReason: "stop",
+    tags: [],
   },
   {
     reqPayload: {
@@ -193,6 +195,7 @@ const MODEL_RESPONSE_TEMPLATES: {
     inputTokens: 222,
     outputTokens: 5,
     finishReason: "stop",
+    tags: [],
   },
   {
     reqPayload: {
@@ -231,6 +234,7 @@ const MODEL_RESPONSE_TEMPLATES: {
     inputTokens: 14,
     outputTokens: 7,
     finishReason: "stop",
+    tags: [{ name: "prompt_id", value: "id2" }],
   },
   {
     reqPayload: {
@@ -306,6 +310,10 @@ const MODEL_RESPONSE_TEMPLATES: {
     inputTokens: 2802,
     outputTokens: 108,
     finishReason: "stop",
+    tags: [
+      { name: "prompt_id", value: "chatcmpl-7lQS3MktOT8BTgNEytl9dkyssCQqL" },
+      { name: "some_other_tag", value: "some_other_value" },
+    ],
   },
 ];
 
@@ -349,6 +357,7 @@ for (let i = 0; i < 1437; i++) {
     cacheHit: false,
     requestedAt,
     projectId: project.id,
+    model: template.reqPayload.model,
     createdAt: requestedAt,
   });
 
@@ -388,11 +397,14 @@ for (let i = 0; i < 1437; i++) {
       modelResponseId: loggedCallModelResponseId,
     },
   });
-  loggedCallTagsToCreate.push({
-    loggedCallId,
-    name: "$model",
-    value: template.reqPayload.model,
-  });
+  for (const tag of template.tags) {
+    loggedCallTagsToCreate.push({
+      projectId: project.id,
+      loggedCallId,
+      name: tag.name,
+      value: tag.value,
+    });
+  }
 }
 
 await prisma.$transaction([

app/src/components/AutoResizeTextArea.tsx

@@ -1,13 +1,13 @@
 import { Textarea, type TextareaProps } from "@chakra-ui/react";
 import ResizeTextarea from "react-textarea-autosize";
-import React, { useLayoutEffect, useState } from "react";
+import React, { useEffect, useState } from "react";
 
 export const AutoResizeTextarea: React.ForwardRefRenderFunction<
   HTMLTextAreaElement,
   TextareaProps & { minRows?: number }
 > = ({ minRows = 1, overflowY = "hidden", ...props }, ref) => {
   const [isRerendered, setIsRerendered] = useState(false);
-  useLayoutEffect(() => setIsRerendered(true), []);
+  useEffect(() => setIsRerendered(true), []);
 
   return (
     <Textarea

app/src/components/ChangeModelModal/ModelStatsCard.tsx

@@ -87,7 +87,7 @@ export const ModelStatsCard = ({
               label="Price"
               info={
                 <Text>
-                  ${model.pricePerSecond.toFixed(3)}
+                  ${model.pricePerSecond.toFixed(4)}
                   <Text color="gray.500"> / second</Text>
                 </Text>
               }

app/src/components/CopiableCode.tsx

@@ -1,15 +1,16 @@
-import { HStack, Icon, IconButton, Tooltip, Text } from "@chakra-ui/react";
+import { HStack, Icon, IconButton, Tooltip, Text, type StackProps } from "@chakra-ui/react";
 import { useState } from "react";
 import { MdContentCopy } from "react-icons/md";
 import { useHandledAsyncCallback } from "~/utils/hooks";
 
-const CopiableCode = ({ code }: { code: string }) => {
+const CopiableCode = ({ code, ...rest }: { code: string } & StackProps) => {
   const [copied, setCopied] = useState(false);
 
   const [copyToClipboard] = useHandledAsyncCallback(async () => {
     await navigator.clipboard.writeText(code);
     setCopied(true);
   }, [code]);
+
   return (
     <HStack
       backgroundColor="blackAlpha.800"
@@ -18,9 +19,19 @@ const CopiableCode = ({ code }: { code: string }) => {
       padding={3}
       w="full"
       justifyContent="space-between"
+      alignItems="flex-start"
+      {...rest}
     >
-      <Text fontFamily="inconsolata" fontWeight="bold" letterSpacing={0.5} overflowX="auto">
+      <Text
+        fontFamily="inconsolata"
+        fontWeight="bold"
+        letterSpacing={0.5}
+        overflowX="auto"
+        whiteSpace="pre-wrap"
+      >
         {code}
+        {/* Necessary for trailing newline to actually be displayed */}
+        {code.endsWith("\n") ? "\n" : ""}
       </Text>
       <Tooltip closeOnClick={false} label={copied ? "Copied!" : "Copy to clipboard"}>
         <IconButton

app/src/components/InputDropdown.tsx

@@ -0,0 +1,80 @@
+import {
+  Input,
+  InputGroup,
+  InputRightElement,
+  Icon,
+  Popover,
+  PopoverTrigger,
+  PopoverContent,
+  VStack,
+  HStack,
+  Button,
+  Text,
+  useDisclosure,
+} from "@chakra-ui/react";
+
+import { FiChevronDown } from "react-icons/fi";
+import { BiCheck } from "react-icons/bi";
+
+type InputDropdownProps<T> = {
+  options: ReadonlyArray<T>;
+  selectedOption: T;
+  onSelect: (option: T) => void;
+};
+
+const InputDropdown = <T,>({ options, selectedOption, onSelect }: InputDropdownProps<T>) => {
+  const popover = useDisclosure();
+
+  return (
+    <Popover placement="bottom-start" {...popover}>
+      <PopoverTrigger>
+        <InputGroup cursor="pointer" w={(selectedOption as string).length * 14 + 180}>
+          <Input
+            value={selectedOption as string}
+            // eslint-disable-next-line @typescript-eslint/no-empty-function -- controlled input requires onChange
+            onChange={() => {}}
+            cursor="pointer"
+            borderColor={popover.isOpen ? "blue.500" : undefined}
+            _hover={popover.isOpen ? { borderColor: "blue.500" } : undefined}
+            contentEditable={false}
+            // disable focus
+            onFocus={(e) => {
+              e.target.blur();
+            }}
+          />
+          <InputRightElement>
+            <Icon as={FiChevronDown} />
+          </InputRightElement>
+        </InputGroup>
+      </PopoverTrigger>
+      <PopoverContent boxShadow="0 0 40px 4px rgba(0, 0, 0, 0.1);" minW={0} w="auto">
+        <VStack spacing={0}>
+          {options?.map((option, index) => (
+            <HStack
+              key={index}
+              as={Button}
+              onClick={() => {
+                onSelect(option);
+                popover.onClose();
+              }}
+              w="full"
+              variant="ghost"
+              justifyContent="space-between"
+              fontWeight="semibold"
+              borderRadius={0}
+              colorScheme="blue"
+              color="black"
+              fontSize="sm"
+              borderBottomWidth={1}
+            >
+              <Text mr={16}>{option as string}</Text>
+              {option === selectedOption && <Icon as={BiCheck} color="blue.500" boxSize={5} />}
+            </HStack>
+          ))}
+        </VStack>
+      </PopoverContent>
+    </Popover>
+  );
+};
+
+export default InputDropdown;

app/src/components/OutputsTable/OutputCell/CellOptions.tsx

@@ -1,7 +1,7 @@
 import { HStack, Icon, IconButton, Spinner, Tooltip, useDisclosure } from "@chakra-ui/react";
 import { BsArrowClockwise, BsInfoCircle } from "react-icons/bs";
 import { useExperimentAccess } from "~/utils/hooks";
-import ExpandedModal from "./PromptModal";
+import PromptModal from "./PromptModal";
 import { type RouterOutputs } from "~/utils/api";
 
 export const CellOptions = ({
@@ -32,7 +32,7 @@ export const CellOptions = ({
               variant="ghost"
             />
           </Tooltip>
-          <ExpandedModal cell={cell} disclosure={modalDisclosure} />
+          <PromptModal cell={cell} disclosure={modalDisclosure} />
         </>
       )}
       {canModify && (

app/src/components/OutputsTable/OutputCell/CellWrapper.tsx

@@ -0,0 +1,29 @@
+import { type StackProps, VStack } from "@chakra-ui/react";
+import { type RouterOutputs } from "~/utils/api";
+import { type Scenario } from "../types";
+import { CellOptions } from "./CellOptions";
+import { OutputStats } from "./OutputStats";
+
+const CellWrapper: React.FC<
+  StackProps & {
+    cell: RouterOutputs["scenarioVariantCells"]["get"] | undefined;
+    hardRefetching: boolean;
+    hardRefetch: () => void;
+    mostRecentResponse:
+      | NonNullable<RouterOutputs["scenarioVariantCells"]["get"]>["modelResponses"][0]
+      | undefined;
+    scenario: Scenario;
+  }
+> = ({ children, cell, hardRefetching, hardRefetch, mostRecentResponse, scenario, ...props }) => (
+  <VStack w="full" alignItems="flex-start" {...props} px={2} py={2} h="100%">
+    {cell && (
+      <CellOptions refetchingOutput={hardRefetching} refetchOutput={hardRefetch} cell={cell} />
+    )}
+    <VStack w="full" alignItems="flex-start" maxH={500} overflowY="auto" flex={1}>
+      {children}
+    </VStack>
+    {mostRecentResponse && <OutputStats modelResponse={mostRecentResponse} scenario={scenario} />}
+  </VStack>
+);
+
+export default CellWrapper;

app/src/components/OutputsTable/OutputCell/OutputCell.tsx

@@ -1,17 +1,16 @@
-import { api } from "~/utils/api";
-import { type PromptVariant, type Scenario } from "../types";
-import { type StackProps, Text, VStack } from "@chakra-ui/react";
-import { useScenarioVars, useHandledAsyncCallback } from "~/utils/hooks";
+import { Text } from "@chakra-ui/react";
+import stringify from "json-stringify-pretty-compact";
+import { Fragment, useEffect, useState, type ReactElement } from "react";
 import SyntaxHighlighter from "react-syntax-highlighter";
 import { docco } from "react-syntax-highlighter/dist/cjs/styles/hljs";
-import stringify from "json-stringify-pretty-compact";
-import { type ReactElement, useState, useEffect, Fragment, useCallback } from "react";
-import useSocket from "~/utils/useSocket";
-import { OutputStats } from "./OutputStats";
-import { RetryCountdown } from "./RetryCountdown";
 import frontendModelProviders from "~/modelProviders/frontendModelProviders";
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback, useScenarioVars } from "~/utils/hooks";
+import useSocket from "~/utils/useSocket";
+import { type PromptVariant, type Scenario } from "../types";
+import CellWrapper from "./CellWrapper";
 import { ResponseLog } from "./ResponseLog";
-import { CellOptions } from "./TopActions";
+import { RetryCountdown } from "./RetryCountdown";
 
 const WAITING_MESSAGE_INTERVAL = 20000;
 
@@ -72,35 +71,26 @@ export default function OutputCell({
 
   const mostRecentResponse = cell?.modelResponses[cell.modelResponses.length - 1];
 
-  const CellWrapper = useCallback(
-    ({ children, ...props }: StackProps) => (
-      <VStack w="full" alignItems="flex-start" {...props} px={2} py={2} h="100%">
-        {cell && (
-          <CellOptions refetchingOutput={hardRefetching} refetchOutput={hardRefetch} cell={cell} />
-        )}
-        <VStack w="full" alignItems="flex-start" maxH={500} overflowY="auto" flex={1}>
-          {children}
-        </VStack>
-        {mostRecentResponse && (
-          <OutputStats modelResponse={mostRecentResponse} scenario={scenario} />
-        )}
-      </VStack>
-    ),
-    [hardRefetching, hardRefetch, mostRecentResponse, scenario, cell],
-  );
+  const wrapperProps: Parameters<typeof CellWrapper>[0] = {
+    cell,
+    hardRefetching,
+    hardRefetch,
+    mostRecentResponse,
+    scenario,
+  };
 
   if (!vars) return null;
 
   if (!cell && !fetchingOutput)
     return (
-      <CellWrapper>
+      <CellWrapper {...wrapperProps}>
         <Text color="gray.500">Error retrieving output</Text>
       </CellWrapper>
     );
 
   if (cell && cell.errorMessage) {
     return (
-      <CellWrapper>
+      <CellWrapper {...wrapperProps}>
         <Text color="red.500">{cell.errorMessage}</Text>
       </CellWrapper>
     );
@@ -112,7 +102,12 @@ export default function OutputCell({
 
   if (showLogs)
     return (
-      <CellWrapper alignItems="flex-start" fontFamily="inconsolata, monospace" spacing={0}>
+      <CellWrapper
+        {...wrapperProps}
+        alignItems="flex-start"
+        fontFamily="inconsolata, monospace"
+        spacing={0}
+      >
         {cell?.jobQueuedAt && <ResponseLog time={cell.jobQueuedAt} title="Job queued" />}
         {cell?.jobStartedAt && <ResponseLog time={cell.jobStartedAt} title="Job started" />}
         {cell?.modelResponses?.map((response) => {
@@ -174,7 +169,7 @@ export default function OutputCell({
 
   if (mostRecentResponse?.respPayload && normalizedOutput?.type === "json") {
     return (
-      <CellWrapper>
+      <CellWrapper {...wrapperProps}>
         <SyntaxHighlighter
           customStyle={{ overflowX: "unset", width: "100%", flex: 1 }}
           language="json"
@@ -193,7 +188,7 @@ export default function OutputCell({
   const contentToDisplay = (normalizedOutput?.type === "text" && normalizedOutput.value) || "";
 
   return (
-    <CellWrapper>
+    <CellWrapper {...wrapperProps}>
       <Text whiteSpace="pre-wrap">{contentToDisplay}</Text>
     </CellWrapper>
   );

app/src/components/OutputsTable/OutputCell/PromptModal.tsx

@@ -5,30 +5,103 @@ import {
   ModalContent,
   ModalHeader,
   ModalOverlay,
+  VStack,
+  Text,
+  Box,
   type UseDisclosureReturn,
+  Link,
 } from "@chakra-ui/react";
-import { type RouterOutputs } from "~/utils/api";
+import { api, type RouterOutputs } from "~/utils/api";
 import { JSONTree } from "react-json-tree";
+import CopiableCode from "~/components/CopiableCode";
 
-export default function ExpandedModal(props: {
+const theme = {
+  scheme: "chalk",
+  author: "chris kempson (http://chriskempson.com)",
+  base00: "transparent",
+  base01: "#202020",
+  base02: "#303030",
+  base03: "#505050",
+  base04: "#b0b0b0",
+  base05: "#d0d0d0",
+  base06: "#e0e0e0",
+  base07: "#f5f5f5",
+  base08: "#fb9fb1",
+  base09: "#eda987",
+  base0A: "#ddb26f",
+  base0B: "#acc267",
+  base0C: "#12cfc0",
+  base0D: "#6fc2ef",
+  base0E: "#e1a3ee",
+  base0F: "#deaf8f",
+};
+
+export default function PromptModal(props: {
   cell: NonNullable<RouterOutputs["scenarioVariantCells"]["get"]>;
   disclosure: UseDisclosureReturn;
 }) {
+  const { data } = api.scenarioVariantCells.getTemplatedPromptMessage.useQuery(
+    {
+      cellId: props.cell.id,
+    },
+    {
+      enabled: props.disclosure.isOpen,
+    },
+  );
+
   return (
-    <Modal isOpen={props.disclosure.isOpen} onClose={props.disclosure.onClose} size="2xl">
+    <Modal isOpen={props.disclosure.isOpen} onClose={props.disclosure.onClose} size="xl">
       <ModalOverlay />
       <ModalContent>
-        <ModalHeader>Prompt</ModalHeader>
+        <ModalHeader>Prompt Details</ModalHeader>
         <ModalCloseButton />
         <ModalBody>
-          <JSONTree
-            data={props.cell.prompt}
-            invertTheme={true}
-            theme="chalk"
-            shouldExpandNodeInitially={() => true}
-            getItemString={() => ""}
-            hideRoot
-          />
+          <VStack py={4} w="">
+            <VStack w="full" alignItems="flex-start">
+              <Text fontWeight="bold">Full Prompt</Text>
+              <Box
+                w="full"
+                p={4}
+                alignItems="flex-start"
+                backgroundColor="blackAlpha.800"
+                borderRadius={4}
+              >
+                <JSONTree
+                  data={props.cell.prompt}
+                  theme={theme}
+                  shouldExpandNodeInitially={() => true}
+                  getItemString={() => ""}
+                  hideRoot
+                />
+              </Box>
+            </VStack>
+            {data?.templatedPrompt && (
+              <VStack w="full" mt={4} alignItems="flex-start">
+                <Text fontWeight="bold">Templated prompt message:</Text>
+                <CopiableCode
+                  w="full"
+                  // bgColor="gray.100"
+                  p={4}
+                  borderWidth={1}
+                  whiteSpace="pre-wrap"
+                  code={data.templatedPrompt}
+                />
+              </VStack>
+            )}
+            {data?.learnMoreUrl && (
+              <Link
+                href={data.learnMoreUrl}
+                isExternal
+                color="blue.500"
+                fontWeight="bold"
+                fontSize="sm"
+                mt={4}
+                alignSelf="flex-end"
+              >
+                Learn More
+              </Link>
+            )}
+          </VStack>
         </ModalBody>
       </ModalContent>
     </Modal>

app/src/components/OutputsTable/VariantStats.tsx

@@ -21,14 +21,18 @@ export default function VariantStats(props: { variant: PromptVariant }) {
         outputTokens: 0,
         scenarioCount: 0,
         outputCount: 0,
+        awaitingCompletions: false,
         awaitingEvals: false,
       },
       refetchInterval,
     },
   );
 
-  // Poll every two seconds while we are waiting for LLM retrievals to finish
-  useEffect(() => setRefetchInterval(data.awaitingEvals ? 5000 : 0), [data.awaitingEvals]);
+  // Poll every five seconds while we are waiting for LLM retrievals to finish
+  useEffect(
+    () => setRefetchInterval(data.awaitingCompletions || data.awaitingEvals ? 5000 : 0),
+    [data.awaitingCompletions, data.awaitingEvals],
+  );
 
   const [passColor, neutralColor, failColor] = useToken("colors", [
     "green.500",

app/src/components/Paginator.tsx

@@ -37,6 +37,8 @@ const Paginator = ({
   const goToLastPage = () => setPageParams({ page: lastPage }, "replace");
   const goToFirstPage = () => setPageParams({ page: 1 }, "replace");
 
+  if (count === 0) return null;
+
   return (
     <HStack
       pt={4}

app/src/components/nav/ProjectMenu.tsx

@@ -15,7 +15,7 @@ import {
   Image,
   Box,
 } from "@chakra-ui/react";
-import React, { useEffect, useState } from "react";
+import { useEffect } from "react";
 import Link from "next/link";
 import { BsPlus, BsPersonCircle } from "react-icons/bs";
 import { type Project } from "@prisma/client";
@@ -105,8 +105,8 @@ export default function ProjectMenu() {
         </PopoverTrigger>
         <PopoverContent
           _focusVisible={{ outline: "unset" }}
-          ml={-1}
-          w={224}
+          w={220}
+          ml={{ base: 2, md: 0 }}
           boxShadow="0 0 40px 4px rgba(0, 0, 0, 0.1);"
           fontSize="sm"
         >
@@ -176,7 +176,6 @@ const ProjectOption = ({
   onClose: () => void;
 }) => {
   const setSelectedProjectId = useAppStore((s) => s.setSelectedProjectId);
-  const [gearHovered, setGearHovered] = useState(false);
 
   return (
     <HStack
@@ -188,8 +187,8 @@ const ProjectOption = ({
       }}
       w="full"
       justifyContent="space-between"
-      _hover={gearHovered ? undefined : { bgColor: "gray.200", textDecoration: "none" }}
-      color={isActive ? "blue.400" : undefined}
+      _hover={{ bgColor: "gray.200", textDecoration: "none" }}
+      bgColor={isActive ? "gray.100" : undefined}
       py={2}
       px={4}
       borderRadius={4}

app/src/components/projectSettings/InviteMemberModal.tsx

@@ -0,0 +1,128 @@
+import {
+  Button,
+  FormControl,
+  FormLabel,
+  Input,
+  FormHelperText,
+  HStack,
+  Modal,
+  ModalBody,
+  ModalCloseButton,
+  ModalContent,
+  ModalFooter,
+  ModalHeader,
+  ModalOverlay,
+  Spinner,
+  Text,
+  VStack,
+  RadioGroup,
+  Radio,
+} from "@chakra-ui/react";
+import { useState, useEffect } from "react";
+
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback, useSelectedProject } from "~/utils/hooks";
+import { maybeReportError } from "~/utils/errorHandling/maybeReportError";
+import { type ProjectUserRole } from "@prisma/client";
+
+export const InviteMemberModal = ({
+  isOpen,
+  onClose,
+}: {
+  isOpen: boolean;
+  onClose: () => void;
+}) => {
+  const selectedProject = useSelectedProject().data;
+  const utils = api.useContext();
+
+  const [email, setEmail] = useState("");
+  const [role, setRole] = useState<ProjectUserRole>("MEMBER");
+
+  useEffect(() => {
+    setEmail("");
+    setRole("MEMBER");
+  }, [isOpen]);
+
+  const emailIsValid = !email || !email.match(/.+@.+\..+/);
+
+  const inviteMemberMutation = api.users.inviteToProject.useMutation();
+
+  const [inviteMember, isInviting] = useHandledAsyncCallback(async () => {
+    if (!selectedProject?.id || !role) return;
+    const resp = await inviteMemberMutation.mutateAsync({
+      projectId: selectedProject.id,
+      email,
+      role,
+    });
+    if (maybeReportError(resp)) return;
+    await utils.projects.get.invalidate();
+    onClose();
+  }, [inviteMemberMutation, email, role, selectedProject?.id, onClose]);
+
+  return (
+    <Modal isOpen={isOpen} onClose={onClose}>
+      <ModalOverlay />
+      <ModalContent w={1200}>
+        <ModalHeader>
+          <HStack>
+            <Text>Invite Member</Text>
+          </HStack>
+        </ModalHeader>
+        <ModalCloseButton />
+        <ModalBody>
+          <VStack spacing={8} alignItems="flex-start">
+            <Text>
+              Invite a new member to <b>{selectedProject?.name}</b>.
+            </Text>
+
+            <RadioGroup
+              value={role}
+              onChange={(e) => setRole(e as ProjectUserRole)}
+              colorScheme="orange"
+            >
+              <VStack w="full" alignItems="flex-start">
+                <Radio value="MEMBER">
+                  <Text fontSize="sm">MEMBER</Text>
+                </Radio>
+                <Radio value="ADMIN">
+                  <Text fontSize="sm">ADMIN</Text>
+                </Radio>
+              </VStack>
+            </RadioGroup>
+            <FormControl>
+              <FormLabel>Email</FormLabel>
+              <Input
+                type="email"
+                value={email}
+                onChange={(e) => setEmail(e.target.value)}
+                onKeyDown={(e) => {
+                  if (e.key === "Enter" && (e.metaKey || e.ctrlKey || e.shiftKey)) {
+                    e.preventDefault();
+                    e.currentTarget.blur();
+                    inviteMember();
+                  }
+                }}
+              />
+              <FormHelperText>Enter the email of the person you want to invite.</FormHelperText>
+            </FormControl>
+          </VStack>
+        </ModalBody>
+        <ModalFooter mt={4}>
+          <HStack>
+            <Button colorScheme="gray" onClick={onClose} minW={24}>
+              <Text>Cancel</Text>
+            </Button>
+            <Button
+              colorScheme="orange"
+              onClick={inviteMember}
+              minW={24}
+              isDisabled={emailIsValid || isInviting}
+            >
+              {isInviting ? <Spinner boxSize={4} /> : <Text>Send Invitation</Text>}
+            </Button>
+          </HStack>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};

app/src/components/projectSettings/MemberTable.tsx

@@ -0,0 +1,145 @@
+import { useMemo, useState } from "react";
+import {
+  Table,
+  Thead,
+  Tr,
+  Th,
+  Tbody,
+  Td,
+  IconButton,
+  useDisclosure,
+  Text,
+  Button,
+} from "@chakra-ui/react";
+import { useSession } from "next-auth/react";
+import { BsTrash } from "react-icons/bs";
+import { type User } from "@prisma/client";
+
+import { useHandledAsyncCallback, useSelectedProject } from "~/utils/hooks";
+import { InviteMemberModal } from "./InviteMemberModal";
+import { RemoveMemberDialog } from "./RemoveMemberDialog";
+import { api } from "~/utils/api";
+import { maybeReportError } from "~/utils/errorHandling/maybeReportError";
+
+const MemberTable = () => {
+  const selectedProject = useSelectedProject().data;
+  const session = useSession().data;
+
+  const utils = api.useContext();
+
+  const [memberToRemove, setMemberToRemove] = useState<User | null>(null);
+  const inviteMemberModal = useDisclosure();
+
+  const cancelInvitationMutation = api.users.cancelProjectInvitation.useMutation();
+
+  const [cancelInvitation, isCancelling] = useHandledAsyncCallback(
+    async (invitationToken: string) => {
+      if (!selectedProject?.id) return;
+      const resp = await cancelInvitationMutation.mutateAsync({
+        invitationToken,
+      });
+      if (maybeReportError(resp)) return;
+      await utils.projects.get.invalidate();
+    },
+    [selectedProject?.id, cancelInvitationMutation],
+  );
+
+  const sortedMembers = useMemo(() => {
+    if (!selectedProject?.projectUsers) return [];
+    return selectedProject.projectUsers.sort((a, b) => {
+      if (a.role === b.role) return a.createdAt < b.createdAt ? -1 : 1;
+      // Take advantage of fact that ADMIN is alphabetically before MEMBER
+      return a.role < b.role ? -1 : 1;
+    });
+  }, [selectedProject?.projectUsers]);
+
+  return (
+    <>
+      <Table fontSize={{ base: "sm", md: "md" }}>
+        <Thead
+          sx={{
+            th: {
+              base: { px: 0 },
+              md: { px: 6 },
+            },
+          }}
+        >
+          <Tr>
+            <Th>Name</Th>
+            <Th display={{ base: "none", md: "table-cell" }}>Email</Th>
+            <Th>Role</Th>
+            {selectedProject?.role === "ADMIN" && <Th />}
+          </Tr>
+        </Thead>
+        <Tbody
+          sx={{
+            td: {
+              base: { px: 0 },
+              md: { px: 6 },
+            },
+          }}
+        >
+          {selectedProject &&
+            sortedMembers.map((member) => {
+              return (
+                <Tr key={member.id}>
+                  <Td>
+                    <Text fontWeight="bold">{member.user.name}</Text>
+                  </Td>
+                  <Td display={{ base: "none", md: "table-cell" }} h="full">
+                    {member.user.email}
+                  </Td>
+                  <Td fontSize={{ base: "xs", md: "sm" }}>{member.role}</Td>
+                  {selectedProject.role === "ADMIN" && (
+                    <Td textAlign="end">
+                      {member.user.id !== session?.user?.id &&
+                        member.user.id !== selectedProject.personalProjectUserId && (
+                          <IconButton
+                            aria-label="Remove member"
+                            colorScheme="red"
+                            icon={<BsTrash />}
+                            onClick={() => setMemberToRemove(member.user)}
+                          />
+                        )}
+                    </Td>
+                  )}
+                </Tr>
+              );
+            })}
+          {selectedProject?.projectUserInvitations?.map((invitation) => {
+            return (
+              <Tr key={invitation.id}>
+                <Td>
+                  <Text as="i">Invitation pending</Text>
+                </Td>
+                <Td>{invitation.email}</Td>
+                <Td fontSize="sm">{invitation.role}</Td>
+                {selectedProject.role === "ADMIN" && (
+                  <Td textAlign="end">
+                    <Button
+                      size="sm"
+                      colorScheme="red"
+                      variant="ghost"
+                      onClick={() => cancelInvitation(invitation.invitationToken)}
+                      isLoading={isCancelling}
+                    >
+                      Cancel
+                    </Button>
+                  </Td>
+                )}
+              </Tr>
+            );
+          })}
+        </Tbody>
+      </Table>
+      <InviteMemberModal isOpen={inviteMemberModal.isOpen} onClose={inviteMemberModal.onClose} />
+      <RemoveMemberDialog
+        member={memberToRemove}
+        isOpen={!!memberToRemove}
+        onClose={() => setMemberToRemove(null)}
+      />
+    </>
+  );
+};
+
+export default MemberTable;

app/src/components/projectSettings/RemoveMemberDialog.tsx

@@ -0,0 +1,71 @@
+import {
+  Button,
+  AlertDialog,
+  AlertDialogBody,
+  AlertDialogFooter,
+  AlertDialogHeader,
+  AlertDialogContent,
+  AlertDialogOverlay,
+  Text,
+  VStack,
+  Spinner,
+} from "@chakra-ui/react";
+import { type User } from "@prisma/client";
+
+import { useRouter } from "next/router";
+import { useRef } from "react";
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback, useSelectedProject } from "~/utils/hooks";
+
+export const RemoveMemberDialog = ({
+  isOpen,
+  onClose,
+  member,
+}: {
+  isOpen: boolean;
+  onClose: () => void;
+  member: User | null;
+}) => {
+  const selectedProject = useSelectedProject();
+  const removeUserMutation = api.users.removeUserFromProject.useMutation();
+  const utils = api.useContext();
+  const router = useRouter();
+
+  const cancelRef = useRef<HTMLButtonElement>(null);
+
+  const [onRemoveConfirm, isRemoving] = useHandledAsyncCallback(async () => {
+    if (!selectedProject.data?.id || !member?.id) return;
+    await removeUserMutation.mutateAsync({ projectId: selectedProject.data.id, userId: member.id });
+    await utils.projects.get.invalidate();
+    onClose();
+  }, [removeUserMutation, selectedProject, router]);
+
+  return (
+    <AlertDialog isOpen={isOpen} leastDestructiveRef={cancelRef} onClose={onClose}>
+      <AlertDialogOverlay>
+        <AlertDialogContent>
+          <AlertDialogHeader fontSize="lg" fontWeight="bold">
+            Remove Member
+          </AlertDialogHeader>
+
+          <AlertDialogBody>
+            <VStack spacing={4} alignItems="flex-start">
+              <Text>
+                Are you sure you want to remove <b>{member?.name}</b> from the project?
+              </Text>
+            </VStack>
+          </AlertDialogBody>
+
+          <AlertDialogFooter>
+            <Button ref={cancelRef} onClick={onClose}>
+              Cancel
+            </Button>
+            <Button colorScheme="red" onClick={onRemoveConfirm} ml={3} w={20}>
+              {isRemoving ? <Spinner /> : "Remove"}
+            </Button>
+          </AlertDialogFooter>
+        </AlertDialogContent>
+      </AlertDialogOverlay>
+    </AlertDialog>
+  );
+};

app/src/components/requestLogs/LogFilters/AddFilterButton.tsx

@@ -0,0 +1,30 @@
+import { Button, HStack, Icon, Text } from "@chakra-ui/react";
+import { BsPlus } from "react-icons/bs";
+import { comparators, defaultFilterableFields } from "~/state/logFiltersSlice";
+import { useAppStore } from "~/state/store";
+
+const AddFilterButton = () => {
+  const addFilter = useAppStore((s) => s.logFilters.addFilter);
+
+  return (
+    <HStack
+      as={Button}
+      variant="ghost"
+      onClick={() =>
+        addFilter({
+          id: Date.now().toString(),
+          field: defaultFilterableFields[0],
+          comparator: comparators[0],
+          value: "",
+        })
+      }
+      spacing={0}
+      fontSize="sm"
+    >
+      <Icon as={BsPlus} boxSize={5} />
+      <Text>Add Filter</Text>
+    </HStack>
+  );
+};
+
+export default AddFilterButton;

app/src/components/requestLogs/LogFilters/LogFilter.tsx

@@ -0,0 +1,44 @@
+import { useCallback, useState } from "react";
+import { HStack, IconButton, Input } from "@chakra-ui/react";
+import { BsTrash } from "react-icons/bs";
+
+import { type LogFilter } from "~/state/logFiltersSlice";
+import { useAppStore } from "~/state/store";
+import { debounce } from "lodash-es";
+import SelectFieldDropdown from "./SelectFieldDropdown";
+import SelectComparatorDropdown from "./SelectComparatorDropdown";
+
+const LogFilter = ({ filter }: { filter: LogFilter }) => {
+  const updateFilter = useAppStore((s) => s.logFilters.updateFilter);
+  const deleteFilter = useAppStore((s) => s.logFilters.deleteFilter);
+
+  const [editedValue, setEditedValue] = useState(filter.value);
+
+  const debouncedUpdateFilter = useCallback(
+    debounce((filter: LogFilter) => updateFilter(filter), 500, {
+      leading: true,
+    }),
+    [updateFilter],
+  );
+
+  return (
+    <HStack>
+      <SelectFieldDropdown filter={filter} />
+      <SelectComparatorDropdown filter={filter} />
+      <Input
+        value={editedValue}
+        onChange={(e) => {
+          setEditedValue(e.target.value);
+          debouncedUpdateFilter({ ...filter, value: e.target.value });
+        }}
+      />
+      <IconButton
+        aria-label="Delete Filter"
+        icon={<BsTrash />}
+        onClick={() => deleteFilter(filter.id)}
+      />
+    </HStack>
+  );
+};
+
+export default LogFilter;

app/src/components/requestLogs/LogFilters/LogFilters.tsx

@@ -0,0 +1,30 @@
+import { VStack, Text } from "@chakra-ui/react";
+
+import AddFilterButton from "./AddFilterButton";
+import { useAppStore } from "~/state/store";
+import LogFilter from "./LogFilter";
+
+const LogFilters = () => {
+  const filters = useAppStore((s) => s.logFilters.filters);
+  return (
+    <VStack
+      bgColor="white"
+      borderRadius={8}
+      borderWidth={1}
+      w="full"
+      alignItems="flex-start"
+      p={4}
+      spacing={4}
+    >
+      <Text fontWeight="bold" color="gray.500">
+        Filters
+      </Text>
+      {filters.map((filter) => (
+        <LogFilter key={filter.id} filter={filter} />
+      ))}
+      <AddFilterButton />
+    </VStack>
+  );
+};
+
+export default LogFilters;

app/src/components/requestLogs/LogFilters/SelectComparatorDropdown.tsx

@@ -0,0 +1,19 @@
+import { comparators, type LogFilter } from "~/state/logFiltersSlice";
+import { useAppStore } from "~/state/store";
+import InputDropdown from "~/components/InputDropdown";
+
+const SelectComparatorDropdown = ({ filter }: { filter: LogFilter }) => {
+  const updateFilter = useAppStore((s) => s.logFilters.updateFilter);
+
+  const { comparator } = filter;
+
+  return (
+    <InputDropdown
+      options={comparators}
+      selectedOption={comparator}
+      onSelect={(option) => updateFilter({ ...filter, comparator: option })}
+    />
+  );
+};
+
+export default SelectComparatorDropdown;

app/src/components/requestLogs/LogFilters/SelectFieldDropdown.tsx

@@ -0,0 +1,22 @@
+import { defaultFilterableFields, type LogFilter } from "~/state/logFiltersSlice";
+import { useAppStore } from "~/state/store";
+import { useTagNames } from "~/utils/hooks";
+import InputDropdown from "~/components/InputDropdown";
+
+const SelectFieldDropdown = ({ filter }: { filter: LogFilter }) => {
+  const tagNames = useTagNames().data;
+
+  const updateFilter = useAppStore((s) => s.logFilters.updateFilter);
+
+  const { field } = filter;
+
+  return (
+    <InputDropdown
+      options={[...defaultFilterableFields, ...(tagNames || [])]}
+      selectedOption={field}
+      onSelect={(option) => updateFilter({ ...filter, field: option })}
+    />
+  );
+};
+
+export default SelectFieldDropdown;

app/src/components/requestLogs/LoggedCallsTable.tsx

@@ -5,14 +5,14 @@ import { TableHeader, TableRow } from "./TableRow";
 
 export default function LoggedCallsTable() {
   const [expandedRow, setExpandedRow] = useState<string | null>(null);
-  const { data: loggedCalls } = useLoggedCalls();
+  const loggedCalls = useLoggedCalls().data;
 
   return (
-    <Card width="100%" overflow="hidden">
+    <Card width="100%" overflowX="auto">
       <Table>
         <TableHeader showCheckbox />
         <Tbody>
-          {loggedCalls?.calls.map((loggedCall) => {
+          {loggedCalls?.calls?.map((loggedCall) => {
             return (
               <TableRow
                 key={loggedCall.id}

app/src/components/requestLogs/TableRow.tsx

@@ -21,7 +21,7 @@ import Link from "next/link";
 import { type RouterOutputs } from "~/utils/api";
 import { FormattedJson } from "./FormattedJson";
 import { useAppStore } from "~/state/store";
-import { useLoggedCalls } from "~/utils/hooks";
+import { useLoggedCalls, useTagNames } from "~/utils/hooks";
 import { useMemo } from "react";
 
 dayjs.extend(relativeTime);
@@ -34,27 +34,32 @@ export const TableHeader = ({ showCheckbox }: { showCheckbox?: boolean }) => {
   const addAll = useAppStore((s) => s.selectedLogs.addSelectedLogIds);
   const clearAll = useAppStore((s) => s.selectedLogs.clearSelectedLogIds);
   const allSelected = useMemo(() => {
-    if (!matchingLogIds) return false;
+    if (!matchingLogIds || !matchingLogIds.length) return false;
     return matchingLogIds.every((id) => selectedLogIds.has(id));
   }, [selectedLogIds, matchingLogIds]);
+  const tagNames = useTagNames().data;
   return (
     <Thead>
       <Tr>
         {showCheckbox && (
-          <Th>
-            <HStack w={8}>
+          <Th pr={0}>
+            <HStack minW={16}>
               <Checkbox
                 isChecked={allSelected}
                 onChange={() => {
                   allSelected ? clearAll() : addAll(matchingLogIds || []);
                 }}
               />
-              <Text>({selectedLogIds.size})</Text>
+              <Text>
+                ({selectedLogIds.size ? `${selectedLogIds.size}/` : ""}
+                {matchingLogIds?.length || 0})
+              </Text>
             </HStack>
           </Th>
         )}
-        <Th>Time</Th>
+        <Th>Sent At</Th>
         <Th>Model</Th>
+        {tagNames?.map((tagName) => <Th key={tagName}>{tagName}</Th>)}
         <Th isNumeric>Duration</Th>
         <Th isNumeric>Input tokens</Th>
         <Th isNumeric>Output tokens</Th>
@@ -76,22 +81,14 @@ export const TableRow = ({
   showCheckbox?: boolean;
 }) => {
   const isError = loggedCall.modelResponse?.statusCode !== 200;
-  const timeAgo = dayjs(loggedCall.requestedAt).fromNow();
+  const requestedAt = dayjs(loggedCall.requestedAt).format("MMMM D h:mm A");
   const fullTime = dayjs(loggedCall.requestedAt).toString();
 
-  const durationCell = (
-    <Td isNumeric>
-      {loggedCall.cacheHit ? (
-        <Text color="gray.500">Cached</Text>
-      ) : (
-        ((loggedCall.modelResponse?.durationMs ?? 0) / 1000).toFixed(2) + "s"
-      )}
-    </Td>
-  );
-
   const isChecked = useAppStore((s) => s.selectedLogs.selectedLogIds.has(loggedCall.id));
   const toggleChecked = useAppStore((s) => s.selectedLogs.toggleSelectedLogId);
 
+  const tagNames = useTagNames().data;
+
   return (
     <>
       <Tr
@@ -101,6 +98,7 @@ export const TableRow = ({
         sx={{
           "> td": { borderBottom: "none" },
         }}
+        fontSize="sm"
       >
         {showCheckbox && (
           <Td>
@@ -110,11 +108,11 @@ export const TableRow = ({
         <Td>
           <Tooltip label={fullTime} placement="top">
             <Box whiteSpace="nowrap" minW="120px">
-              {timeAgo}
+              {requestedAt}
             </Box>
           </Tooltip>
         </Td>
-        <Td width="100%">
+        <Td>
           <HStack justifyContent="flex-start">
             <Text
               colorScheme="purple"
@@ -124,12 +122,20 @@ export const TableRow = ({
               borderRadius={4}
               borderWidth={1}
               fontSize="xs"
+              whiteSpace="nowrap"
             >
               {loggedCall.model}
             </Text>
           </HStack>
         </Td>
-        {durationCell}
+        {tagNames?.map((tagName) => <Td key={tagName}>{loggedCall.tags[tagName]}</Td>)}
+        <Td isNumeric>
+          {loggedCall.cacheHit ? (
+            <Text color="gray.500">Cached</Text>
+          ) : (
+            ((loggedCall.modelResponse?.durationMs ?? 0) / 1000).toFixed(2) + "s"
+          )}
+        </Td>
         <Td isNumeric>{loggedCall.modelResponse?.inputTokens}</Td>
         <Td isNumeric>{loggedCall.modelResponse?.outputTokens}</Td>
         <Td sx={{ color: isError ? "red.500" : "green.500", fontWeight: "semibold" }} isNumeric>

app/src/env.mjs

@@ -21,6 +21,11 @@ export const env = createEnv({
     ANTHROPIC_API_KEY: z.string().default("placeholder"),
     SENTRY_AUTH_TOKEN: z.string().optional(),
     OPENPIPE_API_KEY: z.string().optional(),
+    SENDER_EMAIL: z.string().default("placeholder"),
+    SMTP_HOST: z.string().default("placeholder"),
+    SMTP_PORT: z.string().default("placeholder"),
+    SMTP_LOGIN: z.string().default("placeholder"),
+    SMTP_PASSWORD: z.string().default("placeholder"),
   },
 
   /**
@@ -58,6 +63,11 @@ export const env = createEnv({
     SENTRY_AUTH_TOKEN: process.env.SENTRY_AUTH_TOKEN,
     OPENPIPE_API_KEY: process.env.OPENPIPE_API_KEY,
     NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS: process.env.NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS,
+    SENDER_EMAIL: process.env.SENDER_EMAIL,
+    SMTP_HOST: process.env.SMTP_HOST,
+    SMTP_PORT: process.env.SMTP_PORT,
+    SMTP_LOGIN: process.env.SMTP_LOGIN,
+    SMTP_PASSWORD: process.env.SMTP_PASSWORD,
   },
   /**
    * Run `build` or `dev` with `SKIP_ENV_VALIDATION` to skip env validation.

app/src/modelProviders/frontendModelProviders.ts

@@ -1,6 +1,7 @@
 import openaiChatCompletionFrontend from "./openai-ChatCompletion/frontend";
 import replicateLlama2Frontend from "./replicate-llama2/frontend";
 import anthropicFrontend from "./anthropic-completion/frontend";
+import openpipeFrontend from "./openpipe-chat/frontend";
 import { type SupportedProvider, type FrontendModelProvider } from "./types";
 
 // Keep attributes here that need to be accessible from the frontend. We can't
@@ -10,6 +11,7 @@ const frontendModelProviders: Record<SupportedProvider, FrontendModelProvider<an
   "openai/ChatCompletion": openaiChatCompletionFrontend,
   "replicate/llama2": replicateLlama2Frontend,
   "anthropic/completion": anthropicFrontend,
+  "openpipe/Chat": openpipeFrontend,
 };
 
 export default frontendModelProviders;

app/src/modelProviders/modelProviders.ts

@@ -1,12 +1,14 @@
 import openaiChatCompletion from "./openai-ChatCompletion";
 import replicateLlama2 from "./replicate-llama2";
 import anthropicCompletion from "./anthropic-completion";
+import openpipeChatCompletion from "./openpipe-chat";
 import { type SupportedProvider, type ModelProvider } from "./types";
 
 const modelProviders: Record<SupportedProvider, ModelProvider<any, any, any>> = {
   "openai/ChatCompletion": openaiChatCompletion,
   "replicate/llama2": replicateLlama2,
   "anthropic/completion": anthropicCompletion,
+  "openpipe/Chat": openpipeChatCompletion,
 };
 
 export default modelProviders;

app/src/modelProviders/openai-ChatCompletion/getCompletion.ts

@@ -1,54 +1,10 @@
 /* eslint-disable @typescript-eslint/no-unsafe-call */
-import {
-  type ChatCompletionChunk,
-  type ChatCompletion,
-  type CompletionCreateParams,
-} from "openai/resources/chat";
-import { type CompletionResponse } from "../types";
-import { isArray, isString, omit } from "lodash-es";
-import { openai } from "~/server/utils/openai";
+import { isArray, isString } from "lodash-es";
 import { APIError } from "openai";
-
-const mergeStreamedChunks = (
-  base: ChatCompletion | null,
-  chunk: ChatCompletionChunk,
-): ChatCompletion => {
-  if (base === null) {
-    return mergeStreamedChunks({ ...chunk, choices: [] }, chunk);
-  }
-
-  const choices = [...base.choices];
-  for (const choice of chunk.choices) {
-    const baseChoice = choices.find((c) => c.index === choice.index);
-    if (baseChoice) {
-      baseChoice.finish_reason = choice.finish_reason ?? baseChoice.finish_reason;
-      baseChoice.message = baseChoice.message ?? { role: "assistant" };
-
-      if (choice.delta?.content)
-        baseChoice.message.content =
-          ((baseChoice.message.content as string) ?? "") + (choice.delta.content ?? "");
-      if (choice.delta?.function_call) {
-        const fnCall = baseChoice.message.function_call ?? {};
-        fnCall.name =
-          ((fnCall.name as string) ?? "") + ((choice.delta.function_call.name as string) ?? "");
-        fnCall.arguments =
-          ((fnCall.arguments as string) ?? "") +
-          ((choice.delta.function_call.arguments as string) ?? "");
-      }
-    } else {
-      // @ts-expect-error the types are correctly telling us that finish_reason
-      // could be null, but don't want to fix it right now.
-      choices.push({ ...omit(choice, "delta"), message: { role: "assistant", ...choice.delta } });
-    }
-  }
-
-  const merged: ChatCompletion = {
-    ...base,
-    choices,
-  };
-
-  return merged;
-};
+import { type ChatCompletion, type CompletionCreateParams } from "openai/resources/chat";
+import mergeChunks from "openpipe/src/openai/mergeChunks";
+import { openai } from "~/server/utils/openai";
+import { type CompletionResponse } from "../types";
 
 export async function getCompletion(
   input: CompletionCreateParams,
@@ -59,7 +15,6 @@ export async function getCompletion(
 
   try {
     if (onStream) {
-      console.log("got started");
       const resp = await openai.chat.completions.create(
         { ...input, stream: true },
         {
@@ -67,11 +22,9 @@ export async function getCompletion(
         },
       );
       for await (const part of resp) {
-        console.log("got part", part);
-        finalCompletion = mergeStreamedChunks(finalCompletion, part);
+        finalCompletion = mergeChunks(finalCompletion, part);
         onStream(finalCompletion);
       }
-      console.log("got final", finalCompletion);
       if (!finalCompletion) {
         return {
           type: "error",

app/src/modelProviders/openai-ChatCompletion/refinementActions.ts

@@ -120,9 +120,9 @@ export const refinementActions: Record<string, RefinementAction> = {
   "Convert to function call": {
     icon: TfiThought,
     description: "Use function calls to get output from the model in a more structured way.",
-    instructions: `OpenAI functions are a specialized way for an LLM to return output.
+    instructions: `OpenAI functions are a specialized way for an LLM to return its final output.
   
-      This is what a prompt looks like before adding a function:
+      Example 1 before:
   
       definePrompt("openai/ChatCompletion", {
         model: "gpt-4",
@@ -139,7 +139,7 @@ export const refinementActions: Record<string, RefinementAction> = {
         ],
       });
   
-      This is what one looks like after adding a function:
+      Example 1 after:
   
       definePrompt("openai/ChatCompletion", {
         model: "gpt-4",
@@ -156,7 +156,7 @@ export const refinementActions: Record<string, RefinementAction> = {
         ],
         functions: [
           {
-            name: "extract_sentiment",
+            name: "log_extracted_sentiment",
             parameters: {
               type: "object", // parameters must always be an object with a properties key
               properties: { // properties key is required
@@ -169,13 +169,13 @@ export const refinementActions: Record<string, RefinementAction> = {
           },
         ],
         function_call: {
-          name: "extract_sentiment",
+          name: "log_extracted_sentiment",
         },
       });
   
-      Here's another example of adding a function:
-  
-      Before:
+      =========
+
+      Example 2 before:
   
       definePrompt("openai/ChatCompletion", {
           model: "gpt-3.5-turbo",
@@ -197,7 +197,7 @@ export const refinementActions: Record<string, RefinementAction> = {
           temperature: 0,
       });
   
-      After:
+      Example 2 after:
   
       definePrompt("openai/ChatCompletion", {
           model: "gpt-3.5-turbo",
@@ -215,7 +215,7 @@ export const refinementActions: Record<string, RefinementAction> = {
           temperature: 0,
           functions: [
             {
-              name: "score_post",
+              name: "log_post_score",
               parameters: {
                 type: "object",
                 properties: {
@@ -227,13 +227,13 @@ export const refinementActions: Record<string, RefinementAction> = {
             },
           ],
           function_call: {
-            name: "score_post",
+            name: "log_post_score",
           },
         });
   
-      Another example
+      =========
   
-      Before:
+      Example 3 before:
   
       definePrompt("openai/ChatCompletion", {
         model: "gpt-3.5-turbo",
@@ -246,7 +246,7 @@ export const refinementActions: Record<string, RefinementAction> = {
         ],
       });
   
-      After:
+      Example 3 after:
   
       definePrompt("openai/ChatCompletion", {
         model: "gpt-3.5-turbo",
@@ -258,21 +258,24 @@ export const refinementActions: Record<string, RefinementAction> = {
         ],
         functions: [
           {
-            name: "write_in_language",
+            name: "log_translated_text",
             parameters: {
               type: "object",
               properties: {
-                text: {
+                translated_text: {
                   type: "string",
+                  description: "The text, written in the language specified in the prompt",
                 },
               },
             },
           },
         ],
         function_call: {
-          name: "write_in_language",
+          name: "log_translated_text",
         },
       });
+
+      =========
   
       Add an OpenAI function that takes one or more nested parameters that match the expected output from this prompt.`,
   },

app/src/modelProviders/openpipe-chat/frontend.ts

@@ -0,0 +1,68 @@
+import { type OpenpipeChatOutput, type SupportedModel } from ".";
+import { type FrontendModelProvider } from "../types";
+import { refinementActions } from "./refinementActions";
+import {
+  templateOpenOrcaPrompt,
+  //   templateAlpacaInstructPrompt,
+  //   templateSystemUserAssistantPrompt,
+  templateInstructionInputResponsePrompt,
+  templateAiroborosPrompt,
+} from "./templatePrompt";
+
+const frontendModelProvider: FrontendModelProvider<SupportedModel, OpenpipeChatOutput> = {
+  name: "OpenAI ChatCompletion",
+
+  models: {
+    "Open-Orca/OpenOrcaxOpenChat-Preview2-13B": {
+      name: "OpenOrcaxOpenChat-Preview2-13B",
+      contextWindow: 4096,
+      pricePerSecond: 0.0003,
+      speed: "medium",
+      provider: "openpipe/Chat",
+      learnMoreUrl: "https://huggingface.co/Open-Orca/OpenOrcaxOpenChat-Preview2-13B",
+      templatePrompt: templateOpenOrcaPrompt,
+    },
+    // "Open-Orca/OpenOrca-Platypus2-13B": {
+    //   name: "OpenOrca-Platypus2-13B",
+    //   contextWindow: 4096,
+    //   pricePerSecond: 0.0003,
+    //   speed: "medium",
+    //   provider: "openpipe/Chat",
+    //   learnMoreUrl: "https://huggingface.co/Open-Orca/OpenOrca-Platypus2-13B",
+    //   templatePrompt: templateAlpacaInstructPrompt,
+    // },
+    // "stabilityai/StableBeluga-13B": {
+    //   name: "StableBeluga-13B",
+    //   contextWindow: 4096,
+    //   pricePerSecond: 0.0003,
+    //   speed: "medium",
+    //   provider: "openpipe/Chat",
+    //   learnMoreUrl: "https://huggingface.co/stabilityai/StableBeluga-13B",
+    //   templatePrompt: templateSystemUserAssistantPrompt,
+    // },
+    "NousResearch/Nous-Hermes-Llama2-13b": {
+      name: "Nous-Hermes-Llama2-13b",
+      contextWindow: 4096,
+      pricePerSecond: 0.0003,
+      speed: "medium",
+      provider: "openpipe/Chat",
+      learnMoreUrl: "https://huggingface.co/NousResearch/Nous-Hermes-Llama2-13b",
+      templatePrompt: templateInstructionInputResponsePrompt,
+    },
+    "jondurbin/airoboros-l2-13b-gpt4-2.0": {
+      name: "airoboros-l2-13b-gpt4-2.0",
+      contextWindow: 4096,
+      pricePerSecond: 0.0003,
+      speed: "medium",
+      provider: "openpipe/Chat",
+      learnMoreUrl: "https://huggingface.co/jondurbin/airoboros-l2-13b-gpt4-2.0",
+      templatePrompt: templateAiroborosPrompt,
+    },
+  },
+
+  refinementActions,
+
+  normalizeOutput: (output) => ({ type: "text", value: output }),
+};
+
+export default frontendModelProvider;

app/src/modelProviders/openpipe-chat/getCompletion.ts

@@ -0,0 +1,108 @@
+/* eslint-disable @typescript-eslint/no-unsafe-call */
+import { isArray, isString } from "lodash-es";
+import OpenAI, { APIError } from "openai";
+
+import { type CompletionResponse } from "../types";
+import { type OpenpipeChatInput, type OpenpipeChatOutput } from ".";
+import frontendModelProvider from "./frontend";
+
+const modelEndpoints: Record<OpenpipeChatInput["model"], string> = {
+  "Open-Orca/OpenOrcaxOpenChat-Preview2-13B": "https://5ef82gjxk8kdys-8000.proxy.runpod.net/v1",
+  // "Open-Orca/OpenOrca-Platypus2-13B": "https://lt5qlel6qcji8t-8000.proxy.runpod.net/v1",
+  // "stabilityai/StableBeluga-13B": "https://vcorl8mxni2ou1-8000.proxy.runpod.net/v1",
+  "NousResearch/Nous-Hermes-Llama2-13b": "https://ncv8pw3u0vb8j2-8000.proxy.runpod.net/v1",
+  "jondurbin/airoboros-l2-13b-gpt4-2.0": "https://9nrbx7oph4btou-8000.proxy.runpod.net/v1",
+};
+
+export async function getCompletion(
+  input: OpenpipeChatInput,
+  onStream: ((partialOutput: OpenpipeChatOutput) => void) | null,
+): Promise<CompletionResponse<OpenpipeChatOutput>> {
+  const { model, messages, ...rest } = input;
+
+  const templatedPrompt = frontendModelProvider.models[model].templatePrompt?.(messages);
+
+  if (!templatedPrompt) {
+    return {
+      type: "error",
+      message: "Failed to generate prompt",
+      autoRetry: false,
+    };
+  }
+
+  const openai = new OpenAI({
+    baseURL: modelEndpoints[model],
+  });
+  const start = Date.now();
+  let finalCompletion: OpenpipeChatOutput = "";
+
+  try {
+    if (onStream) {
+      const resp = await openai.completions.create(
+        { model, prompt: templatedPrompt, ...rest, stream: true },
+        {
+          maxRetries: 0,
+        },
+      );
+
+      for await (const part of resp) {
+        finalCompletion += part.choices[0]?.text;
+        onStream(finalCompletion);
+      }
+      if (!finalCompletion) {
+        return {
+          type: "error",
+          message: "Streaming failed to return a completion",
+          autoRetry: false,
+        };
+      }
+    } else {
+      const resp = await openai.completions.create(
+        { model, prompt: templatedPrompt, ...rest, stream: false },
+        {
+          maxRetries: 0,
+        },
+      );
+      finalCompletion = resp.choices[0]?.text || "";
+      if (!finalCompletion) {
+        return {
+          type: "error",
+          message: "Failed to return a completion",
+          autoRetry: false,
+        };
+      }
+    }
+    const timeToComplete = Date.now() - start;
+
+    return {
+      type: "success",
+      statusCode: 200,
+      value: finalCompletion,
+      timeToComplete,
+    };
+  } catch (error: unknown) {
+    if (error instanceof APIError) {
+      // The types from the sdk are wrong
+      const rawMessage = error.message as string | string[];
+      // If the message is not a string, stringify it
+      const message = isString(rawMessage)
+        ? rawMessage
+        : isArray(rawMessage)
+        ? rawMessage.map((m) => m.toString()).join("\n")
+        : (rawMessage as any).toString();
+      return {
+        type: "error",
+        message,
+        autoRetry: error.status === 429 || error.status === 503,
+        statusCode: error.status,
+      };
+    } else {
+      console.error(error);
+      return {
+        type: "error",
+        message: (error as Error).message,
+        autoRetry: true,
+      };
+    }
+  }
+}

app/src/modelProviders/openpipe-chat/index.ts

@@ -0,0 +1,51 @@
+import { type JSONSchema4 } from "json-schema";
+import { type ModelProvider } from "../types";
+import inputSchema from "./input.schema.json";
+import { getCompletion } from "./getCompletion";
+import frontendModelProvider from "./frontend";
+
+const supportedModels = [
+  "Open-Orca/OpenOrcaxOpenChat-Preview2-13B",
+  // "Open-Orca/OpenOrca-Platypus2-13B",
+  // "stabilityai/StableBeluga-13B",
+  "NousResearch/Nous-Hermes-Llama2-13b",
+  "jondurbin/airoboros-l2-13b-gpt4-2.0",
+] as const;
+
+export type SupportedModel = (typeof supportedModels)[number];
+
+export type OpenpipeChatInput = {
+  model: SupportedModel;
+  messages: {
+    role: "system" | "user" | "assistant";
+    content: string;
+  }[];
+  temperature?: number;
+  top_p?: number;
+  stop?: string[] | string;
+  max_tokens?: number;
+  presence_penalty?: number;
+  frequency_penalty?: number;
+};
+
+export type OpenpipeChatOutput = string;
+
+export type OpenpipeChatModelProvider = ModelProvider<
+  SupportedModel,
+  OpenpipeChatInput,
+  OpenpipeChatOutput
+>;
+
+const modelProvider: OpenpipeChatModelProvider = {
+  getModel: (input) => input.model,
+  inputSchema: inputSchema as JSONSchema4,
+  canStream: true,
+  getCompletion,
+  getUsage: (input, output) => {
+    // TODO: Implement this
+    return null;
+  },
+  ...frontendModelProvider,
+};
+
+export default modelProvider;

app/src/modelProviders/openpipe-chat/input.schema.json

@@ -0,0 +1,92 @@
+{
+  "type": "object",
+  "properties": {
+    "model": {
+      "description": "ID of the model to use.",
+      "example": "Open-Orca/OpenOrcaxOpenChat-Preview2-13B",
+      "type": "string",
+      "enum": [
+        "Open-Orca/OpenOrcaxOpenChat-Preview2-13B",
+        "NousResearch/Nous-Hermes-Llama2-13b",
+        "jondurbin/airoboros-l2-13b-gpt4-2.0"
+      ]
+    },
+    "messages": {
+      "description": "A list of messages comprising the conversation so far.",
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "type": "object",
+        "properties": {
+          "role": {
+            "type": "string",
+            "enum": ["system", "user", "assistant"],
+            "description": "The role of the messages author. One of `system`, `user`, or `assistant`."
+          },
+          "content": {
+            "type": "string",
+            "description": "The contents of the message. `content` is required for all messages."
+          }
+        },
+        "required": ["role", "content"]
+      }
+    },
+    "temperature": {
+      "type": "number",
+      "minimum": 0,
+      "maximum": 2,
+      "default": 1,
+      "example": 1,
+      "nullable": true,
+      "description": "What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.\n\nWe generally recommend altering this or `top_p` but not both.\n"
+    },
+    "top_p": {
+      "type": "number",
+      "minimum": 0,
+      "maximum": 1,
+      "default": 1,
+      "example": 1,
+      "nullable": true,
+      "description": "An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.\n\nWe generally recommend altering this or `temperature` but not both.\n"
+    },
+    "stop": {
+      "description": "Up to 4 sequences where the API will stop generating further tokens.\n",
+      "default": null,
+      "oneOf": [
+        {
+          "type": "string",
+          "nullable": true
+        },
+        {
+          "type": "array",
+          "minItems": 1,
+          "maxItems": 4,
+          "items": {
+            "type": "string"
+          }
+        }
+      ]
+    },
+    "max_tokens": {
+      "description": "The maximum number of [tokens](/tokenizer) to generate in the chat completion.\n\nThe total length of input tokens and generated tokens is limited by the model's context length. [Example Python code](https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb) for counting tokens.\n",
+      "type": "integer"
+    },
+    "presence_penalty": {
+      "type": "number",
+      "default": 0,
+      "minimum": -2,
+      "maximum": 2,
+      "nullable": true,
+      "description": "Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.\n\n[See more information about frequency and presence penalties.](/docs/api-reference/parameter-details)\n"
+    },
+    "frequency_penalty": {
+      "type": "number",
+      "default": 0,
+      "minimum": -2,
+      "maximum": 2,
+      "nullable": true,
+      "description": "Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.\n\n[See more information about frequency and presence penalties.](/docs/api-reference/parameter-details)\n"
+    }
+  },
+  "required": ["model", "messages"]
+}

app/src/modelProviders/openpipe-chat/refinementActions.ts

@@ -0,0 +1,3 @@
+import { type RefinementAction } from "../types";
+
+export const refinementActions: Record<string, RefinementAction> = {};

app/src/modelProviders/openpipe-chat/templatePrompt.ts

@@ -0,0 +1,181 @@
+import { type OpenpipeChatInput } from ".";
+
+// User: Hello<|end_of_turn|>Assistant: Hi<|end_of_turn|>User: How are you today?<|end_of_turn|>Assistant:
+export const templateOpenOrcaPrompt = (messages: OpenpipeChatInput["messages"]) => {
+  const splitter = "<|end_of_turn|>";
+
+  const formattedMessages = messages.map((message) => {
+    if (message.role === "system" || message.role === "user") {
+      return "User: " + message.content;
+    } else {
+      return "Assistant: " + message.content;
+    }
+  });
+
+  let prompt = formattedMessages.join(splitter);
+
+  // Ensure that the prompt ends with an assistant message
+  const lastUserIndex = prompt.lastIndexOf("User:");
+  const lastAssistantIndex = prompt.lastIndexOf("Assistant:");
+  if (lastUserIndex > lastAssistantIndex) {
+    prompt += splitter + "Assistant:";
+  }
+
+  return prompt;
+};
+
+// ### Instruction:
+
+// <prompt> (without the <>)
+
+// ### Response:
+export const templateAlpacaInstructPrompt = (messages: OpenpipeChatInput["messages"]) => {
+  const splitter = "\n\n";
+
+  const userTag = "### Instruction:\n\n";
+  const assistantTag = "### Response: \n\n";
+
+  const formattedMessages = messages.map((message) => {
+    if (message.role === "system" || message.role === "user") {
+      return userTag + message.content;
+    } else {
+      return assistantTag + message.content;
+    }
+  });
+
+  let prompt = formattedMessages.join(splitter);
+
+  // Ensure that the prompt ends with an assistant message
+  const lastUserIndex = prompt.lastIndexOf(userTag);
+  const lastAssistantIndex = prompt.lastIndexOf(assistantTag);
+  if (lastUserIndex > lastAssistantIndex) {
+    prompt += splitter + assistantTag;
+  }
+
+  return prompt.trim();
+};
+
+// ### System:
+// This is a system prompt, please behave and help the user.
+
+// ### User:
+// Your prompt here
+
+// ### Assistant
+// The output of Stable Beluga 13B
+export const templateSystemUserAssistantPrompt = (messages: OpenpipeChatInput["messages"]) => {
+  const splitter = "\n\n";
+
+  const systemTag = "### System:\n";
+  const userTag = "### User:\n";
+  const assistantTag = "### Assistant\n";
+
+  const formattedMessages = messages.map((message) => {
+    if (message.role === "system") {
+      return systemTag + message.content;
+    } else if (message.role === "user") {
+      return userTag + message.content;
+    } else {
+      return assistantTag + message.content;
+    }
+  });
+
+  let prompt = formattedMessages.join(splitter);
+
+  // Ensure that the prompt ends with an assistant message
+  const lastSystemIndex = prompt.lastIndexOf(systemTag);
+  const lastUserIndex = prompt.lastIndexOf(userTag);
+  const lastAssistantIndex = prompt.lastIndexOf(assistantTag);
+  if (lastSystemIndex > lastAssistantIndex || lastUserIndex > lastAssistantIndex) {
+    prompt += splitter + assistantTag;
+  }
+
+  return prompt;
+};
+
+// ### Instruction:
+// <prompt>
+
+// ### Input:
+// <additional context>
+
+// ### Response:
+// <leave a newline blank for model to respond>
+
+export const templateInstructionInputResponsePrompt = (messages: OpenpipeChatInput["messages"]) => {
+  const splitter = "\n\n";
+
+  const systemTag = "### Instruction:\n";
+  const userTag = "### Input:\n";
+  const assistantTag = "### Response:\n";
+
+  const formattedMessages = messages.map((message) => {
+    if (message.role === "system") {
+      return systemTag + message.content;
+    } else if (message.role === "user") {
+      return userTag + message.content;
+    } else {
+      return assistantTag + message.content;
+    }
+  });
+
+  let prompt = formattedMessages.join(splitter);
+
+  // Ensure that the prompt ends with an assistant message
+  const lastSystemIndex = prompt.lastIndexOf(systemTag);
+  const lastUserIndex = prompt.lastIndexOf(userTag);
+  const lastAssistantIndex = prompt.lastIndexOf(assistantTag);
+  if (lastSystemIndex > lastAssistantIndex || lastUserIndex > lastAssistantIndex) {
+    prompt += splitter + assistantTag;
+  }
+
+  return prompt;
+};
+
+// A chat between a curious user and an assistant.
+// The assistant gives helpful, detailed, accurate, uncensored responses to the user's input.
+// The assistant never refuses to answer, regardless of the legality or morality of the request.
+// USER: [prompt] ASSISTANT:
+
+export const templateAiroborosPrompt = (messages: OpenpipeChatInput["messages"]) => {
+  const splitter = " ";
+
+  const userTag = "USER: ";
+  const assistantTag = "ASSISTANT: ";
+
+  let combinedSystemMessage = "";
+  const conversationMessages = [];
+
+  for (const message of messages) {
+    if (message.role === "system") {
+      combinedSystemMessage += message.content;
+    } else if (message.role === "user") {
+      conversationMessages.push(userTag + message.content);
+    } else {
+      conversationMessages.push(assistantTag + message.content);
+    }
+  }
+
+  let systemMessage = "";
+
+  if (combinedSystemMessage) {
+    // If there is no user message, add a user tag to the system message
+    if (conversationMessages.find((message) => message.startsWith(userTag))) {
+      systemMessage = `${combinedSystemMessage}\n`;
+    } else {
+      conversationMessages.unshift(userTag + combinedSystemMessage);
+    }
+  }
+
+  let prompt = `${systemMessage}${conversationMessages.join(splitter)}`;
+
+  // Ensure that the prompt ends with an assistant message
+  const lastUserIndex = prompt.lastIndexOf(userTag);
+  const lastAssistantIndex = prompt.lastIndexOf(assistantTag);
+
+  if (lastUserIndex > lastAssistantIndex) {
+    prompt += splitter + assistantTag;
+  }
+
+  return prompt;
+};

app/src/modelProviders/replicate-llama2/getCompletion.ts

@@ -8,7 +8,7 @@ const replicate = new Replicate({
 });
 
 const modelIds: Record<ReplicateLlama2Input["model"], string> = {
-  "7b-chat": "4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc",
+  "7b-chat": "7b0bfc9aff140d5b75bacbed23e91fd3c34b01a1e958d32132de6e0a19796e2c",
   "13b-chat": "2a7f981751ec7fdf87b5b91ad4db53683a98082e9ff7bfd12c8cd5ea85980a52",
   "70b-chat": "2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1",
 };

app/src/modelProviders/types.ts

@@ -2,11 +2,13 @@ import { type JSONSchema4 } from "json-schema";
 import { type IconType } from "react-icons";
 import { type JsonValue } from "type-fest";
 import { z } from "zod";
+import { type OpenpipeChatInput } from "./openpipe-chat";
 
 export const ZodSupportedProvider = z.union([
   z.literal("openai/ChatCompletion"),
   z.literal("replicate/llama2"),
   z.literal("anthropic/completion"),
+  z.literal("openpipe/Chat"),
 ]);
 
 export type SupportedProvider = z.infer<typeof ZodSupportedProvider>;
@@ -22,6 +24,7 @@ export type Model = {
   description?: string;
   learnMoreUrl?: string;
   apiDocsUrl?: string;
+  templatePrompt?: (initialPrompt: OpenpipeChatInput["messages"]) => string;
 };
 
 export type ProviderModel = { provider: z.infer<typeof ZodSupportedProvider>; model: string };

app/src/pages/api/sentry-example-api.js

@@ -1,6 +0,0 @@
-// A faulty API route to test Sentry's error monitoring
-// @ts-expect-error just a test file, don't care about types
-export default function handler(_req, res) {
-  throw new Error("Sentry Example API Route Error");
-  res.status(200).json({ name: "John Doe" });
-}

app/src/pages/api/v1/[...trpc].ts

@@ -1,17 +1,14 @@
 import { type NextApiRequest, type NextApiResponse } from "next";
 import cors from "nextjs-cors";
 import { createOpenApiNextHandler } from "trpc-openapi";
-import { createProcedureCache } from "trpc-openapi/dist/adapters/node-http/procedures";
-import { appRouter } from "~/server/api/root.router";
-import { createTRPCContext } from "~/server/api/trpc";
+import { v1ApiRouter } from "~/server/api/external/v1Api.router";
+import { createOpenApiContext } from "~/server/api/external/openApiTrpc";
 
 const openApiHandler = createOpenApiNextHandler({
-  router: appRouter,
-  createContext: createTRPCContext,
+  router: v1ApiRouter,
+  createContext: createOpenApiContext,
 });
 
-const cache = createProcedureCache(appRouter);
-
 const handler = async (req: NextApiRequest, res: NextApiResponse) => {
   // Setup CORS
   await cors(req, res);

app/src/pages/api/v1/openapi.json.ts

@@ -1,12 +1,12 @@
 import { type NextApiRequest, type NextApiResponse } from "next";
 import { generateOpenApiDocument } from "trpc-openapi";
-import { appRouter } from "~/server/api/root.router";
+import { v1ApiRouter } from "~/server/api/external/v1Api.router";
 
-export const openApiDocument = generateOpenApiDocument(appRouter, {
+export const openApiDocument = generateOpenApiDocument(v1ApiRouter, {
   title: "OpenPipe API",
   description: "The public API for reporting API calls to OpenPipe",
-  version: "0.1.0",
-  baseUrl: "https://app.openpipe.ai/api",
+  version: "0.1.1",
+  baseUrl: "https://app.openpipe.ai/api/v1",
 });
 // Respond with our OpenAPI schema
 const hander = (req: NextApiRequest, res: NextApiResponse) => {

app/src/pages/experiments/[id].tsx

@@ -26,26 +26,6 @@ import Head from "next/head";
 import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
 import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
 
-// TODO: import less to fix deployment with server side props
-// export const getServerSideProps = async (context: GetServerSidePropsContext<{ id: string }>) => {
-//   const experimentId = context.params?.id as string;
-
-//   const helpers = createServerSideHelpers({
-//     router: appRouter,
-//     ctx: createInnerTRPCContext({ session: null }),
-//     transformer: superjson, // optional - adds superjson serialization
-//   });
-
-//   // prefetch query
-//   await helpers.experiments.stats.prefetch({ id: experimentId });
-
-//   return {
-//     props: {
-//       trpcState: helpers.dehydrate(),
-//     },
-//   };
-// };
-
 export default function Experiment() {
   const router = useRouter();
   const utils = api.useContext();

app/src/pages/invitations/[invitationToken].tsx

@@ -0,0 +1,110 @@
+import { Center, Text, VStack, HStack, Button, Card } from "@chakra-ui/react";
+
+import { useRouter } from "next/router";
+import AppShell from "~/components/nav/AppShell";
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback } from "~/utils/hooks";
+import { useAppStore } from "~/state/store";
+import { useSyncVariantEditor } from "~/state/sync";
+import { maybeReportError } from "~/utils/errorHandling/maybeReportError";
+
+export default function Invitation() {
+  const router = useRouter();
+  const utils = api.useContext();
+  useSyncVariantEditor();
+
+  const setSelectedProjectId = useAppStore((state) => state.setSelectedProjectId);
+
+  const invitationToken = router.query.invitationToken as string | undefined;
+
+  const invitation = api.users.getProjectInvitation.useQuery(
+    { invitationToken: invitationToken as string },
+    { enabled: !!invitationToken },
+  );
+
+  const cancelMutation = api.users.cancelProjectInvitation.useMutation();
+  const [declineInvitation, isDeclining] = useHandledAsyncCallback(async () => {
+    if (invitationToken) {
+      await cancelMutation.mutateAsync({
+        invitationToken,
+      });
+      await router.replace("/");
+    }
+  }, [cancelMutation, invitationToken]);
+
+  const acceptMutation = api.users.acceptProjectInvitation.useMutation();
+  const [acceptInvitation, isAccepting] = useHandledAsyncCallback(async () => {
+    if (invitationToken) {
+      const resp = await acceptMutation.mutateAsync({
+        invitationToken,
+      });
+      if (!maybeReportError(resp) && resp) {
+        await utils.projects.list.invalidate();
+        setSelectedProjectId(resp.payload);
+      }
+      await router.replace("/");
+    }
+  }, [acceptMutation, invitationToken]);
+
+  if (invitation.isLoading) {
+    return (
+      <AppShell requireAuth title="Loading...">
+        <Center h="full">
+          <Text>Loading...</Text>
+        </Center>
+      </AppShell>
+    );
+  }
+
+  if (!invitationToken || !invitation.data) {
+    return (
+      <AppShell requireAuth title="Invalid invitation token">
+        <Center h="full">
+          <Text>
+            The invitation you've received is invalid or expired. Please ask your project admin for
+            a new token.
+          </Text>
+        </Center>
+      </AppShell>
+    );
+  }
+
+  return (
+    <>
+      <AppShell requireAuth title="Invitation">
+        <Center h="full">
+          <Card>
+            <VStack
+              spacing={8}
+              w="full"
+              maxW="2xl"
+              p={16}
+              borderWidth={1}
+              borderRadius={8}
+              bgColor="white"
+            >
+              <Text fontSize="lg" fontWeight="bold">
+                You're invited! 🎉
+              </Text>
+              <Text textAlign="center">
+                You've been invited to join <b>{invitation.data.project.name}</b> by{" "}
+                <b>
+                  {invitation.data.sender.name} ({invitation.data.sender.email})
+                </b>
+                .
+              </Text>
+              <HStack spacing={4}>
+                <Button colorScheme="gray" isLoading={isDeclining} onClick={declineInvitation}>
+                  Decline
+                </Button>
+                <Button colorScheme="orange" isLoading={isAccepting} onClick={acceptInvitation}>
+                  Accept
+                </Button>
+              </HStack>
+            </VStack>
+          </Card>
+        </Center>
+      </AppShell>
+    </>
+  );
+}

app/src/pages/project/settings/index.tsx

@@ -9,9 +9,11 @@ import {
   Divider,
   Icon,
   useDisclosure,
+  Box,
+  Tooltip,
 } from "@chakra-ui/react";
 import { useEffect, useState } from "react";
-import { BsTrash } from "react-icons/bs";
+import { BsPlus, BsTrash } from "react-icons/bs";
 
 import AppShell from "~/components/nav/AppShell";
 import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
@@ -21,6 +23,8 @@ import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContent
 import CopiableCode from "~/components/CopiableCode";
 import { DeleteProjectDialog } from "~/components/projectSettings/DeleteProjectDialog";
 import AutoResizeTextArea from "~/components/AutoResizeTextArea";
+import MemberTable from "~/components/projectSettings/MemberTable";
+import { InviteMemberModal } from "~/components/projectSettings/InviteMemberModal";
 
 export default function Settings() {
   const utils = api.useContext();
@@ -50,12 +54,13 @@ export default function Settings() {
     setName(selectedProject?.name);
   }, [selectedProject?.name]);
 
-  const deleteProjectOpen = useDisclosure();
+  const inviteMemberModal = useDisclosure();
+  const deleteProjectDialog = useDisclosure();
 
   return (
     <>
-      <AppShell>
-        <PageHeaderContainer>
+      <AppShell requireAuth>
+        <PageHeaderContainer px={{ base: 4, md: 8 }}>
           <Breadcrumb>
             <BreadcrumbItem>
               <ProjectBreadcrumbContents />
@@ -65,7 +70,7 @@ export default function Settings() {
             </BreadcrumbItem>
           </Breadcrumb>
         </PageHeaderContainer>
-        <VStack px={8} py={4} alignItems="flex-start" spacing={4}>
+        <VStack px={{ base: 4, md: 8 }} py={4} alignItems="flex-start" spacing={4}>
           <VStack spacing={0} alignItems="flex-start">
             <Text fontSize="2xl" fontWeight="bold">
               Project Settings
@@ -109,6 +114,37 @@ export default function Settings() {
               </Button>
             </VStack>
             <Divider backgroundColor="gray.300" />
+            <VStack w="full" alignItems="flex-start">
+              <Subtitle>Project Members</Subtitle>
+
+              <Text fontSize="sm">
+                Add members to your project to allow them to view and edit your project's data.
+              </Text>
+              <Box mt={4} w="full">
+                <MemberTable />
+              </Box>
+              <Tooltip
+                isDisabled={selectedProject?.role === "ADMIN"}
+                label="Only admins can invite new members"
+                hasArrow
+              >
+                <Button
+                  variant="outline"
+                  colorScheme="orange"
+                  borderRadius={4}
+                  onClick={inviteMemberModal.onOpen}
+                  mt={2}
+                  _disabled={{
+                    opacity: 0.6,
+                  }}
+                  isDisabled={selectedProject?.role !== "ADMIN"}
+                >
+                  <Icon as={BsPlus} boxSize={5} />
+                  <Text>Invite New Member</Text>
+                </Button>
+              </Tooltip>
+            </VStack>
+            <Divider backgroundColor="gray.300" />
             <VStack alignItems="flex-start">
               <Subtitle>Project API Key</Subtitle>
               <Text fontSize="sm">
@@ -141,7 +177,7 @@ export default function Settings() {
                   borderRadius={4}
                   mt={2}
                   height="auto"
-                  onClick={deleteProjectOpen.onOpen}
+                  onClick={deleteProjectDialog.onOpen}
                 >
                   <Icon as={BsTrash} />
                   <Text overflowWrap="break-word" whiteSpace="normal" py={2}>
@@ -153,7 +189,11 @@ export default function Settings() {
           </VStack>
         </VStack>
       </AppShell>
-      <DeleteProjectDialog isOpen={deleteProjectOpen.isOpen} onClose={deleteProjectOpen.onClose} />
+      <InviteMemberModal isOpen={inviteMemberModal.isOpen} onClose={inviteMemberModal.onClose} />
+      <DeleteProjectDialog
+        isOpen={deleteProjectDialog.isOpen}
+        onClose={deleteProjectDialog.onClose}
+      />
     </>
   );
 }

app/src/pages/request-logs/index.tsx

@@ -1,3 +1,4 @@
+import { useState } from "react";
 import { Text, VStack, Divider, HStack } from "@chakra-ui/react";
 
 import AppShell from "~/components/nav/AppShell";
@@ -6,9 +7,14 @@ import LoggedCallsPaginator from "~/components/requestLogs/LoggedCallsPaginator"
 import ActionButton from "~/components/requestLogs/ActionButton";
 import { useAppStore } from "~/state/store";
 import { RiFlaskLine } from "react-icons/ri";
+import { FiFilter } from "react-icons/fi";
+import LogFilters from "~/components/requestLogs/LogFilters/LogFilters";
 
 export default function LoggedCalls() {
   const selectedLogIds = useAppStore((s) => s.selectedLogs.selectedLogIds);
+
+  const [filtersShown, setFiltersShown] = useState(true);
+
   return (
     <AppShell title="Request Logs" requireAuth>
       <VStack px={8} py={8} alignItems="flex-start" spacing={4} w="full">
@@ -17,6 +23,13 @@ export default function LoggedCalls() {
         </Text>
         <Divider />
         <HStack w="full" justifyContent="flex-end">
+          <ActionButton
+            onClick={() => {
+              setFiltersShown(!filtersShown);
+            }}
+            label={filtersShown ? "Hide Filters" : "Show Filters"}
+            icon={FiFilter}
+          />
           <ActionButton
             onClick={() => {
               console.log("experimenting with these ids", selectedLogIds);
@@ -26,6 +39,7 @@ export default function LoggedCalls() {
             isDisabled={selectedLogIds.size === 0}
           />
         </HStack>
+        {filtersShown && <LogFilters />}
         <LoggedCallTable />
         <LoggedCallsPaginator />
       </VStack>

app/src/server/api/external/openApiTrpc.ts

@@ -0,0 +1,95 @@
+import type { ApiKey, Project } from "@prisma/client";
+import { TRPCError, initTRPC } from "@trpc/server";
+import { type CreateNextContextOptions } from "@trpc/server/adapters/next";
+import superjson from "superjson";
+import { type OpenApiMeta } from "trpc-openapi";
+import { ZodError } from "zod";
+import { prisma } from "~/server/db";
+
+type CreateContextOptions = {
+  key:
+    | (ApiKey & {
+        project: Project;
+      })
+    | null;
+};
+
+/**
+ * This helper generates the "internals" for a tRPC context. If you need to use it, you can export
+ * it from here.
+ *
+ * Examples of things you may need it for:
+ * - testing, so we don't have to mock Next.js' req/res
+ * - tRPC's `createSSGHelpers`, where we don't have req/res
+ *
+ * @see https://create.t3.gg/en/usage/trpc#-serverapitrpcts
+ */
+export const createInnerTRPCContext = (opts: CreateContextOptions) => {
+  return {
+    key: opts.key,
+  };
+};
+
+export const createOpenApiContext = async (opts: CreateNextContextOptions) => {
+  const { req, res } = opts;
+
+  const apiKey = req.headers.authorization?.split(" ")[1] as string | null;
+
+  if (!apiKey) {
+    throw new TRPCError({ code: "UNAUTHORIZED" });
+  }
+  const key = await prisma.apiKey.findUnique({
+    where: { apiKey },
+    include: { project: true },
+  });
+  if (!key) {
+    throw new TRPCError({ code: "UNAUTHORIZED" });
+  }
+
+  return createInnerTRPCContext({
+    key,
+  });
+};
+
+export type TRPCContext = Awaited<ReturnType<typeof createOpenApiContext>>;
+
+const t = initTRPC
+  .context<typeof createOpenApiContext>()
+  .meta<OpenApiMeta>()
+  .create({
+    transformer: superjson,
+    errorFormatter({ shape, error }) {
+      return {
+        ...shape,
+        data: {
+          ...shape.data,
+          zodError: error.cause instanceof ZodError ? error.cause.flatten() : null,
+        },
+      };
+    },
+  });
+
+export const createOpenApiRouter = t.router;
+
+export const openApiPublicProc = t.procedure;
+
+/** Reusable middleware that enforces users are logged in before running the procedure. */
+const enforceApiKey = t.middleware(async ({ ctx, next }) => {
+  if (!ctx.key) {
+    throw new TRPCError({ code: "UNAUTHORIZED" });
+  }
+
+  return next({
+    ctx: { key: ctx.key },
+  });
+});
+
+/**
+ * Protected (authenticated) procedure
+ *
+ * If you want a query or mutation to ONLY be accessible to logged in users, use this. It verifies
+ * the session is valid and guarantees `ctx.session.user` is not null.
+ *
+ * @see https://trpc.io/docs/procedures
+ */
+export const openApiProtectedProc = t.procedure.use(enforceApiKey);

app/src/server/api/external/v1Api.router.ts

@@ -2,9 +2,6 @@ import { type Prisma } from "@prisma/client";
 import { type JsonValue } from "type-fest";
 import { z } from "zod";
 import { v4 as uuidv4 } from "uuid";
-import { TRPCError } from "@trpc/server";
-
-import { createTRPCRouter, publicProcedure } from "~/server/api/trpc";
 import { prisma } from "~/server/db";
 import { hashRequest } from "~/server/utils/hashObject";
 import modelProvider from "~/modelProviders/openai-ChatCompletion";
@@ -12,6 +9,7 @@ import {
   type ChatCompletion,
   type CompletionCreateParams,
 } from "openai/resources/chat/completions";
+import { createOpenApiRouter, openApiProtectedProc } from "./openApiTrpc";
 
 const reqValidator = z.object({
   model: z.string(),
@@ -28,12 +26,12 @@ const respValidator = z.object({
   ),
 });
 
-export const externalApiRouter = createTRPCRouter({
-  checkCache: publicProcedure
+export const v1ApiRouter = createOpenApiRouter({
+  checkCache: openApiProtectedProc
     .meta({
       openapi: {
         method: "POST",
-        path: "/v1/check-cache",
+        path: "/check-cache",
         description: "Check if a prompt is cached",
         protect: true,
       },
@@ -47,7 +45,8 @@ export const externalApiRouter = createTRPCRouter({
           .optional()
           .describe(
             'Extra tags to attach to the call for filtering. Eg { "userId": "123", "promptId": "populate-title" }',
-          ),
+          )
+          .default({}),
       }),
     )
     .output(
@@ -56,18 +55,8 @@ export const externalApiRouter = createTRPCRouter({
       }),
     )
     .mutation(async ({ input, ctx }) => {
-      const apiKey = ctx.apiKey;
-      if (!apiKey) {
-        throw new TRPCError({ code: "UNAUTHORIZED" });
-      }
-      const key = await prisma.apiKey.findUnique({
-        where: { apiKey },
-      });
-      if (!key) {
-        throw new TRPCError({ code: "UNAUTHORIZED" });
-      }
       const reqPayload = await reqValidator.spa(input.reqPayload);
-      const cacheKey = hashRequest(key.projectId, reqPayload as JsonValue);
+      const cacheKey = hashRequest(ctx.key.projectId, reqPayload as JsonValue);
 
       const existingResponse = await prisma.loggedCallModelResponse.findFirst({
         where: { cacheKey },
@@ -79,23 +68,28 @@ export const externalApiRouter = createTRPCRouter({
 
       await prisma.loggedCall.create({
         data: {
-          projectId: key.projectId,
+          projectId: ctx.key.projectId,
           requestedAt: new Date(input.requestedAt),
           cacheHit: true,
           modelResponseId: existingResponse.id,
         },
       });
 
+      await createTags(
+        existingResponse.originalLoggedCall.projectId,
+        existingResponse.originalLoggedCallId,
+        input.tags,
+      );
       return {
         respPayload: existingResponse.respPayload,
       };
     }),
 
-  report: publicProcedure
+  report: openApiProtectedProc
     .meta({
       openapi: {
         method: "POST",
-        path: "/v1/report",
+        path: "/report",
         description: "Report an API call",
         protect: true,
       },
@@ -113,26 +107,16 @@ export const externalApiRouter = createTRPCRouter({
           .optional()
           .describe(
             'Extra tags to attach to the call for filtering. Eg { "userId": "123", "promptId": "populate-title" }',
-          ),
+          )
+          .default({}),
       }),
     )
-    .output(z.void())
+    .output(z.object({ status: z.literal("ok") }))
     .mutation(async ({ input, ctx }) => {
-      console.log("GOT TAGS", input.tags);
-      const apiKey = ctx.apiKey;
-      if (!apiKey) {
-        throw new TRPCError({ code: "UNAUTHORIZED" });
-      }
-      const key = await prisma.apiKey.findUnique({
-        where: { apiKey },
-      });
-      if (!key) {
-        throw new TRPCError({ code: "UNAUTHORIZED" });
-      }
       const reqPayload = await reqValidator.spa(input.reqPayload);
       const respPayload = await respValidator.spa(input.respPayload);
 
-      const requestHash = hashRequest(key.projectId, reqPayload as JsonValue);
+      const requestHash = hashRequest(ctx.key.projectId, reqPayload as JsonValue);
 
       const newLoggedCallId = uuidv4();
       const newModelResponseId = uuidv4();
@@ -151,7 +135,7 @@ export const externalApiRouter = createTRPCRouter({
         prisma.loggedCall.create({
           data: {
             id: newLoggedCallId,
-            projectId: key.projectId,
+            projectId: ctx.key.projectId,
             requestedAt: new Date(input.requestedAt),
             cacheHit: false,
             model,
@@ -185,14 +169,78 @@ export const externalApiRouter = createTRPCRouter({
         }),
       ]);
 
-      const tagsToCreate = Object.entries(input.tags ?? {}).map(([name, value]) => ({
-        loggedCallId: newLoggedCallId,
-        // sanitize tags
-        name: name.replaceAll(/[^a-zA-Z0-9_]/g, "_"),
-        value,
-      }));
-      await prisma.loggedCallTag.createMany({
-        data: tagsToCreate,
+      await createTags(ctx.key.projectId, newLoggedCallId, input.tags);
+      return { status: "ok" };
+    }),
+  localTestingOnlyGetLatestLoggedCall: openApiProtectedProc
+    .meta({
+      openapi: {
+        method: "GET",
+        path: "/local-testing-only-get-latest-logged-call",
+        description: "Get the latest logged call (only for local testing)",
+        protect: true, // Make sure to protect this endpoint
+      },
+    })
+    .input(z.void())
+    .output(
+      z
+        .object({
+          createdAt: z.date(),
+          cacheHit: z.boolean(),
+          tags: z.record(z.string().nullable()),
+          modelResponse: z
+            .object({
+              id: z.string(),
+              statusCode: z.number().nullable(),
+              errorMessage: z.string().nullable(),
+              reqPayload: z.unknown(),
+              respPayload: z.unknown(),
+            })
+            .nullable(),
+        })
+        .nullable(),
+    )
+    .mutation(async ({ ctx }) => {
+      if (process.env.NODE_ENV === "production") {
+        throw new Error("This operation is not allowed in production environment");
+      }
+
+      const latestLoggedCall = await prisma.loggedCall.findFirst({
+        where: { projectId: ctx.key.projectId },
+        orderBy: { requestedAt: "desc" },
+        select: {
+          createdAt: true,
+          cacheHit: true,
+          tags: true,
+          modelResponse: {
+            select: {
+              id: true,
+              statusCode: true,
+              errorMessage: true,
+              reqPayload: true,
+              respPayload: true,
+            },
+          },
+        },
       });
+
+      return (
+        latestLoggedCall && {
+          ...latestLoggedCall,
+          tags: Object.fromEntries(latestLoggedCall.tags.map((tag) => [tag.name, tag.value])),
+        }
+      );
     }),
 });
+
+async function createTags(projectId: string, loggedCallId: string, tags: Record<string, string>) {
+  const tagsToCreate = Object.entries(tags).map(([name, value]) => ({
+    projectId,
+    loggedCallId,
+    name: name.replaceAll(/[^a-zA-Z0-9_$]/g, "_"),
+    value,
+  }));
+  await prisma.loggedCallTag.createMany({
+    data: tagsToCreate,
+  });
+}

app/src/server/api/root.router.ts

@@ -8,10 +8,10 @@ import { evaluationsRouter } from "./routers/evaluations.router";
 import { worldChampsRouter } from "./routers/worldChamps.router";
 import { datasetsRouter } from "./routers/datasets.router";
 import { datasetEntries } from "./routers/datasetEntries.router";
-import { externalApiRouter } from "./routers/externalApi.router";
 import { projectsRouter } from "./routers/projects.router";
 import { dashboardRouter } from "./routers/dashboard.router";
 import { loggedCallsRouter } from "./routers/loggedCalls.router";
+import { usersRouter } from "./routers/users.router";
 
 /**
  * This is the primary router for your server.
@@ -31,7 +31,7 @@ export const appRouter = createTRPCRouter({
   projects: projectsRouter,
   dashboard: dashboardRouter,
   loggedCalls: loggedCallsRouter,
-  externalApi: externalApiRouter,
+  users: usersRouter,
 });
 
 // export type definition of API

app/src/server/api/routers/loggedCalls.router.ts

@@ -1,33 +1,183 @@
 import { z } from "zod";
+import { type Expression, type SqlBool, sql, type RawBuilder } from "kysely";
+import { jsonArrayFrom } from "kysely/helpers/postgres";
+
 import { createTRPCRouter, protectedProcedure } from "~/server/api/trpc";
-import { prisma } from "~/server/db";
+import { kysely, prisma } from "~/server/db";
+import { comparators, defaultFilterableFields } from "~/state/logFiltersSlice";
 import { requireCanViewProject } from "~/utils/accessControl";
 
+// create comparator type based off of comparators
+const comparatorToSqlExpression = (comparator: (typeof comparators)[number], value: string) => {
+  return (reference: RawBuilder<unknown>): Expression<SqlBool> => {
+    switch (comparator) {
+      case "=":
+        return sql`${reference} = ${value}`;
+      case "!=":
+        // Handle NULL values
+        return sql`${reference} IS DISTINCT FROM ${value}`;
+      case "CONTAINS":
+        return sql`${reference} LIKE ${"%" + value + "%"}`;
+      case "NOT_CONTAINS":
+        return sql`(${reference} NOT LIKE ${"%" + value + "%"} OR ${reference} IS NULL)`;
+      default:
+        throw new Error("Unknown comparator");
+    }
+  };
+};
+
 export const loggedCallsRouter = createTRPCRouter({
   list: protectedProcedure
-    .input(z.object({ projectId: z.string(), page: z.number(), pageSize: z.number() }))
+    .input(
+      z.object({
+        projectId: z.string(),
+        page: z.number(),
+        pageSize: z.number(),
+        filters: z.array(
+          z.object({
+            field: z.string(),
+            comparator: z.enum(comparators),
+            value: z.string(),
+          }),
+        ),
+      }),
+    )
     .query(async ({ input, ctx }) => {
       const { projectId, page, pageSize } = input;
 
       await requireCanViewProject(projectId, ctx);
 
-      const calls = await prisma.loggedCall.findMany({
-        where: { projectId },
-        orderBy: { requestedAt: "desc" },
-        include: { tags: true, modelResponse: true },
-        skip: (page - 1) * pageSize,
-        take: pageSize,
+      const baseQuery = kysely
+        .selectFrom("LoggedCall as lc")
+        .leftJoin("LoggedCallModelResponse as lcmr", "lc.id", "lcmr.originalLoggedCallId")
+        .where((eb) => {
+          const wheres: Expression<SqlBool>[] = [eb("lc.projectId", "=", projectId)];
+
+          for (const filter of input.filters) {
+            if (!filter.value) continue;
+            const filterExpression = comparatorToSqlExpression(filter.comparator, filter.value);
+
+            if (filter.field === "Request") {
+              wheres.push(filterExpression(sql.raw(`lcmr."reqPayload"::text`)));
+            }
+            if (filter.field === "Response") {
+              wheres.push(filterExpression(sql.raw(`lcmr."respPayload"::text`)));
+            }
+            if (filter.field === "Model") {
+              wheres.push(filterExpression(sql.raw(`lc."model"`)));
+            }
+            if (filter.field === "Status Code") {
+              wheres.push(filterExpression(sql.raw(`lcmr."statusCode"::text`)));
+            }
+          }
+
+          return eb.and(wheres);
+        });
+
+      const tagFilters = input.filters.filter(
+        (filter) =>
+          !defaultFilterableFields.includes(
+            filter.field as (typeof defaultFilterableFields)[number],
+          ),
+      );
+
+      let updatedBaseQuery = baseQuery;
+
+      for (let i = 0; i < tagFilters.length; i++) {
+        const filter = tagFilters[i];
+        if (!filter?.value) continue;
+        const tableAlias = `lct${i}`;
+        const filterExpression = comparatorToSqlExpression(filter.comparator, filter.value);
+
+        updatedBaseQuery = updatedBaseQuery
+          .leftJoin(`LoggedCallTag as ${tableAlias}`, (join) =>
+            join
+              .onRef("lc.id", "=", `${tableAlias}.loggedCallId`)
+              .on(`${tableAlias}.name`, "=", filter.field),
+          )
+          .where(filterExpression(sql.raw(`${tableAlias}.value`))) as unknown as typeof baseQuery;
+      }
+
+      const rawCalls = await updatedBaseQuery
+        .select((eb) => [
+          "lc.id as id",
+          "lc.requestedAt as requestedAt",
+          "model",
+          "cacheHit",
+          "lc.requestedAt",
+          "receivedAt",
+          "reqPayload",
+          "respPayload",
+          "model",
+          "inputTokens",
+          "outputTokens",
+          "cost",
+          "statusCode",
+          "durationMs",
+          jsonArrayFrom(
+            eb
+              .selectFrom("LoggedCallTag")
+              .select(["name", "value"])
+              .whereRef("loggedCallId", "=", "lc.id"),
+          ).as("tags"),
+        ])
+        .orderBy("lc.requestedAt", "desc")
+        .limit(pageSize)
+        .offset((page - 1) * pageSize)
+        .execute();
+
+      const calls = rawCalls.map((rawCall) => {
+        const tagsObject = rawCall.tags.reduce(
+          (acc, tag) => {
+            acc[tag.name] = tag.value;
+            return acc;
+          },
+          {} as Record<string, string | null>,
+        );
+
+        return {
+          id: rawCall.id,
+          requestedAt: rawCall.requestedAt,
+          model: rawCall.model,
+          cacheHit: rawCall.cacheHit,
+          modelResponse: {
+            receivedAt: rawCall.receivedAt,
+            reqPayload: rawCall.reqPayload,
+            respPayload: rawCall.respPayload,
+            inputTokens: rawCall.inputTokens,
+            outputTokens: rawCall.outputTokens,
+            cost: rawCall.cost,
+            statusCode: rawCall.statusCode,
+            durationMs: rawCall.durationMs,
+          },
+          tags: tagsObject,
+        };
       });
 
-      const matchingLogs = await prisma.loggedCall.findMany({
-        where: { projectId },
-        select: { id: true },
+      const matchingLogIds = await updatedBaseQuery.select(["lc.id"]).execute();
+
+      const count = matchingLogIds.length;
+
+      return { calls, count, matchingLogIds: matchingLogIds.map((log) => log.id) };
+    }),
+  getTagNames: protectedProcedure
+    .input(z.object({ projectId: z.string() }))
+    .query(async ({ input, ctx }) => {
+      await requireCanViewProject(input.projectId, ctx);
+
+      const tags = await prisma.loggedCallTag.findMany({
+        distinct: ["name"],
+        where: {
+          projectId: input.projectId,
+        },
+        select: {
+          name: true,
+        },
+        orderBy: {
+          name: "asc",
+        },
       });
 
-      const count = await prisma.loggedCall.count({
-        where: { projectId },
-      });
-
-      return { calls, count, matchingLogIds: matchingLogs.map((log) => log.id) };
+      return tags.map((tag) => tag.name);
     }),
 });

app/src/server/api/routers/projects.router.ts

@@ -51,6 +51,12 @@ export const projectsRouter = createTRPCRouter({
         include: {
           apiKeys: true,
           personalProjectUser: true,
+          projectUsers: {
+            include: {
+              user: true,
+            },
+          },
+          projectUserInvitations: true,
         },
       }),
       prisma.projectUser.findFirst({
@@ -58,7 +64,7 @@ export const projectsRouter = createTRPCRouter({
           userId: ctx.session.user.id,
           projectId: input.id,
           role: {
-            in: ["ADMIN", "MEMBER"],
+            in: ["ADMIN", "MEMBER", "VIEWER"],
           },
         },
       }),

app/src/server/api/routers/promptVariants.router.ts

@@ -131,6 +131,8 @@ export const promptVariantsRouter = createTRPCRouter({
       const inputTokens = overallTokens._sum?.inputTokens ?? 0;
       const outputTokens = overallTokens._sum?.outputTokens ?? 0;
 
+      const awaitingCompletions = outputCount < scenarioCount;
+
       const awaitingEvals = !!evalResults.find(
         (result) => result.totalCount < scenarioCount * evals.length,
       );
@@ -142,6 +144,7 @@ export const promptVariantsRouter = createTRPCRouter({
         overallCost: overallTokens._sum?.cost ?? 0,
         scenarioCount,
         outputCount,
+        awaitingCompletions,
         awaitingEvals,
       };
     }),

app/src/server/api/routers/scenarioVariantCells.router.ts

@@ -1,4 +1,6 @@
+import { TRPCError } from "@trpc/server";
 import { z } from "zod";
+import modelProviders from "~/modelProviders/modelProviders";
 import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/api/trpc";
 import { prisma } from "~/server/db";
 import { queueQueryModel } from "~/server/tasks/queryModel.task";
@@ -96,4 +98,46 @@ export const scenarioVariantCellsRouter = createTRPCRouter({
 
       await queueQueryModel(cell.id, true);
     }),
+  getTemplatedPromptMessage: publicProcedure
+    .input(
+      z.object({
+        cellId: z.string(),
+      }),
+    )
+    .query(async ({ input }) => {
+      const cell = await prisma.scenarioVariantCell.findUnique({
+        where: { id: input.cellId },
+        include: {
+          promptVariant: true,
+          modelResponses: true,
+        },
+      });
+
+      if (!cell) {
+        throw new TRPCError({
+          code: "NOT_FOUND",
+        });
+      }
+
+      const promptMessages = (cell.prompt as { messages: [] })["messages"];
+
+      if (!promptMessages) return null;
+
+      const { modelProvider, model } = cell.promptVariant;
+
+      const provider = modelProviders[modelProvider as keyof typeof modelProviders];
+
+      if (!provider) return null;
+
+      const modelObj = provider.models[model as keyof typeof provider.models];
+
+      const templatePrompt = modelObj?.templatePrompt;
+
+      if (!templatePrompt) return null;
+
+      return {
+        templatedPrompt: templatePrompt(promptMessages),
+        learnMoreUrl: modelObj.learnMoreUrl,
+      };
+    }),
 });

app/src/server/api/routers/users.router.ts

@@ -0,0 +1,232 @@
+import { v4 as uuidv4 } from "uuid";
+import { z } from "zod";
+
+import { createTRPCRouter, protectedProcedure } from "~/server/api/trpc";
+import { prisma } from "~/server/db";
+import { error, success } from "~/utils/errorHandling/standardResponses";
+import { requireIsProjectAdmin, requireNothing } from "~/utils/accessControl";
+import { TRPCError } from "@trpc/server";
+import { sendProjectInvitation } from "~/server/emails/sendProjectInvitation";
+
+export const usersRouter = createTRPCRouter({
+  inviteToProject: protectedProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        email: z.string().email(),
+        role: z.enum(["ADMIN", "MEMBER", "VIEWER"]),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      await requireIsProjectAdmin(input.projectId, ctx);
+
+      const user = await prisma.user.findUnique({
+        where: {
+          email: input.email,
+        },
+      });
+
+      if (user) {
+        const existingMembership = await prisma.projectUser.findUnique({
+          where: {
+            projectId_userId: {
+              projectId: input.projectId,
+              userId: user.id,
+            },
+          },
+        });
+
+        if (existingMembership) {
+          return error(`A user with ${input.email} is already a member of this project`);
+        }
+      }
+
+      const invitation = await prisma.userInvitation.upsert({
+        where: {
+          projectId_email: {
+            projectId: input.projectId,
+            email: input.email,
+          },
+        },
+        update: {
+          role: input.role,
+        },
+        create: {
+          projectId: input.projectId,
+          email: input.email,
+          role: input.role,
+          invitationToken: uuidv4(),
+          senderId: ctx.session.user.id,
+        },
+        include: {
+          project: {
+            select: {
+              name: true,
+            },
+          },
+        },
+      });
+
+      try {
+        await sendProjectInvitation({
+          invitationToken: invitation.invitationToken,
+          recipientEmail: input.email,
+          invitationSenderName: ctx.session.user.name || "",
+          invitationSenderEmail: ctx.session.user.email || "",
+          projectName: invitation.project.name,
+        });
+      } catch (e) {
+        // If we fail to send the email, we should delete the invitation
+        await prisma.userInvitation.delete({
+          where: {
+            invitationToken: invitation.invitationToken,
+          },
+        });
+        return error("Failed to send email");
+      }
+
+      return success();
+    }),
+  getProjectInvitation: protectedProcedure
+    .input(
+      z.object({
+        invitationToken: z.string(),
+      }),
+    )
+    .query(async ({ input, ctx }) => {
+      requireNothing(ctx);
+
+      const invitation = await prisma.userInvitation.findUnique({
+        where: {
+          invitationToken: input.invitationToken,
+        },
+        include: {
+          project: {
+            select: {
+              name: true,
+            },
+          },
+          sender: {
+            select: {
+              name: true,
+              email: true,
+            },
+          },
+        },
+      });
+
+      if (!invitation) {
+        throw new TRPCError({ code: "NOT_FOUND" });
+      }
+
+      return invitation;
+    }),
+  acceptProjectInvitation: protectedProcedure
+    .input(
+      z.object({
+        invitationToken: z.string(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      requireNothing(ctx);
+
+      const invitation = await prisma.userInvitation.findUnique({
+        where: {
+          invitationToken: input.invitationToken,
+        },
+      });
+
+      if (!invitation) {
+        throw new TRPCError({ code: "NOT_FOUND" });
+      }
+
+      await prisma.projectUser.create({
+        data: {
+          projectId: invitation.projectId,
+          userId: ctx.session.user.id,
+          role: invitation.role,
+        },
+      });
+
+      await prisma.userInvitation.delete({
+        where: {
+          invitationToken: input.invitationToken,
+        },
+      });
+
+      return success(invitation.projectId);
+    }),
+  cancelProjectInvitation: protectedProcedure
+    .input(
+      z.object({
+        invitationToken: z.string(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      requireNothing(ctx);
+
+      const invitation = await prisma.userInvitation.findUnique({
+        where: {
+          invitationToken: input.invitationToken,
+        },
+      });
+
+      if (!invitation) {
+        throw new TRPCError({ code: "NOT_FOUND" });
+      }
+
+      await prisma.userInvitation.delete({
+        where: {
+          invitationToken: input.invitationToken,
+        },
+      });
+
+      return success();
+    }),
+  editProjectUserRole: protectedProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        userId: z.string(),
+        role: z.enum(["ADMIN", "MEMBER", "VIEWER"]),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      await requireIsProjectAdmin(input.projectId, ctx);
+
+      await prisma.projectUser.update({
+        where: {
+          projectId_userId: {
+            projectId: input.projectId,
+            userId: input.userId,
+          },
+        },
+        data: {
+          role: input.role,
+        },
+      });
+
+      return success();
+    }),
+  removeUserFromProject: protectedProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        userId: z.string(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      await requireIsProjectAdmin(input.projectId, ctx);
+
+      await prisma.projectUser.delete({
+        where: {
+          projectId_userId: {
+            projectId: input.projectId,
+            userId: input.userId,
+          },
+        },
+      });
+
+      return success();
+    }),
+});

app/src/server/api/trpc.ts

@@ -27,7 +27,6 @@ import { capturePath } from "~/utils/analytics/serverAnalytics";
 
 type CreateContextOptions = {
   session: Session | null;
-  apiKey: string | null;
 };
 
 // eslint-disable-next-line @typescript-eslint/no-empty-function
@@ -46,7 +45,6 @@ const noOp = () => {};
 export const createInnerTRPCContext = (opts: CreateContextOptions) => {
   return {
     session: opts.session,
-    apiKey: opts.apiKey,
     prisma,
     markAccessControlRun: noOp,
   };
@@ -64,11 +62,8 @@ export const createTRPCContext = async (opts: CreateNextContextOptions) => {
   // Get the session from the server using the getServerSession wrapper function
   const session = await getServerAuthSession({ req, res });
 
-  const apiKey = req.headers.authorization?.split(" ")[1] as string | null;
-
   return createInnerTRPCContext({
     session,
-    apiKey,
   });
 };
 

app/src/server/emails/sendEmail.ts

@@ -0,0 +1,31 @@
+import { marked } from "marked";
+import nodemailer from "nodemailer";
+import { env } from "~/env.mjs";
+
+const transporter = nodemailer.createTransport({
+  // All the SMTP_ env vars come from https://app.brevo.com/settings/keys/smtp
+  // @ts-expect-error nodemailer types are wrong
+  host: env.SMTP_HOST,
+  port: env.SMTP_PORT,
+  auth: {
+    user: env.SMTP_LOGIN,
+    pass: env.SMTP_PASSWORD,
+  },
+});
+
+export const sendEmail = async (options: { to: string; subject: string; body: string }) => {
+  const bodyHtml = await marked.parseInline(options.body, { mangle: false });
+
+  try {
+    await transporter.sendMail({
+      from: env.SENDER_EMAIL,
+      to: options.to,
+      subject: options.subject,
+      html: bodyHtml,
+      text: options.body,
+    });
+  } catch (error) {
+    console.error("error sending email", error);
+    throw error;
+  }
+};

app/src/server/emails/sendProjectInvitation.ts

@@ -0,0 +1,29 @@
+import { env } from "~/env.mjs";
+import { sendEmail } from "./sendEmail";
+
+export const sendProjectInvitation = async ({
+  invitationToken,
+  recipientEmail,
+  invitationSenderName,
+  invitationSenderEmail,
+  projectName,
+}: {
+  invitationToken: string;
+  recipientEmail: string;
+  invitationSenderName: string;
+  invitationSenderEmail: string;
+  projectName: string;
+}) => {
+  const invitationLink = `${env.NEXT_PUBLIC_HOST}/invitations/${invitationToken}`;
+
+  const emailBody = `
+    <p>You have been invited to join ${projectName} by ${invitationSenderName} (${invitationSenderEmail}).</p>
+    <p>Click <a href="${invitationLink}">here</a> to accept the invitation.</p>
+    `;
+
+  await sendEmail({
+    to: recipientEmail,
+    subject: "You've been invited to join a project",
+    body: emailBody,
+  });
+};

app/src/server/scripts/client-codegen.ts

@@ -1,8 +1,9 @@
 import "dotenv/config";
-import { openApiDocument } from "~/pages/api/openapi.json";
+import { openApiDocument } from "~/pages/api/v1/openapi.json";
 import fs from "fs";
 import path from "path";
 import { execSync } from "child_process";
+import { generate } from "openapi-typescript-codegen";
 
 const scriptPath = import.meta.url.replace("file://", "");
 const clientLibsPath = path.join(path.dirname(scriptPath), "../../../../client-libs");
@@ -18,13 +19,20 @@ console.log("Generating TypeScript client");
 const tsClientPath = path.join(clientLibsPath, "typescript/src/codegen");
 
 fs.rmSync(tsClientPath, { recursive: true, force: true });
+fs.mkdirSync(tsClientPath, { recursive: true });
 
-execSync(
-  `pnpm dlx @openapitools/openapi-generator-cli generate -i "${schemaPath}" -g typescript-axios -o "${tsClientPath}"`,
-  {
-    stdio: "inherit",
-  },
-);
+await generate({
+  input: openApiDocument,
+  output: tsClientPath,
+  clientName: "OPClient",
+  httpClient: "node",
+});
+// execSync(
+//   `pnpm run openapi generate --input "${schemaPath}" --output "${tsClientPath}" --name OPClient --client node`,
+//   {
+//     stdio: "inherit",
+//   },
+// );
 
 console.log("Generating Python client");
 

app/src/server/utils/deriveNewContructFn.ts

@@ -51,7 +51,7 @@ const requestUpdatedPromptFunction = async (
             originalModelProvider.inputSchema,
             null,
             2,
-          )}\n\nDo not add any assistant messages.`,
+          )}`,
         },
         {
           role: "user",

app/src/server/utils/generateApiKey.ts

@@ -2,4 +2,4 @@ import cryptoRandomString from "crypto-random-string";
 
 const KEY_LENGTH = 42;
 
-export const generateApiKey = () => `opc_${cryptoRandomString({ length: KEY_LENGTH })}`;
+export const generateApiKey = () => `opk_${cryptoRandomString({ length: KEY_LENGTH })}`;

app/src/server/utils/openai.ts

@@ -1,7 +1,6 @@
-import { type ClientOptions } from "openai";
 import fs from "fs";
 import path from "path";
-import OpenAI from "openpipe/src/openai";
+import OpenAI, { type ClientOptions } from "openpipe/src/openai";
 
 import { env } from "~/env.mjs";
 
@@ -16,7 +15,13 @@ try {
   config = JSON.parse(jsonData.toString());
 } catch (error) {
   // Set a dummy key so it doesn't fail at build time
-  config = { apiKey: env.OPENAI_API_KEY ?? "dummy-key" };
+  config = {
+    apiKey: env.OPENAI_API_KEY ?? "dummy-key",
+    openpipe: {
+      apiKey: env.OPENPIPE_API_KEY,
+      baseUrl: "http://localhost:3000/api/v1",
+    },
+  };
 }
 
 // export const openai = env.OPENPIPE_API_KEY ? new OpenAI.OpenAI(config) : new OriginalOpenAI(config);

app/src/state/logFiltersSlice.ts

@@ -0,0 +1,42 @@
+import { type SliceCreator } from "./store";
+
+export const comparators = ["=", "!=", "CONTAINS", "NOT_CONTAINS"] as const;
+
+export const defaultFilterableFields = ["Request", "Response", "Model", "Status Code"] as const;
+
+export interface LogFilter {
+  id: string;
+  field: string;
+  comparator: (typeof comparators)[number];
+  value: string;
+}
+
+export type LogFiltersSlice = {
+  filters: LogFilter[];
+  addFilter: (filter: LogFilter) => void;
+  updateFilter: (filter: LogFilter) => void;
+  deleteFilter: (id: string) => void;
+  clearSelectedLogIds: () => void;
+};
+
+export const createLogFiltersSlice: SliceCreator<LogFiltersSlice> = (set, get) => ({
+  filters: [],
+  addFilter: (filter: LogFilter) =>
+    set((state) => {
+      state.logFilters.filters.push(filter);
+    }),
+  updateFilter: (filter: LogFilter) =>
+    set((state) => {
+      const index = state.logFilters.filters.findIndex((f) => f.id === filter.id);
+      state.logFilters.filters[index] = filter;
+    }),
+  deleteFilter: (id: string) =>
+    set((state) => {
+      const index = state.logFilters.filters.findIndex((f) => f.id === id);
+      state.logFilters.filters.splice(index, 1);
+    }),
+  clearSelectedLogIds: () =>
+    set((state) => {
+      state.logFilters.filters = [];
+    }),
+});

app/src/state/selectedLogsSlice.ts

@@ -1,7 +1,5 @@
 import { type SliceCreator } from "./store";
 
-export const editorBackground = "#fafafa";
-
 export type SelectedLogsSlice = {
   selectedLogIds: Set<string>;
   toggleSelectedLogId: (id: string) => void;

app/src/state/store.ts

@@ -10,6 +10,7 @@ import {
 import { type APIClient } from "~/utils/api";
 import { persistOptions, type stateToPersist } from "./persist";
 import { type SelectedLogsSlice, createSelectedLogsSlice } from "./selectedLogsSlice";
+import { type LogFiltersSlice, createLogFiltersSlice } from "./logFiltersSlice";
 
 enableMapSet();
 
@@ -23,6 +24,7 @@ export type State = {
   selectedProjectId: string | null;
   setSelectedProjectId: (id: string) => void;
   selectedLogs: SelectedLogsSlice;
+  logFilters: LogFiltersSlice;
 };
 
 export type SliceCreator<T> = StateCreator<State, [["zustand/immer", never]], [], T>;
@@ -58,6 +60,7 @@ const useBaseStore = create<
           state.selectedProjectId = id;
         }),
       selectedLogs: createSelectedLogsSlice(set, get, ...rest),
+      logFilters: createLogFiltersSlice(set, get, ...rest),
     })),
     persistOptions,
   ),

app/src/theme/ChakraThemeProvider.tsx

@@ -18,7 +18,7 @@ const { definePartsStyle, defineMultiStyleConfig } = createMultiStyleConfigHelpe
 
 const modalTheme = defineMultiStyleConfig({
   baseStyle: definePartsStyle({
-    dialog: { borderRadius: "sm" },
+    dialog: { borderRadius: "md", mx: 4 },
   }),
 });
 
@@ -45,7 +45,7 @@ const theme = extendTheme({
   components: {
     Button: {
       baseStyle: {
-        borderRadius: "sm",
+        borderRadius: "md",
       },
     },
     Input: {

app/src/utils/hooks.ts

@@ -179,9 +179,29 @@ export const useScenarioVars = () => {
 export const useLoggedCalls = () => {
   const selectedProjectId = useAppStore((state) => state.selectedProjectId);
   const { page, pageSize } = usePageParams();
+  const filters = useAppStore((state) => state.logFilters.filters);
 
-  return api.loggedCalls.list.useQuery(
-    { projectId: selectedProjectId ?? "", page, pageSize },
+  const { data, isLoading, ...rest } = api.loggedCalls.list.useQuery(
+    { projectId: selectedProjectId ?? "", page, pageSize, filters },
+    { enabled: !!selectedProjectId },
+  );
+
+  const [stableData, setStableData] = useState(data);
+
+  useEffect(() => {
+    // Prevent annoying flashes while logs are loading from the server
+    if (!isLoading) {
+      setStableData(data);
+    }
+  }, [data, isLoading]);
+
+  return { data: stableData, isLoading, ...rest };
+};
+
+export const useTagNames = () => {
+  const selectedProjectId = useAppStore((state) => state.selectedProjectId);
+  return api.loggedCalls.getTagNames.useQuery(
+    { projectId: selectedProjectId ?? "" },
     { enabled: !!selectedProjectId },
   );
 };

client-libs/openapi.json

@@ -3,17 +3,17 @@
   "info": {
     "title": "OpenPipe API",
     "description": "The public API for reporting API calls to OpenPipe",
-    "version": "0.1.0"
+    "version": "0.1.1"
   },
   "servers": [
     {
-      "url": "https://app.openpipe.ai/api"
+      "url": "https://app.openpipe.ai/api/v1"
     }
   ],
   "paths": {
-    "/v1/check-cache": {
+    "/check-cache": {
       "post": {
-        "operationId": "externalApi-checkCache",
+        "operationId": "checkCache",
         "description": "Check if a prompt is cached",
         "security": [
           {
@@ -39,7 +39,8 @@
                     "additionalProperties": {
                       "type": "string"
                     },
-                    "description": "Extra tags to attach to the call for filtering. Eg { \"userId\": \"123\", \"promptId\": \"populate-title\" }"
+                    "description": "Extra tags to attach to the call for filtering. Eg { \"userId\": \"123\", \"promptId\": \"populate-title\" }",
+                    "default": {}
                   }
                 },
                 "required": [
@@ -74,9 +75,9 @@
         }
       }
     },
-    "/v1/report": {
+    "/report": {
       "post": {
-        "operationId": "externalApi-report",
+        "operationId": "report",
         "description": "Report an API call",
         "security": [
           {
@@ -117,7 +118,8 @@
                     "additionalProperties": {
                       "type": "string"
                     },
-                    "description": "Extra tags to attach to the call for filtering. Eg { \"userId\": \"123\", \"promptId\": \"populate-title\" }"
+                    "description": "Extra tags to attach to the call for filtering. Eg { \"userId\": \"123\", \"promptId\": \"populate-title\" }",
+                    "default": {}
                   }
                 },
                 "required": [
@@ -135,7 +137,97 @@
             "description": "Successful response",
             "content": {
               "application/json": {
-                "schema": {}
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "status": {
+                      "type": "string",
+                      "enum": [
+                        "ok"
+                      ]
+                    }
+                  },
+                  "required": [
+                    "status"
+                  ],
+                  "additionalProperties": false
+                }
+              }
+            }
+          },
+          "default": {
+            "$ref": "#/components/responses/error"
+          }
+        }
+      }
+    },
+    "/local-testing-only-get-latest-logged-call": {
+      "get": {
+        "operationId": "localTestingOnlyGetLatestLoggedCall",
+        "description": "Get the latest logged call (only for local testing)",
+        "security": [
+          {
+            "Authorization": []
+          }
+        ],
+        "parameters": [],
+        "responses": {
+          "200": {
+            "description": "Successful response",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "type": "object",
+                  "properties": {
+                    "createdAt": {
+                      "type": "string",
+                      "format": "date-time"
+                    },
+                    "cacheHit": {
+                      "type": "boolean"
+                    },
+                    "tags": {
+                      "type": "object",
+                      "additionalProperties": {
+                        "type": "string",
+                        "nullable": true
+                      }
+                    },
+                    "modelResponse": {
+                      "type": "object",
+                      "properties": {
+                        "id": {
+                          "type": "string"
+                        },
+                        "statusCode": {
+                          "type": "number",
+                          "nullable": true
+                        },
+                        "errorMessage": {
+                          "type": "string",
+                          "nullable": true
+                        },
+                        "reqPayload": {},
+                        "respPayload": {}
+                      },
+                      "required": [
+                        "id",
+                        "statusCode",
+                        "errorMessage"
+                      ],
+                      "additionalProperties": false,
+                      "nullable": true
+                    }
+                  },
+                  "required": [
+                    "createdAt",
+                    "cacheHit",
+                    "tags",
+                    "modelResponse"
+                  ],
+                  "additionalProperties": false,
+                  "nullable": true
+                }
               }
             }
           },

client-libs/python/openpipe/api_client/api/default/check_cache.py

@@ -5,14 +5,14 @@ import httpx
 
 from ... import errors
 from ...client import AuthenticatedClient, Client
-from ...models.external_api_check_cache_json_body import ExternalApiCheckCacheJsonBody
-from ...models.external_api_check_cache_response_200 import ExternalApiCheckCacheResponse200
+from ...models.check_cache_json_body import CheckCacheJsonBody
+from ...models.check_cache_response_200 import CheckCacheResponse200
 from ...types import Response
 
 
 def _get_kwargs(
     *,
-    json_body: ExternalApiCheckCacheJsonBody,
+    json_body: CheckCacheJsonBody,
 ) -> Dict[str, Any]:
     pass
 
@@ -20,16 +20,16 @@ def _get_kwargs(
 
     return {
         "method": "post",
-        "url": "/v1/check-cache",
+        "url": "/check-cache",
         "json": json_json_body,
     }
 
 
 def _parse_response(
     *, client: Union[AuthenticatedClient, Client], response: httpx.Response
-) -> Optional[ExternalApiCheckCacheResponse200]:
+) -> Optional[CheckCacheResponse200]:
     if response.status_code == HTTPStatus.OK:
-        response_200 = ExternalApiCheckCacheResponse200.from_dict(response.json())
+        response_200 = CheckCacheResponse200.from_dict(response.json())
 
         return response_200
     if client.raise_on_unexpected_status:
@@ -40,7 +40,7 @@ def _parse_response(
 
 def _build_response(
     *, client: Union[AuthenticatedClient, Client], response: httpx.Response
-) -> Response[ExternalApiCheckCacheResponse200]:
+) -> Response[CheckCacheResponse200]:
     return Response(
         status_code=HTTPStatus(response.status_code),
         content=response.content,
@@ -52,19 +52,19 @@ def _build_response(
 def sync_detailed(
     *,
     client: AuthenticatedClient,
-    json_body: ExternalApiCheckCacheJsonBody,
-) -> Response[ExternalApiCheckCacheResponse200]:
+    json_body: CheckCacheJsonBody,
+) -> Response[CheckCacheResponse200]:
     """Check if a prompt is cached
 
     Args:
-        json_body (ExternalApiCheckCacheJsonBody):
+        json_body (CheckCacheJsonBody):
 
     Raises:
         errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
         httpx.TimeoutException: If the request takes longer than Client.timeout.
 
     Returns:
-        Response[ExternalApiCheckCacheResponse200]
+        Response[CheckCacheResponse200]
     """
 
     kwargs = _get_kwargs(
@@ -81,19 +81,19 @@ def sync_detailed(
 def sync(
     *,
     client: AuthenticatedClient,
-    json_body: ExternalApiCheckCacheJsonBody,
-) -> Optional[ExternalApiCheckCacheResponse200]:
+    json_body: CheckCacheJsonBody,
+) -> Optional[CheckCacheResponse200]:
     """Check if a prompt is cached
 
     Args:
-        json_body (ExternalApiCheckCacheJsonBody):
+        json_body (CheckCacheJsonBody):
 
     Raises:
         errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
         httpx.TimeoutException: If the request takes longer than Client.timeout.
 
     Returns:
-        ExternalApiCheckCacheResponse200
+        CheckCacheResponse200
     """
 
     return sync_detailed(
@@ -105,19 +105,19 @@ def sync(
 async def asyncio_detailed(
     *,
     client: AuthenticatedClient,
-    json_body: ExternalApiCheckCacheJsonBody,
-) -> Response[ExternalApiCheckCacheResponse200]:
+    json_body: CheckCacheJsonBody,
+) -> Response[CheckCacheResponse200]:
     """Check if a prompt is cached
 
     Args:
-        json_body (ExternalApiCheckCacheJsonBody):
+        json_body (CheckCacheJsonBody):
 
     Raises:
         errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
         httpx.TimeoutException: If the request takes longer than Client.timeout.
 
     Returns:
-        Response[ExternalApiCheckCacheResponse200]
+        Response[CheckCacheResponse200]
     """
 
     kwargs = _get_kwargs(
@@ -132,19 +132,19 @@ async def asyncio_detailed(
 async def asyncio(
     *,
     client: AuthenticatedClient,
-    json_body: ExternalApiCheckCacheJsonBody,
-) -> Optional[ExternalApiCheckCacheResponse200]:
+    json_body: CheckCacheJsonBody,
+) -> Optional[CheckCacheResponse200]:
     """Check if a prompt is cached
 
     Args:
-        json_body (ExternalApiCheckCacheJsonBody):
+        json_body (CheckCacheJsonBody):
 
     Raises:
         errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
         httpx.TimeoutException: If the request takes longer than Client.timeout.
 
     Returns:
-        ExternalApiCheckCacheResponse200
+        CheckCacheResponse200
     """
 
     return (

client-libs/python/openpipe/api_client/api/default/external_api_report.py

@@ -1,98 +0,0 @@
-from http import HTTPStatus
-from typing import Any, Dict, Optional, Union
-
-import httpx
-
-from ... import errors
-from ...client import AuthenticatedClient, Client
-from ...models.external_api_report_json_body import ExternalApiReportJsonBody
-from ...types import Response
-
-
-def _get_kwargs(
-    *,
-    json_body: ExternalApiReportJsonBody,
-) -> Dict[str, Any]:
-    pass
-
-    json_json_body = json_body.to_dict()
-
-    return {
-        "method": "post",
-        "url": "/v1/report",
-        "json": json_json_body,
-    }
-
-
-def _parse_response(*, client: Union[AuthenticatedClient, Client], response: httpx.Response) -> Optional[Any]:
-    if response.status_code == HTTPStatus.OK:
-        return None
-    if client.raise_on_unexpected_status:
-        raise errors.UnexpectedStatus(response.status_code, response.content)
-    else:
-        return None
-
-
-def _build_response(*, client: Union[AuthenticatedClient, Client], response: httpx.Response) -> Response[Any]:
-    return Response(
-        status_code=HTTPStatus(response.status_code),
-        content=response.content,
-        headers=response.headers,
-        parsed=_parse_response(client=client, response=response),
-    )
-
-
-def sync_detailed(
-    *,
-    client: AuthenticatedClient,
-    json_body: ExternalApiReportJsonBody,
-) -> Response[Any]:
-    """Report an API call
-
-    Args:
-        json_body (ExternalApiReportJsonBody):
-
-    Raises:
-        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
-        httpx.TimeoutException: If the request takes longer than Client.timeout.
-
-    Returns:
-        Response[Any]
-    """
-
-    kwargs = _get_kwargs(
-        json_body=json_body,
-    )
-
-    response = client.get_httpx_client().request(
-        **kwargs,
-    )
-
-    return _build_response(client=client, response=response)
-
-
-async def asyncio_detailed(
-    *,
-    client: AuthenticatedClient,
-    json_body: ExternalApiReportJsonBody,
-) -> Response[Any]:
-    """Report an API call
-
-    Args:
-        json_body (ExternalApiReportJsonBody):
-
-    Raises:
-        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
-        httpx.TimeoutException: If the request takes longer than Client.timeout.
-
-    Returns:
-        Response[Any]
-    """
-
-    kwargs = _get_kwargs(
-        json_body=json_body,
-    )
-
-    response = await client.get_async_httpx_client().request(**kwargs)
-
-    return _build_response(client=client, response=response)

client-libs/python/openpipe/api_client/api/default/local_testing_only_get_latest_logged_call.py

@@ -0,0 +1,133 @@
+from http import HTTPStatus
+from typing import Any, Dict, Optional, Union
+
+import httpx
+
+from ... import errors
+from ...client import AuthenticatedClient, Client
+from ...models.local_testing_only_get_latest_logged_call_response_200 import (
+    LocalTestingOnlyGetLatestLoggedCallResponse200,
+)
+from ...types import Response
+
+
+def _get_kwargs() -> Dict[str, Any]:
+    pass
+
+    return {
+        "method": "get",
+        "url": "/local-testing-only-get-latest-logged-call",
+    }
+
+
+def _parse_response(
+    *, client: Union[AuthenticatedClient, Client], response: httpx.Response
+) -> Optional[Optional[LocalTestingOnlyGetLatestLoggedCallResponse200]]:
+    if response.status_code == HTTPStatus.OK:
+        _response_200 = response.json()
+        response_200: Optional[LocalTestingOnlyGetLatestLoggedCallResponse200]
+        if _response_200 is None:
+            response_200 = None
+        else:
+            response_200 = LocalTestingOnlyGetLatestLoggedCallResponse200.from_dict(_response_200)
+
+        return response_200
+    if client.raise_on_unexpected_status:
+        raise errors.UnexpectedStatus(response.status_code, response.content)
+    else:
+        return None
+
+
+def _build_response(
+    *, client: Union[AuthenticatedClient, Client], response: httpx.Response
+) -> Response[Optional[LocalTestingOnlyGetLatestLoggedCallResponse200]]:
+    return Response(
+        status_code=HTTPStatus(response.status_code),
+        content=response.content,
+        headers=response.headers,
+        parsed=_parse_response(client=client, response=response),
+    )
+
+
+def sync_detailed(
+    *,
+    client: AuthenticatedClient,
+) -> Response[Optional[LocalTestingOnlyGetLatestLoggedCallResponse200]]:
+    """Get the latest logged call (only for local testing)
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        Response[Optional[LocalTestingOnlyGetLatestLoggedCallResponse200]]
+    """
+
+    kwargs = _get_kwargs()
+
+    response = client.get_httpx_client().request(
+        **kwargs,
+    )
+
+    return _build_response(client=client, response=response)
+
+
+def sync(
+    *,
+    client: AuthenticatedClient,
+) -> Optional[Optional[LocalTestingOnlyGetLatestLoggedCallResponse200]]:
+    """Get the latest logged call (only for local testing)
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        Optional[LocalTestingOnlyGetLatestLoggedCallResponse200]
+    """
+
+    return sync_detailed(
+        client=client,
+    ).parsed
+
+
+async def asyncio_detailed(
+    *,
+    client: AuthenticatedClient,
+) -> Response[Optional[LocalTestingOnlyGetLatestLoggedCallResponse200]]:
+    """Get the latest logged call (only for local testing)
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        Response[Optional[LocalTestingOnlyGetLatestLoggedCallResponse200]]
+    """
+
+    kwargs = _get_kwargs()
+
+    response = await client.get_async_httpx_client().request(**kwargs)
+
+    return _build_response(client=client, response=response)
+
+
+async def asyncio(
+    *,
+    client: AuthenticatedClient,
+) -> Optional[Optional[LocalTestingOnlyGetLatestLoggedCallResponse200]]:
+    """Get the latest logged call (only for local testing)
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        Optional[LocalTestingOnlyGetLatestLoggedCallResponse200]
+    """
+
+    return (
+        await asyncio_detailed(
+            client=client,
+        )
+    ).parsed

client-libs/python/openpipe/api_client/api/default/report.py

@@ -0,0 +1,155 @@
+from http import HTTPStatus
+from typing import Any, Dict, Optional, Union
+
+import httpx
+
+from ... import errors
+from ...client import AuthenticatedClient, Client
+from ...models.report_json_body import ReportJsonBody
+from ...models.report_response_200 import ReportResponse200
+from ...types import Response
+
+
+def _get_kwargs(
+    *,
+    json_body: ReportJsonBody,
+) -> Dict[str, Any]:
+    pass
+
+    json_json_body = json_body.to_dict()
+
+    return {
+        "method": "post",
+        "url": "/report",
+        "json": json_json_body,
+    }
+
+
+def _parse_response(
+    *, client: Union[AuthenticatedClient, Client], response: httpx.Response
+) -> Optional[ReportResponse200]:
+    if response.status_code == HTTPStatus.OK:
+        response_200 = ReportResponse200.from_dict(response.json())
+
+        return response_200
+    if client.raise_on_unexpected_status:
+        raise errors.UnexpectedStatus(response.status_code, response.content)
+    else:
+        return None
+
+
+def _build_response(
+    *, client: Union[AuthenticatedClient, Client], response: httpx.Response
+) -> Response[ReportResponse200]:
+    return Response(
+        status_code=HTTPStatus(response.status_code),
+        content=response.content,
+        headers=response.headers,
+        parsed=_parse_response(client=client, response=response),
+    )
+
+
+def sync_detailed(
+    *,
+    client: AuthenticatedClient,
+    json_body: ReportJsonBody,
+) -> Response[ReportResponse200]:
+    """Report an API call
+
+    Args:
+        json_body (ReportJsonBody):
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        Response[ReportResponse200]
+    """
+
+    kwargs = _get_kwargs(
+        json_body=json_body,
+    )
+
+    response = client.get_httpx_client().request(
+        **kwargs,
+    )
+
+    return _build_response(client=client, response=response)
+
+
+def sync(
+    *,
+    client: AuthenticatedClient,
+    json_body: ReportJsonBody,
+) -> Optional[ReportResponse200]:
+    """Report an API call
+
+    Args:
+        json_body (ReportJsonBody):
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        ReportResponse200
+    """
+
+    return sync_detailed(
+        client=client,
+        json_body=json_body,
+    ).parsed
+
+
+async def asyncio_detailed(
+    *,
+    client: AuthenticatedClient,
+    json_body: ReportJsonBody,
+) -> Response[ReportResponse200]:
+    """Report an API call
+
+    Args:
+        json_body (ReportJsonBody):
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        Response[ReportResponse200]
+    """
+
+    kwargs = _get_kwargs(
+        json_body=json_body,
+    )
+
+    response = await client.get_async_httpx_client().request(**kwargs)
+
+    return _build_response(client=client, response=response)
+
+
+async def asyncio(
+    *,
+    client: AuthenticatedClient,
+    json_body: ReportJsonBody,
+) -> Optional[ReportResponse200]:
+    """Report an API call
+
+    Args:
+        json_body (ReportJsonBody):
+
+    Raises:
+        errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True.
+        httpx.TimeoutException: If the request takes longer than Client.timeout.
+
+    Returns:
+        ReportResponse200
+    """
+
+    return (
+        await asyncio_detailed(
+            client=client,
+            json_body=json_body,
+        )
+    ).parsed

client-libs/python/openpipe/api_client/models/__init__.py

@@ -1,15 +1,29 @@
 """ Contains all the data models used in inputs/outputs """
 
-from .external_api_check_cache_json_body import ExternalApiCheckCacheJsonBody
-from .external_api_check_cache_json_body_tags import ExternalApiCheckCacheJsonBodyTags
-from .external_api_check_cache_response_200 import ExternalApiCheckCacheResponse200
-from .external_api_report_json_body import ExternalApiReportJsonBody
-from .external_api_report_json_body_tags import ExternalApiReportJsonBodyTags
+from .check_cache_json_body import CheckCacheJsonBody
+from .check_cache_json_body_tags import CheckCacheJsonBodyTags
+from .check_cache_response_200 import CheckCacheResponse200
+from .local_testing_only_get_latest_logged_call_response_200 import LocalTestingOnlyGetLatestLoggedCallResponse200
+from .local_testing_only_get_latest_logged_call_response_200_model_response import (
+    LocalTestingOnlyGetLatestLoggedCallResponse200ModelResponse,
+)
+from .local_testing_only_get_latest_logged_call_response_200_tags import (
+    LocalTestingOnlyGetLatestLoggedCallResponse200Tags,
+)
+from .report_json_body import ReportJsonBody
+from .report_json_body_tags import ReportJsonBodyTags
+from .report_response_200 import ReportResponse200
+from .report_response_200_status import ReportResponse200Status
 
 __all__ = (
-    "ExternalApiCheckCacheJsonBody",
-    "ExternalApiCheckCacheJsonBodyTags",
-    "ExternalApiCheckCacheResponse200",
-    "ExternalApiReportJsonBody",
-    "ExternalApiReportJsonBodyTags",
+    "CheckCacheJsonBody",
+    "CheckCacheJsonBodyTags",
+    "CheckCacheResponse200",
+    "LocalTestingOnlyGetLatestLoggedCallResponse200",
+    "LocalTestingOnlyGetLatestLoggedCallResponse200ModelResponse",
+    "LocalTestingOnlyGetLatestLoggedCallResponse200Tags",
+    "ReportJsonBody",
+    "ReportJsonBodyTags",
+    "ReportResponse200",
+    "ReportResponse200Status",
 )

client-libs/python/openpipe/api_client/models/check_cache_json_body.py

@@ -5,25 +5,25 @@ from attrs import define
 from ..types import UNSET, Unset
 
 if TYPE_CHECKING:
-    from ..models.external_api_check_cache_json_body_tags import ExternalApiCheckCacheJsonBodyTags
+    from ..models.check_cache_json_body_tags import CheckCacheJsonBodyTags
 
 
-T = TypeVar("T", bound="ExternalApiCheckCacheJsonBody")
+T = TypeVar("T", bound="CheckCacheJsonBody")
 
 
 @define
-class ExternalApiCheckCacheJsonBody:
+class CheckCacheJsonBody:
     """
     Attributes:
         requested_at (float): Unix timestamp in milliseconds
         req_payload (Union[Unset, Any]): JSON-encoded request payload
-        tags (Union[Unset, ExternalApiCheckCacheJsonBodyTags]): Extra tags to attach to the call for filtering. Eg {
-            "userId": "123", "promptId": "populate-title" }
+        tags (Union[Unset, CheckCacheJsonBodyTags]): Extra tags to attach to the call for filtering. Eg { "userId":
+            "123", "promptId": "populate-title" }
     """
 
     requested_at: float
     req_payload: Union[Unset, Any] = UNSET
-    tags: Union[Unset, "ExternalApiCheckCacheJsonBodyTags"] = UNSET
+    tags: Union[Unset, "CheckCacheJsonBodyTags"] = UNSET
 
     def to_dict(self) -> Dict[str, Any]:
         requested_at = self.requested_at
@@ -47,7 +47,7 @@ class ExternalApiCheckCacheJsonBody:
 
     @classmethod
     def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
-        from ..models.external_api_check_cache_json_body_tags import ExternalApiCheckCacheJsonBodyTags
+        from ..models.check_cache_json_body_tags import CheckCacheJsonBodyTags
 
         d = src_dict.copy()
         requested_at = d.pop("requestedAt")
@@ -55,16 +55,16 @@ class ExternalApiCheckCacheJsonBody:
         req_payload = d.pop("reqPayload", UNSET)
 
         _tags = d.pop("tags", UNSET)
-        tags: Union[Unset, ExternalApiCheckCacheJsonBodyTags]
+        tags: Union[Unset, CheckCacheJsonBodyTags]
         if isinstance(_tags, Unset):
             tags = UNSET
         else:
-            tags = ExternalApiCheckCacheJsonBodyTags.from_dict(_tags)
+            tags = CheckCacheJsonBodyTags.from_dict(_tags)
 
-        external_api_check_cache_json_body = cls(
+        check_cache_json_body = cls(
             requested_at=requested_at,
             req_payload=req_payload,
             tags=tags,
         )
 
-        return external_api_check_cache_json_body
+        return check_cache_json_body

client-libs/python/openpipe/api_client/models/check_cache_json_body_tags.py

@@ -2,11 +2,11 @@ from typing import Any, Dict, List, Type, TypeVar
 
 from attrs import define, field
 
-T = TypeVar("T", bound="ExternalApiReportJsonBodyTags")
+T = TypeVar("T", bound="CheckCacheJsonBodyTags")
 
 
 @define
-class ExternalApiReportJsonBodyTags:
+class CheckCacheJsonBodyTags:
     """Extra tags to attach to the call for filtering. Eg { "userId": "123", "promptId": "populate-title" }"""
 
     additional_properties: Dict[str, str] = field(init=False, factory=dict)
@@ -21,10 +21,10 @@ class ExternalApiReportJsonBodyTags:
     @classmethod
     def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
         d = src_dict.copy()
-        external_api_report_json_body_tags = cls()
+        check_cache_json_body_tags = cls()
 
-        external_api_report_json_body_tags.additional_properties = d
-        return external_api_report_json_body_tags
+        check_cache_json_body_tags.additional_properties = d
+        return check_cache_json_body_tags
 
     @property
     def additional_keys(self) -> List[str]:

client-libs/python/openpipe/api_client/models/check_cache_response_200.py

@@ -4,11 +4,11 @@ from attrs import define
 
 from ..types import UNSET, Unset
 
-T = TypeVar("T", bound="ExternalApiCheckCacheResponse200")
+T = TypeVar("T", bound="CheckCacheResponse200")
 
 
 @define
-class ExternalApiCheckCacheResponse200:
+class CheckCacheResponse200:
     """
     Attributes:
         resp_payload (Union[Unset, Any]): JSON-encoded response payload
@@ -31,8 +31,8 @@ class ExternalApiCheckCacheResponse200:
         d = src_dict.copy()
         resp_payload = d.pop("respPayload", UNSET)
 
-        external_api_check_cache_response_200 = cls(
+        check_cache_response_200 = cls(
             resp_payload=resp_payload,
         )
 
-        return external_api_check_cache_response_200
+        return check_cache_response_200

client-libs/python/openpipe/api_client/models/local_testing_only_get_latest_logged_call_response_200.py

@@ -0,0 +1,84 @@
+import datetime
+from typing import TYPE_CHECKING, Any, Dict, Optional, Type, TypeVar
+
+from attrs import define
+from dateutil.parser import isoparse
+
+if TYPE_CHECKING:
+    from ..models.local_testing_only_get_latest_logged_call_response_200_model_response import (
+        LocalTestingOnlyGetLatestLoggedCallResponse200ModelResponse,
+    )
+    from ..models.local_testing_only_get_latest_logged_call_response_200_tags import (
+        LocalTestingOnlyGetLatestLoggedCallResponse200Tags,
+    )
+
+
+T = TypeVar("T", bound="LocalTestingOnlyGetLatestLoggedCallResponse200")
+
+
+@define
+class LocalTestingOnlyGetLatestLoggedCallResponse200:
+    """
+    Attributes:
+        created_at (datetime.datetime):
+        cache_hit (bool):
+        tags (LocalTestingOnlyGetLatestLoggedCallResponse200Tags):
+        model_response (Optional[LocalTestingOnlyGetLatestLoggedCallResponse200ModelResponse]):
+    """
+
+    created_at: datetime.datetime
+    cache_hit: bool
+    tags: "LocalTestingOnlyGetLatestLoggedCallResponse200Tags"
+    model_response: Optional["LocalTestingOnlyGetLatestLoggedCallResponse200ModelResponse"]
+
+    def to_dict(self) -> Dict[str, Any]:
+        created_at = self.created_at.isoformat()
+
+        cache_hit = self.cache_hit
+        tags = self.tags.to_dict()
+
+        model_response = self.model_response.to_dict() if self.model_response else None
+
+        field_dict: Dict[str, Any] = {}
+        field_dict.update(
+            {
+                "createdAt": created_at,
+                "cacheHit": cache_hit,
+                "tags": tags,
+                "modelResponse": model_response,
+            }
+        )
+
+        return field_dict
+
+    @classmethod
+    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
+        from ..models.local_testing_only_get_latest_logged_call_response_200_model_response import (
+            LocalTestingOnlyGetLatestLoggedCallResponse200ModelResponse,
+        )
+        from ..models.local_testing_only_get_latest_logged_call_response_200_tags import (
+            LocalTestingOnlyGetLatestLoggedCallResponse200Tags,
+        )
+
+        d = src_dict.copy()
+        created_at = isoparse(d.pop("createdAt"))
+
+        cache_hit = d.pop("cacheHit")
+
+        tags = LocalTestingOnlyGetLatestLoggedCallResponse200Tags.from_dict(d.pop("tags"))
+
+        _model_response = d.pop("modelResponse")
+        model_response: Optional[LocalTestingOnlyGetLatestLoggedCallResponse200ModelResponse]
+        if _model_response is None:
+            model_response = None
+        else:
+            model_response = LocalTestingOnlyGetLatestLoggedCallResponse200ModelResponse.from_dict(_model_response)
+
+        local_testing_only_get_latest_logged_call_response_200 = cls(
+            created_at=created_at,
+            cache_hit=cache_hit,
+            tags=tags,
+            model_response=model_response,
+        )
+
+        return local_testing_only_get_latest_logged_call_response_200

client-libs/python/openpipe/api_client/models/local_testing_only_get_latest_logged_call_response_200_model_response.py

@@ -0,0 +1,70 @@
+from typing import Any, Dict, Optional, Type, TypeVar, Union
+
+from attrs import define
+
+from ..types import UNSET, Unset
+
+T = TypeVar("T", bound="LocalTestingOnlyGetLatestLoggedCallResponse200ModelResponse")
+
+
+@define
+class LocalTestingOnlyGetLatestLoggedCallResponse200ModelResponse:
+    """
+    Attributes:
+        id (str):
+        status_code (Optional[float]):
+        error_message (Optional[str]):
+        req_payload (Union[Unset, Any]):
+        resp_payload (Union[Unset, Any]):
+    """
+
+    id: str
+    status_code: Optional[float]
+    error_message: Optional[str]
+    req_payload: Union[Unset, Any] = UNSET
+    resp_payload: Union[Unset, Any] = UNSET
+
+    def to_dict(self) -> Dict[str, Any]:
+        id = self.id
+        status_code = self.status_code
+        error_message = self.error_message
+        req_payload = self.req_payload
+        resp_payload = self.resp_payload
+
+        field_dict: Dict[str, Any] = {}
+        field_dict.update(
+            {
+                "id": id,
+                "statusCode": status_code,
+                "errorMessage": error_message,
+            }
+        )
+        if req_payload is not UNSET:
+            field_dict["reqPayload"] = req_payload
+        if resp_payload is not UNSET:
+            field_dict["respPayload"] = resp_payload
+
+        return field_dict
+
+    @classmethod
+    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
+        d = src_dict.copy()
+        id = d.pop("id")
+
+        status_code = d.pop("statusCode")
+
+        error_message = d.pop("errorMessage")
+
+        req_payload = d.pop("reqPayload", UNSET)
+
+        resp_payload = d.pop("respPayload", UNSET)
+
+        local_testing_only_get_latest_logged_call_response_200_model_response = cls(
+            id=id,
+            status_code=status_code,
+            error_message=error_message,
+            req_payload=req_payload,
+            resp_payload=resp_payload,
+        )
+
+        return local_testing_only_get_latest_logged_call_response_200_model_response

client-libs/python/openpipe/api_client/models/local_testing_only_get_latest_logged_call_response_200_tags.py

@@ -0,0 +1,43 @@
+from typing import Any, Dict, List, Optional, Type, TypeVar
+
+from attrs import define, field
+
+T = TypeVar("T", bound="LocalTestingOnlyGetLatestLoggedCallResponse200Tags")
+
+
+@define
+class LocalTestingOnlyGetLatestLoggedCallResponse200Tags:
+    """ """
+
+    additional_properties: Dict[str, Optional[str]] = field(init=False, factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        field_dict: Dict[str, Any] = {}
+        field_dict.update(self.additional_properties)
+        field_dict.update({})
+
+        return field_dict
+
+    @classmethod
+    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
+        d = src_dict.copy()
+        local_testing_only_get_latest_logged_call_response_200_tags = cls()
+
+        local_testing_only_get_latest_logged_call_response_200_tags.additional_properties = d
+        return local_testing_only_get_latest_logged_call_response_200_tags
+
+    @property
+    def additional_keys(self) -> List[str]:
+        return list(self.additional_properties.keys())
+
+    def __getitem__(self, key: str) -> Optional[str]:
+        return self.additional_properties[key]
+
+    def __setitem__(self, key: str, value: Optional[str]) -> None:
+        self.additional_properties[key] = value
+
+    def __delitem__(self, key: str) -> None:
+        del self.additional_properties[key]
+
+    def __contains__(self, key: str) -> bool:
+        return key in self.additional_properties

client-libs/python/openpipe/api_client/models/report_json_body.py

@@ -5,14 +5,14 @@ from attrs import define
 from ..types import UNSET, Unset
 
 if TYPE_CHECKING:
-    from ..models.external_api_report_json_body_tags import ExternalApiReportJsonBodyTags
+    from ..models.report_json_body_tags import ReportJsonBodyTags
 
 
-T = TypeVar("T", bound="ExternalApiReportJsonBody")
+T = TypeVar("T", bound="ReportJsonBody")
 
 
 @define
-class ExternalApiReportJsonBody:
+class ReportJsonBody:
     """
     Attributes:
         requested_at (float): Unix timestamp in milliseconds
@@ -21,8 +21,8 @@ class ExternalApiReportJsonBody:
         resp_payload (Union[Unset, Any]): JSON-encoded response payload
         status_code (Union[Unset, float]): HTTP status code of response
         error_message (Union[Unset, str]): User-friendly error message
-        tags (Union[Unset, ExternalApiReportJsonBodyTags]): Extra tags to attach to the call for filtering. Eg {
-            "userId": "123", "promptId": "populate-title" }
+        tags (Union[Unset, ReportJsonBodyTags]): Extra tags to attach to the call for filtering. Eg { "userId": "123",
+            "promptId": "populate-title" }
     """
 
     requested_at: float
@@ -31,7 +31,7 @@ class ExternalApiReportJsonBody:
     resp_payload: Union[Unset, Any] = UNSET
     status_code: Union[Unset, float] = UNSET
     error_message: Union[Unset, str] = UNSET
-    tags: Union[Unset, "ExternalApiReportJsonBodyTags"] = UNSET
+    tags: Union[Unset, "ReportJsonBodyTags"] = UNSET
 
     def to_dict(self) -> Dict[str, Any]:
         requested_at = self.requested_at
@@ -66,7 +66,7 @@ class ExternalApiReportJsonBody:
 
     @classmethod
     def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
-        from ..models.external_api_report_json_body_tags import ExternalApiReportJsonBodyTags
+        from ..models.report_json_body_tags import ReportJsonBodyTags
 
         d = src_dict.copy()
         requested_at = d.pop("requestedAt")
@@ -82,13 +82,13 @@ class ExternalApiReportJsonBody:
         error_message = d.pop("errorMessage", UNSET)
 
         _tags = d.pop("tags", UNSET)
-        tags: Union[Unset, ExternalApiReportJsonBodyTags]
+        tags: Union[Unset, ReportJsonBodyTags]
         if isinstance(_tags, Unset):
             tags = UNSET
         else:
-            tags = ExternalApiReportJsonBodyTags.from_dict(_tags)
+            tags = ReportJsonBodyTags.from_dict(_tags)
 
-        external_api_report_json_body = cls(
+        report_json_body = cls(
             requested_at=requested_at,
             received_at=received_at,
             req_payload=req_payload,
@@ -98,4 +98,4 @@ class ExternalApiReportJsonBody:
             tags=tags,
         )
 
-        return external_api_report_json_body
+        return report_json_body

client-libs/python/openpipe/api_client/models/report_json_body_tags.py

@@ -2,11 +2,11 @@ from typing import Any, Dict, List, Type, TypeVar
 
 from attrs import define, field
 
-T = TypeVar("T", bound="ExternalApiCheckCacheJsonBodyTags")
+T = TypeVar("T", bound="ReportJsonBodyTags")
 
 
 @define
-class ExternalApiCheckCacheJsonBodyTags:
+class ReportJsonBodyTags:
     """Extra tags to attach to the call for filtering. Eg { "userId": "123", "promptId": "populate-title" }"""
 
     additional_properties: Dict[str, str] = field(init=False, factory=dict)
@@ -21,10 +21,10 @@ class ExternalApiCheckCacheJsonBodyTags:
     @classmethod
     def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
         d = src_dict.copy()
-        external_api_check_cache_json_body_tags = cls()
+        report_json_body_tags = cls()
 
-        external_api_check_cache_json_body_tags.additional_properties = d
-        return external_api_check_cache_json_body_tags
+        report_json_body_tags.additional_properties = d
+        return report_json_body_tags
 
     @property
     def additional_keys(self) -> List[str]:

client-libs/python/openpipe/api_client/models/report_response_200.py

@@ -0,0 +1,40 @@
+from typing import Any, Dict, Type, TypeVar
+
+from attrs import define
+
+from ..models.report_response_200_status import ReportResponse200Status
+
+T = TypeVar("T", bound="ReportResponse200")
+
+
+@define
+class ReportResponse200:
+    """
+    Attributes:
+        status (ReportResponse200Status):
+    """
+
+    status: ReportResponse200Status
+
+    def to_dict(self) -> Dict[str, Any]:
+        status = self.status.value
+
+        field_dict: Dict[str, Any] = {}
+        field_dict.update(
+            {
+                "status": status,
+            }
+        )
+
+        return field_dict
+
+    @classmethod
+    def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
+        d = src_dict.copy()
+        status = ReportResponse200Status(d.pop("status"))
+
+        report_response_200 = cls(
+            status=status,
+        )
+
+        return report_response_200

client-libs/python/openpipe/api_client/models/report_response_200_status.py

@@ -0,0 +1,8 @@
+from enum import Enum
+
+
+class ReportResponse200Status(str, Enum):
+    OK = "ok"
+
+    def __str__(self) -> str:
+        return str(self.value)

client-libs/python/openpipe/merge_openai_chunks.py

@@ -1,9 +1,9 @@
 from typing import Any, Optional
 
 
-def merge_streamed_chunks(base: Optional[Any], chunk: Any) -> Any:
+def merge_openai_chunks(base: Optional[Any], chunk: Any) -> Any:
     if base is None:
-        return merge_streamed_chunks({**chunk, "choices": []}, chunk)
+        return merge_openai_chunks({**chunk, "choices": []}, chunk)
 
     choices = base["choices"].copy()
     for choice in chunk["choices"]:
@@ -34,9 +34,7 @@ def merge_streamed_chunks(base: Optional[Any], chunk: Any) -> Any:
                 {**new_choice, "message": {"role": "assistant", **choice["delta"]}}
             )
 
-    merged = {
+    return {
         **base,
         "choices": choices,
     }
-
-    return merged

client-libs/python/openpipe/openai.py

@@ -3,9 +3,16 @@ from openai.openai_object import OpenAIObject
 import time
 import inspect
 
-from openpipe.merge_openai_chunks import merge_streamed_chunks
+from openpipe.merge_openai_chunks import merge_openai_chunks
+from openpipe.openpipe_meta import OpenPipeMeta
 
-from .shared import maybe_check_cache, maybe_check_cache_async, report_async, report
+from .shared import (
+    _should_check_cache,
+    maybe_check_cache,
+    maybe_check_cache_async,
+    report_async,
+    report,
+)
 
 
 class WrappedChatCompletion(original_openai.ChatCompletion):
@@ -29,9 +36,15 @@ class WrappedChatCompletion(original_openai.ChatCompletion):
                 def _gen():
                     assembled_completion = None
                     for chunk in chat_completion:
-                        assembled_completion = merge_streamed_chunks(
+                        assembled_completion = merge_openai_chunks(
                             assembled_completion, chunk
                         )
+
+                        cache_status = (
+                            "MISS" if _should_check_cache(openpipe_options) else "SKIP"
+                        )
+                        chunk.openpipe = OpenPipeMeta(cache_status=cache_status)
+
                         yield chunk
 
                     received_at = int(time.time() * 1000)
@@ -58,6 +71,10 @@ class WrappedChatCompletion(original_openai.ChatCompletion):
                     status_code=200,
                 )
 
+                cache_status = (
+                    "MISS" if _should_check_cache(openpipe_options) else "SKIP"
+                )
+                chat_completion["openpipe"] = OpenPipeMeta(cache_status=cache_status)
             return chat_completion
         except Exception as e:
             received_at = int(time.time() * 1000)
@@ -96,21 +113,28 @@ class WrappedChatCompletion(original_openai.ChatCompletion):
         requested_at = int(time.time() * 1000)
 
         try:
-            chat_completion = original_openai.ChatCompletion.acreate(*args, **kwargs)
+            chat_completion = await original_openai.ChatCompletion.acreate(
+                *args, **kwargs
+            )
 
-            if inspect.isgenerator(chat_completion):
+            if inspect.isasyncgen(chat_completion):
 
-                def _gen():
+                async def _gen():
                     assembled_completion = None
-                    for chunk in chat_completion:
-                        assembled_completion = merge_streamed_chunks(
+                    async for chunk in chat_completion:
+                        assembled_completion = merge_openai_chunks(
                             assembled_completion, chunk
                         )
+                        cache_status = (
+                            "MISS" if _should_check_cache(openpipe_options) else "SKIP"
+                        )
+                        chunk.openpipe = OpenPipeMeta(cache_status=cache_status)
+
                         yield chunk
 
                     received_at = int(time.time() * 1000)
 
-                    report_async(
+                    await report_async(
                         openpipe_options=openpipe_options,
                         requested_at=requested_at,
                         received_at=received_at,
@@ -123,7 +147,7 @@ class WrappedChatCompletion(original_openai.ChatCompletion):
             else:
                 received_at = int(time.time() * 1000)
 
-                report_async(
+                await report_async(
                     openpipe_options=openpipe_options,
                     requested_at=requested_at,
                     received_at=received_at,
@@ -132,12 +156,17 @@ class WrappedChatCompletion(original_openai.ChatCompletion):
                     status_code=200,
                 )
 
+                cache_status = (
+                    "MISS" if _should_check_cache(openpipe_options) else "SKIP"
+                )
+                chat_completion["openpipe"] = OpenPipeMeta(cache_status=cache_status)
+
             return chat_completion
         except Exception as e:
             received_at = int(time.time() * 1000)
 
             if isinstance(e, original_openai.OpenAIError):
-                report_async(
+                await report_async(
                     openpipe_options=openpipe_options,
                     requested_at=requested_at,
                     received_at=received_at,
@@ -147,7 +176,7 @@ class WrappedChatCompletion(original_openai.ChatCompletion):
                     status_code=e.http_status,
                 )
             else:
-                report_async(
+                await report_async(
                     openpipe_options=openpipe_options,
                     requested_at=requested_at,
                     received_at=received_at,

client-libs/python/openpipe/openpipe_meta.py

@@ -0,0 +1,7 @@
+from attr import dataclass
+
+
+@dataclass
+class OpenPipeMeta:
+    # Cache status. One of 'HIT', 'MISS', 'SKIP'
+    cache_status: str

client-libs/python/openpipe/shared.py

@@ -1,10 +1,10 @@
 from openpipe.api_client.api.default import (
-    external_api_report,
-    external_api_check_cache,
+    report as api_report,
+    check_cache,
 )
 from openpipe.api_client.client import AuthenticatedClient
-from openpipe.api_client.models.external_api_report_json_body_tags import (
-    ExternalApiReportJsonBodyTags,
+from openpipe.api_client.models.report_json_body_tags import (
+    ReportJsonBodyTags,
 )
 import toml
 import time
@@ -19,9 +19,9 @@ configured_client = AuthenticatedClient(
 def _get_tags(openpipe_options):
     tags = openpipe_options.get("tags") or {}
     tags["$sdk"] = "python"
-    tags["$sdk_version"] = version
+    tags["$sdk.version"] = version
 
-    return ExternalApiReportJsonBodyTags.from_dict(tags)
+    return ReportJsonBodyTags.from_dict(tags)
 
 
 def _should_check_cache(openpipe_options):
@@ -31,7 +31,7 @@ def _should_check_cache(openpipe_options):
 
 
 def _process_cache_payload(
-    payload: external_api_check_cache.ExternalApiCheckCacheResponse200,
+    payload: check_cache.CheckCacheResponse200,
 ):
     if not payload or not payload.resp_payload:
         return None
@@ -47,9 +47,9 @@ def maybe_check_cache(
     if not _should_check_cache(openpipe_options):
         return None
     try:
-        payload = external_api_check_cache.sync(
+        payload = check_cache.sync(
             client=configured_client,
-            json_body=external_api_check_cache.ExternalApiCheckCacheJsonBody(
+            json_body=check_cache.CheckCacheJsonBody(
                 req_payload=req_payload,
                 requested_at=int(time.time() * 1000),
                 tags=_get_tags(openpipe_options),
@@ -72,9 +72,9 @@ async def maybe_check_cache_async(
         return None
 
     try:
-        payload = await external_api_check_cache.asyncio(
+        payload = await check_cache.asyncio(
             client=configured_client,
-            json_body=external_api_check_cache.ExternalApiCheckCacheJsonBody(
+            json_body=check_cache.CheckCacheJsonBody(
                 req_payload=req_payload,
                 requested_at=int(time.time() * 1000),
                 tags=_get_tags(openpipe_options),
@@ -94,9 +94,9 @@ def report(
     **kwargs,
 ):
     try:
-        external_api_report.sync_detailed(
+        api_report.sync_detailed(
             client=configured_client,
-            json_body=external_api_report.ExternalApiReportJsonBody(
+            json_body=api_report.ReportJsonBody(
                 **kwargs,
                 tags=_get_tags(openpipe_options),
             ),
@@ -112,9 +112,9 @@ async def report_async(
     **kwargs,
 ):
     try:
-        await external_api_report.asyncio_detailed(
+        await api_report.asyncio_detailed(
             client=configured_client,
-            json_body=external_api_report.ExternalApiReportJsonBody(
+            json_body=api_report.ReportJsonBody(
                 **kwargs,
                 tags=_get_tags(openpipe_options),
             ),

client-libs/python/openpipe/test_client.py

@@ -1,55 +1,106 @@
+from functools import reduce
 from dotenv import load_dotenv
-from . import openai, configure_openpipe
 import os
 import pytest
+from . import openai, configure_openpipe, configured_client
+from .api_client.api.default import local_testing_only_get_latest_logged_call
+from .merge_openai_chunks import merge_openai_chunks
+import random
+import string
+
+
+def random_string(length):
+    letters = string.ascii_lowercase
+    return "".join(random.choice(letters) for i in range(length))
+
 
 load_dotenv()
 
 openai.api_key = os.getenv("OPENAI_API_KEY")
 
 configure_openpipe(
-    base_url="http://localhost:3000/api", api_key=os.getenv("OPENPIPE_API_KEY")
+    base_url="http://localhost:3000/api/v1", api_key=os.getenv("OPENPIPE_API_KEY")
 )
 
 
+def last_logged_call():
+    return local_testing_only_get_latest_logged_call.sync(client=configured_client)
+
+
 def test_sync():
     completion = openai.ChatCompletion.create(
         model="gpt-3.5-turbo",
-        messages=[{"role": "system", "content": "count to 10"}],
+        messages=[{"role": "system", "content": "count to 3"}],
     )
 
-    print(completion.choices[0].message.content)
+    last_logged = last_logged_call()
+    assert (
+        last_logged.model_response.resp_payload["choices"][0]["message"]["content"]
+        == completion.choices[0].message.content
+    )
+    assert (
+        last_logged.model_response.req_payload["messages"][0]["content"] == "count to 3"
+    )
+
+    assert completion.openpipe.cache_status == "SKIP"
 
 
 def test_streaming():
     completion = openai.ChatCompletion.create(
         model="gpt-3.5-turbo",
-        messages=[{"role": "system", "content": "count to 10"}],
+        messages=[{"role": "system", "content": "count to 4"}],
         stream=True,
     )
 
-    for chunk in completion:
-        print(chunk)
+    merged = reduce(merge_openai_chunks, completion, None)
+    last_logged = last_logged_call()
+    assert (
+        last_logged.model_response.resp_payload["choices"][0]["message"]["content"]
+        == merged["choices"][0]["message"]["content"]
+    )
 
 
 async def test_async():
-    acompletion = await openai.ChatCompletion.acreate(
+    completion = await openai.ChatCompletion.acreate(
         model="gpt-3.5-turbo",
         messages=[{"role": "user", "content": "count down from 5"}],
     )
+    last_logged = last_logged_call()
+    assert (
+        last_logged.model_response.resp_payload["choices"][0]["message"]["content"]
+        == completion.choices[0].message.content
+    )
+    assert (
+        last_logged.model_response.req_payload["messages"][0]["content"]
+        == "count down from 5"
+    )
 
-    print(acompletion.choices[0].message.content)
+    assert completion.openpipe.cache_status == "SKIP"
 
 
 async def test_async_streaming():
-    acompletion = await openai.ChatCompletion.acreate(
+    completion = await openai.ChatCompletion.acreate(
         model="gpt-3.5-turbo",
         messages=[{"role": "user", "content": "count down from 5"}],
         stream=True,
     )
 
-    async for chunk in acompletion:
-        print(chunk)
+    merged = None
+    async for chunk in completion:
+        assert chunk.openpipe.cache_status == "SKIP"
+        merged = merge_openai_chunks(merged, chunk)
+
+    last_logged = last_logged_call()
+
+    assert (
+        last_logged.model_response.resp_payload["choices"][0]["message"]["content"]
+        == merged["choices"][0]["message"]["content"]
+    )
+    assert (
+        last_logged.model_response.req_payload["messages"][0]["content"]
+        == "count down from 5"
+    )
+    assert merged["openpipe"].cache_status == "SKIP"
 
 
 def test_sync_with_tags():
@@ -58,31 +109,54 @@ def test_sync_with_tags():
         messages=[{"role": "system", "content": "count to 10"}],
         openpipe={"tags": {"promptId": "testprompt"}},
     )
-    print("finished")
 
-    print(completion.choices[0].message.content)
+    last_logged = last_logged_call()
+    assert (
+        last_logged.model_response.resp_payload["choices"][0]["message"]["content"]
+        == completion.choices[0].message.content
+    )
+    print(last_logged.tags)
+    assert last_logged.tags["promptId"] == "testprompt"
+    assert last_logged.tags["$sdk"] == "python"
 
 
 def test_bad_call():
-    completion = openai.ChatCompletion.create(
-        model="gpt-3.5-turbo-blaster",
-        messages=[{"role": "system", "content": "count to 10"}],
-        stream=True,
+    try:
+        completion = openai.ChatCompletion.create(
+            model="gpt-3.5-turbo-blaster",
+            messages=[{"role": "system", "content": "count to 10"}],
+            stream=True,
+        )
+        assert False
+    except Exception as e:
+        pass
+    last_logged = last_logged_call()
+    print(last_logged)
+    assert (
+        last_logged.model_response.error_message
+        == "The model `gpt-3.5-turbo-blaster` does not exist"
     )
+    assert last_logged.model_response.status_code == 404
 
 
-@pytest.mark.focus
 async def test_caching():
+    messages = [{"role": "system", "content": f"repeat '{random_string(10)}'"}]
     completion = openai.ChatCompletion.create(
         model="gpt-3.5-turbo",
-        messages=[{"role": "system", "content": "count to 10"}],
+        messages=messages,
         openpipe={"cache": True},
     )
+    assert completion.openpipe.cache_status == "MISS"
+
+    first_logged = last_logged_call()
+    assert (
+        completion.choices[0].message.content
+        == first_logged.model_response.resp_payload["choices"][0]["message"]["content"]
+    )
 
     completion2 = await openai.ChatCompletion.acreate(
         model="gpt-3.5-turbo",
-        messages=[{"role": "system", "content": "count to 10"}],
+        messages=messages,
         openpipe={"cache": True},
     )
-
-    print(completion2)
+    assert completion2.openpipe.cache_status == "HIT"

client-libs/typescript/index.ts

@@ -1,3 +1,3 @@
 // main.ts or index.ts at the root level
-export * as OpenAI from './openai';
-export * as OpenAILegacy from './openai-legacy';
+export * as OpenAI from "./src/openai";
+export * as OpenAILegacy from "./src/openai-legacy";

client-libs/typescript/package.json

@@ -4,7 +4,8 @@
   "type": "module",
   "description": "Metrics and auto-evaluation for LLM calls",
   "scripts": {
-    "build": "tsc"
+    "build": "tsc",
+    "test": "vitest"
   },
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -12,14 +13,18 @@
   "author": "",
   "license": "Apache-2.0",
   "dependencies": {
-    "axios": "^0.26.0",
+    "form-data": "^4.0.0",
+    "lodash-es": "^4.17.21",
+    "node-fetch": "^3.3.2",
     "openai-beta": "npm:openai@4.0.0-beta.7",
     "openai-legacy": "npm:openai@3.3.0"
   },
   "devDependencies": {
+    "@types/lodash-es": "^4.17.8",
     "@types/node": "^20.4.8",
     "dotenv": "^16.3.1",
     "tsx": "^3.12.7",
-    "typescript": "^5.0.4"
+    "typescript": "^5.0.4",
+    "vitest": "^0.33.0"
   }
 }

client-libs/typescript/src/codegen/.gitignore

@@ -1,4 +0,0 @@
-wwwroot/*.js
-node_modules
-typings
-dist

client-libs/typescript/src/codegen/.npmignore

@@ -1 +0,0 @@
-# empty npmignore to ensure all required files (e.g., in the dist folder) are published by npm

client-libs/typescript/src/codegen/.openapi-generator-ignore

@@ -1,23 +0,0 @@
-# OpenAPI Generator Ignore
-# Generated by openapi-generator https://github.com/openapitools/openapi-generator
-
-# Use this file to prevent files from being overwritten by the generator.
-# The patterns follow closely to .gitignore or .dockerignore.
-
-# As an example, the C# client generator defines ApiClient.cs.
-# You can make changes and tell OpenAPI Generator to ignore just this file by uncommenting the following line:
-#ApiClient.cs
-
-# You can match any string of characters against a directory, file or extension with a single asterisk (*):
-#foo/*/qux
-# The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux
-
-# You can recursively match patterns against a directory, file or extension with a double asterisk (**):
-#foo/**/qux
-# This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux
-
-# You can also negate patterns with an exclamation (!).
-# For example, you can ignore all files in a docs folder with the file extension .md:
-#docs/*.md
-# Then explicitly reverse the ignore rule for a single file:
-#!docs/README.md

client-libs/typescript/src/codegen/.openapi-generator/FILES

@@ -1,9 +0,0 @@
-.gitignore
-.npmignore
-.openapi-generator-ignore
-api.ts
-base.ts
-common.ts
-configuration.ts
-git_push.sh
-index.ts

client-libs/typescript/src/codegen/.openapi-generator/VERSION

@@ -1 +0,0 @@
-6.6.0