make max pool size configurable

Previously, an unhandled promise rejection in the background worker would crash the process. This way we log it and don't crash.

app/.env.example

@@ -34,3 +34,9 @@ GITHUB_CLIENT_SECRET="your_secret"
 
 OPENPIPE_BASE_URL="http://localhost:3000/api/v1"
 OPENPIPE_API_KEY="your_key"
+
+SENDER_EMAIL="placeholder"
+SMTP_HOST="placeholder"
+SMTP_PORT="placeholder"
+SMTP_LOGIN="placeholder"
+SMTP_PASSWORD="placeholder"

app/@types/nextjs-routes.d.ts

@@ -12,6 +12,7 @@ declare module "nextjs-routes" {
 
   export type Route =
     | StaticRoute<"/account/signin">
+    | StaticRoute<"/admin/jobs">
     | DynamicRoute<"/api/auth/[...nextauth]", { "nextauth": string[] }>
     | StaticRoute<"/api/experiments/og-image">
     | DynamicRoute<"/api/trpc/[trpc]", { "trpc": string }>
@@ -20,9 +21,10 @@ declare module "nextjs-routes" {
     | StaticRoute<"/dashboard">
     | DynamicRoute<"/data/[id]", { "id": string }>
     | StaticRoute<"/data">
-    | DynamicRoute<"/experiments/[id]", { "id": string }>
+    | DynamicRoute<"/experiments/[experimentSlug]", { "experimentSlug": string }>
     | StaticRoute<"/experiments">
     | StaticRoute<"/">
+    | DynamicRoute<"/invitations/[invitationToken]", { "invitationToken": string }>
     | StaticRoute<"/project/settings">
     | StaticRoute<"/request-logs">
     | StaticRoute<"/sentry-example-page">

app/Dockerfile

@@ -45,4 +45,4 @@ EXPOSE 3000
 ENV PORT 3000
 
 # Run the "run-prod.sh" script
-CMD /code/app/run-prod.sh
+CMD /code/app/scripts/run-prod.sh

app/package.json

@@ -12,12 +12,13 @@
     "build": "next build",
     "dev:next": "TZ=UTC next dev",
     "dev:wss": "pnpm tsx --watch src/wss-server.ts",
-    "dev:worker": "NODE_ENV='development' pnpm tsx --watch src/server/tasks/worker.ts",
-    "dev": "concurrently --kill-others 'pnpm dev:next' 'pnpm dev:wss' 'pnpm dev:worker'",
+    "worker": "NODE_ENV='development' pnpm tsx --watch src/server/tasks/worker.ts",
+    "dev": "concurrently --kill-others 'pnpm dev:next' 'pnpm dev:wss' 'pnpm worker --watch'",
     "postinstall": "prisma generate",
     "lint": "next lint",
     "start": "TZ=UTC next start",
     "codegen:clients": "tsx src/server/scripts/client-codegen.ts",
+    "codegen:db": "prisma generate && kysely-codegen --dialect postgres --out-file src/server/db.types.ts",
     "seed": "tsx prisma/seed.ts",
     "check": "concurrently 'pnpm lint' 'pnpm tsc' 'pnpm prettier . --check'",
     "test": "pnpm vitest"
@@ -37,6 +38,7 @@
     "@monaco-editor/loader": "^1.3.3",
     "@next-auth/prisma-adapter": "^1.0.5",
     "@prisma/client": "^4.14.0",
+    "@sendinblue/client": "^3.3.1",
     "@sentry/nextjs": "^7.61.0",
     "@t3-oss/env-nextjs": "^0.3.1",
     "@tabler/icons-react": "^2.22.0",
@@ -64,13 +66,16 @@
     "json-stringify-pretty-compact": "^4.0.0",
     "jsonschema": "^1.4.1",
     "kysely": "^0.26.1",
+    "kysely-codegen": "^0.10.1",
     "lodash-es": "^4.17.21",
     "lucide-react": "^0.265.0",
+    "marked": "^7.0.3",
     "next": "^13.4.2",
     "next-auth": "^4.22.1",
     "next-query-params": "^4.2.3",
     "nextjs-cors": "^2.1.2",
     "nextjs-routes": "^2.0.1",
+    "nodemailer": "^6.9.4",
     "openai": "4.0.0-beta.7",
     "openpipe": "workspace:*",
     "pg": "^8.11.2",
@@ -114,6 +119,7 @@
     "@types/json-schema": "^7.0.12",
     "@types/lodash-es": "^4.17.8",
     "@types/node": "^18.16.0",
+    "@types/nodemailer": "^6.4.9",
     "@types/pg": "^8.10.2",
     "@types/pluralize": "^0.0.30",
     "@types/prismjs": "^1.26.0",

app/prisma/migrations/20230815234811_add_user_invitations/migration.sql

@@ -0,0 +1,25 @@
+-- CreateTable
+CREATE TABLE "UserInvitation" (
+    "id" UUID NOT NULL,
+    "projectId" UUID NOT NULL,
+    "email" TEXT NOT NULL,
+    "role" "ProjectUserRole" NOT NULL,
+    "invitationToken" TEXT NOT NULL,
+    "senderId" UUID NOT NULL,
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "UserInvitation_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "UserInvitation_invitationToken_key" ON "UserInvitation"("invitationToken");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "UserInvitation_projectId_email_key" ON "UserInvitation"("projectId", "email");
+
+-- AddForeignKey
+ALTER TABLE "UserInvitation" ADD CONSTRAINT "UserInvitation_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "UserInvitation" ADD CONSTRAINT "UserInvitation_senderId_fkey" FOREIGN KEY ("senderId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE;

app/prisma/migrations/20230818055303_add_slug_to_experiment/migration.sql

@@ -0,0 +1,88 @@
+/*
+ * Copyright 2023 Viascom Ltd liab. Co
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+CREATE EXTENSION IF NOT EXISTS pgcrypto;
+
+CREATE OR REPLACE FUNCTION nanoid(
+    size int DEFAULT 21,
+    alphabet text DEFAULT '_-0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
+)
+    RETURNS text
+    LANGUAGE plpgsql
+    volatile
+AS
+$$
+DECLARE
+    idBuilder      text := '';
+    counter        int  := 0;
+    bytes          bytea;
+    alphabetIndex  int;
+    alphabetArray  text[];
+    alphabetLength int;
+    mask           int;
+    step           int;
+BEGIN
+    alphabetArray := regexp_split_to_array(alphabet, '');
+    alphabetLength := array_length(alphabetArray, 1);
+    mask := (2 << cast(floor(log(alphabetLength - 1) / log(2)) as int)) - 1;
+    step := cast(ceil(1.6 * mask * size / alphabetLength) AS int);
+
+    while true
+        loop
+            bytes := gen_random_bytes(step);
+            while counter < step
+                loop
+                    alphabetIndex := (get_byte(bytes, counter) & mask) + 1;
+                    if alphabetIndex <= alphabetLength then
+                        idBuilder := idBuilder || alphabetArray[alphabetIndex];
+                        if length(idBuilder) = size then
+                            return idBuilder;
+                        end if;
+                    end if;
+                    counter := counter + 1;
+                end loop;
+
+            counter := 0;
+        end loop;
+END
+$$;
+
+
+-- Make a short_nanoid function that uses the default alphabet and length of 15
+CREATE OR REPLACE FUNCTION short_nanoid()
+    RETURNS text
+    LANGUAGE plpgsql
+    volatile
+AS
+$$
+BEGIN
+    RETURN nanoid(15, '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ');
+END
+$$;
+
+-- AlterTable
+ALTER TABLE "Experiment" ADD COLUMN     "slug" TEXT NOT NULL DEFAULT short_nanoid();
+
+-- For existing experiments, keep the existing id as the slug for backwards compatibility
+UPDATE "Experiment" SET "slug" = "id";
+
+-- CreateIndex
+CREATE UNIQUE INDEX "Experiment_slug_key" ON "Experiment"("slug");

app/prisma/schema.prisma

@@ -11,7 +11,9 @@ datasource db {
 }
 
 model Experiment {
-    id    String @id @default(uuid()) @db.Uuid
+    id String @id @default(uuid()) @db.Uuid
+
+    slug  String @unique @default(dbgenerated("short_nanoid()"))
     label String
 
     sortIndex Int @default(0)
@@ -207,13 +209,14 @@ model Project {
     personalProjectUserId String? @unique @db.Uuid
     personalProjectUser   User?   @relation(fields: [personalProjectUserId], references: [id], onDelete: Cascade)
 
-    createdAt    DateTime      @default(now())
-    updatedAt    DateTime      @updatedAt
-    projectUsers ProjectUser[]
-    experiments  Experiment[]
-    datasets     Dataset[]
-    loggedCalls  LoggedCall[]
-    apiKeys      ApiKey[]
+    createdAt              DateTime         @default(now())
+    updatedAt              DateTime         @updatedAt
+    projectUsers           ProjectUser[]
+    projectUserInvitations UserInvitation[]
+    experiments            Experiment[]
+    datasets               Dataset[]
+    loggedCalls            LoggedCall[]
+    apiKeys                ApiKey[]
 }
 
 enum ProjectUserRole {
@@ -323,10 +326,10 @@ model LoggedCallModelResponse {
 }
 
 model LoggedCallTag {
-    id    String  @id @default(uuid()) @db.Uuid
-    name  String
-    value String?
-    projectId String   @db.Uuid
+    id        String  @id @default(uuid()) @db.Uuid
+    name      String
+    value     String?
+    projectId String  @db.Uuid
 
     loggedCallId String     @db.Uuid
     loggedCall   LoggedCall @relation(fields: [loggedCallId], references: [id], onDelete: Cascade)
@@ -390,16 +393,33 @@ model User {
 
     role UserRole @default(USER)
 
-    accounts          Account[]
-    sessions          Session[]
-    projectUsers      ProjectUser[]
-    projects          Project[]
-    worldChampEntrant WorldChampEntrant?
+    accounts            Account[]
+    sessions            Session[]
+    projectUsers        ProjectUser[]
+    projects            Project[]
+    worldChampEntrant   WorldChampEntrant?
+    sentUserInvitations UserInvitation[]
 
     createdAt DateTime @default(now())
     updatedAt DateTime @default(now()) @updatedAt
 }
 
+model UserInvitation {
+    id String @id @default(uuid()) @db.Uuid
+
+    projectId       String          @db.Uuid
+    project         Project         @relation(fields: [projectId], references: [id], onDelete: Cascade)
+    email           String
+    role            ProjectUserRole
+    invitationToken String          @unique
+    senderId        String          @db.Uuid
+    sender          User            @relation(fields: [senderId], references: [id], onDelete: Cascade)
+    createdAt       DateTime        @default(now())
+    updatedAt       DateTime        @updatedAt
+
+    @@unique([projectId, email])
+}
+
 model VerificationToken {
     identifier String
     token      String   @unique

app/prisma/seed.ts

@@ -10,6 +10,14 @@ await prisma.project.deleteMany({
   where: { id: defaultId },
 });
 
+// Mark all users as admins
+await prisma.user.updateMany({
+  where: {},
+  data: {
+    role: "ADMIN",
+  },
+});
+
 // If there's an existing project, just seed into it
 const project =
   (await prisma.project.findFirst({})) ??
@@ -18,12 +26,16 @@ const project =
   }));
 
 if (env.OPENPIPE_API_KEY) {
-  await prisma.apiKey.create({
-    data: {
+  await prisma.apiKey.upsert({
+    where: {
+      apiKey: env.OPENPIPE_API_KEY,
+    },
+    create: {
       projectId: project.id,
       name: "Default API Key",
       apiKey: env.OPENPIPE_API_KEY,
     },
+    update: {},
   });
 }
 

app/scripts/debug-prod.sh

@@ -0,0 +1,6 @@
+#! /bin/bash
+
+set -e
+cd "$(dirname "$0")/.."
+apt-get update
+apt-get install -y htop psql

app/scripts/run-prod.sh

@@ -10,6 +10,4 @@ pnpm tsx src/promptConstructor/migrate.ts
 
 echo "Starting the server"
 
-pnpm concurrently --kill-others \
-  "pnpm start" \
-  "pnpm tsx src/server/tasks/worker.ts"
+pnpm start

app/scripts/run-workers-prod.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+set -e
+
+echo "Migrating the database"
+pnpm prisma migrate deploy
+
+echo "Starting 4 workers"
+
+pnpm concurrently "pnpm worker" "pnpm worker" "pnpm worker" "pnpm worker"

app/scripts/test-docker.sh

@@ -0,0 +1,13 @@
+#! /bin/bash
+
+set -e
+
+cd "$(dirname "$0")/../.."
+
+echo "Env is"
+echo $ENVIRONMENT
+
+docker build . --file app/Dockerfile --tag "openpipe-prod"
+
+# Run the image
+docker run --env-file app/.env -it --entrypoint "/bin/bash" "openpipe-prod"

app/sentry.server.config.ts

@@ -3,6 +3,7 @@
 // https://docs.sentry.io/platforms/javascript/guides/nextjs/
 
 import * as Sentry from "@sentry/nextjs";
+import { isError } from "lodash-es";
 import { env } from "~/env.mjs";
 
 if (env.NEXT_PUBLIC_SENTRY_DSN) {
@@ -15,4 +16,10 @@ if (env.NEXT_PUBLIC_SENTRY_DSN) {
     // Setting this option to true will print useful information to the console while you're setting up Sentry.
     debug: false,
   });
+} else {
+  // Install local debug exception handler for rejected promises
+  process.on("unhandledRejection", (reason) => {
+    const reasonDetails = isError(reason) ? reason?.stack : reason;
+    console.log("Unhandled Rejection at:", reasonDetails);
+  });
 }

app/src/components/AutoResizeTextArea.tsx

@@ -1,13 +1,13 @@
 import { Textarea, type TextareaProps } from "@chakra-ui/react";
 import ResizeTextarea from "react-textarea-autosize";
-import React, { useLayoutEffect, useState } from "react";
+import React, { useEffect, useState } from "react";
 
 export const AutoResizeTextarea: React.ForwardRefRenderFunction<
   HTMLTextAreaElement,
   TextareaProps & { minRows?: number }
 > = ({ minRows = 1, overflowY = "hidden", ...props }, ref) => {
   const [isRerendered, setIsRerendered] = useState(false);
-  useLayoutEffect(() => setIsRerendered(true), []);
+  useEffect(() => setIsRerendered(true), []);
 
   return (
     <Textarea

app/src/components/ChangeModelModal/ModelStatsCard.tsx

@@ -87,7 +87,7 @@ export const ModelStatsCard = ({
               label="Price"
               info={
                 <Text>
-                  ${model.pricePerSecond.toFixed(3)}
+                  ${model.pricePerSecond.toFixed(4)}
                   <Text color="gray.500"> / second</Text>
                 </Text>
               }

app/src/components/CopiableCode.tsx

@@ -1,15 +1,16 @@
-import { HStack, Icon, IconButton, Tooltip, Text } from "@chakra-ui/react";
+import { HStack, Icon, IconButton, Tooltip, Text, type StackProps } from "@chakra-ui/react";
 import { useState } from "react";
 import { MdContentCopy } from "react-icons/md";
 import { useHandledAsyncCallback } from "~/utils/hooks";
 
-const CopiableCode = ({ code }: { code: string }) => {
+const CopiableCode = ({ code, ...rest }: { code: string } & StackProps) => {
   const [copied, setCopied] = useState(false);
 
   const [copyToClipboard] = useHandledAsyncCallback(async () => {
     await navigator.clipboard.writeText(code);
     setCopied(true);
   }, [code]);
+
   return (
     <HStack
       backgroundColor="blackAlpha.800"
@@ -18,9 +19,19 @@ const CopiableCode = ({ code }: { code: string }) => {
       padding={3}
       w="full"
       justifyContent="space-between"
+      alignItems="flex-start"
+      {...rest}
     >
-      <Text fontFamily="inconsolata" fontWeight="bold" letterSpacing={0.5} overflowX="auto">
+      <Text
+        fontFamily="inconsolata"
+        fontWeight="bold"
+        letterSpacing={0.5}
+        overflowX="auto"
+        whiteSpace="pre-wrap"
+      >
         {code}
+        {/* Necessary for trailing newline to actually be displayed */}
+        {code.endsWith("\n") ? "\n" : ""}
       </Text>
       <Tooltip closeOnClick={false} label={copied ? "Copied!" : "Copy to clipboard"}>
         <IconButton

app/src/components/OutputsTable/AddVariantButton.tsx

@@ -8,7 +8,7 @@ import {
   useHandledAsyncCallback,
   useVisibleScenarioIds,
 } from "~/utils/hooks";
-import { cellPadding } from "../constants";
+import { cellPadding } from "./constants";
 import { ActionButton } from "./ScenariosHeader";
 
 export default function AddVariantButton() {

app/src/components/OutputsTable/OutputCell/CellOptions.tsx

@@ -1,7 +1,7 @@
 import { HStack, Icon, IconButton, Spinner, Tooltip, useDisclosure } from "@chakra-ui/react";
 import { BsArrowClockwise, BsInfoCircle } from "react-icons/bs";
 import { useExperimentAccess } from "~/utils/hooks";
-import ExpandedModal from "./PromptModal";
+import PromptModal from "./PromptModal";
 import { type RouterOutputs } from "~/utils/api";
 
 export const CellOptions = ({
@@ -32,7 +32,7 @@ export const CellOptions = ({
               variant="ghost"
             />
           </Tooltip>
-          <ExpandedModal cell={cell} disclosure={modalDisclosure} />
+          <PromptModal cell={cell} disclosure={modalDisclosure} />
         </>
       )}
       {canModify && (

app/src/components/OutputsTable/OutputCell/CellWrapper.tsx

@@ -0,0 +1,29 @@
+import { type StackProps, VStack } from "@chakra-ui/react";
+import { type RouterOutputs } from "~/utils/api";
+import { type Scenario } from "../types";
+import { CellOptions } from "./CellOptions";
+import { OutputStats } from "./OutputStats";
+
+const CellWrapper: React.FC<
+  StackProps & {
+    cell: RouterOutputs["scenarioVariantCells"]["get"] | undefined;
+    hardRefetching: boolean;
+    hardRefetch: () => void;
+    mostRecentResponse:
+      | NonNullable<RouterOutputs["scenarioVariantCells"]["get"]>["modelResponses"][0]
+      | undefined;
+    scenario: Scenario;
+  }
+> = ({ children, cell, hardRefetching, hardRefetch, mostRecentResponse, scenario, ...props }) => (
+  <VStack w="full" alignItems="flex-start" {...props} px={2} py={2} h="100%">
+    {cell && (
+      <CellOptions refetchingOutput={hardRefetching} refetchOutput={hardRefetch} cell={cell} />
+    )}
+    <VStack w="full" alignItems="flex-start" maxH={500} overflowY="auto" flex={1}>
+      {children}
+    </VStack>
+    {mostRecentResponse && <OutputStats modelResponse={mostRecentResponse} scenario={scenario} />}
+  </VStack>
+);
+
+export default CellWrapper;

app/src/components/OutputsTable/OutputCell/OutputCell.tsx

@@ -1,17 +1,16 @@
-import { api } from "~/utils/api";
-import { type PromptVariant, type Scenario } from "../types";
-import { type StackProps, Text, VStack } from "@chakra-ui/react";
-import { useScenarioVars, useHandledAsyncCallback } from "~/utils/hooks";
+import { Text } from "@chakra-ui/react";
+import stringify from "json-stringify-pretty-compact";
+import { Fragment, useEffect, useState, type ReactElement } from "react";
 import SyntaxHighlighter from "react-syntax-highlighter";
 import { docco } from "react-syntax-highlighter/dist/cjs/styles/hljs";
-import stringify from "json-stringify-pretty-compact";
-import { type ReactElement, useState, useEffect, Fragment, useCallback } from "react";
-import useSocket from "~/utils/useSocket";
-import { OutputStats } from "./OutputStats";
-import { RetryCountdown } from "./RetryCountdown";
 import frontendModelProviders from "~/modelProviders/frontendModelProviders";
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback, useScenarioVars } from "~/utils/hooks";
+import useSocket from "~/utils/useSocket";
+import { type PromptVariant, type Scenario } from "../types";
+import CellWrapper from "./CellWrapper";
 import { ResponseLog } from "./ResponseLog";
-import { CellOptions } from "./TopActions";
+import { RetryCountdown } from "./RetryCountdown";
 
 const WAITING_MESSAGE_INTERVAL = 20000;
 
@@ -44,7 +43,7 @@ export default function OutputCell({
 
   type OutputSchema = Parameters<typeof provider.normalizeOutput>[0];
 
-  const { mutateAsync: hardRefetchMutate } = api.scenarioVariantCells.forceRefetch.useMutation();
+  const { mutateAsync: hardRefetchMutate } = api.scenarioVariantCells.hardRefetch.useMutation();
   const [hardRefetch, hardRefetching] = useHandledAsyncCallback(async () => {
     await hardRefetchMutate({ scenarioId: scenario.id, variantId: variant.id });
     await utils.scenarioVariantCells.get.invalidate({
@@ -72,35 +71,26 @@ export default function OutputCell({
 
   const mostRecentResponse = cell?.modelResponses[cell.modelResponses.length - 1];
 
-  const CellWrapper = useCallback(
-    ({ children, ...props }: StackProps) => (
-      <VStack w="full" alignItems="flex-start" {...props} px={2} py={2} h="100%">
-        {cell && (
-          <CellOptions refetchingOutput={hardRefetching} refetchOutput={hardRefetch} cell={cell} />
-        )}
-        <VStack w="full" alignItems="flex-start" maxH={500} overflowY="auto" flex={1}>
-          {children}
-        </VStack>
-        {mostRecentResponse && (
-          <OutputStats modelResponse={mostRecentResponse} scenario={scenario} />
-        )}
-      </VStack>
-    ),
-    [hardRefetching, hardRefetch, mostRecentResponse, scenario, cell],
-  );
+  const wrapperProps: Parameters<typeof CellWrapper>[0] = {
+    cell,
+    hardRefetching,
+    hardRefetch,
+    mostRecentResponse,
+    scenario,
+  };
 
   if (!vars) return null;
 
   if (!cell && !fetchingOutput)
     return (
-      <CellWrapper>
+      <CellWrapper {...wrapperProps}>
         <Text color="gray.500">Error retrieving output</Text>
       </CellWrapper>
     );
 
   if (cell && cell.errorMessage) {
     return (
-      <CellWrapper>
+      <CellWrapper {...wrapperProps}>
         <Text color="red.500">{cell.errorMessage}</Text>
       </CellWrapper>
     );
@@ -112,7 +102,12 @@ export default function OutputCell({
 
   if (showLogs)
     return (
-      <CellWrapper alignItems="flex-start" fontFamily="inconsolata, monospace" spacing={0}>
+      <CellWrapper
+        {...wrapperProps}
+        alignItems="flex-start"
+        fontFamily="inconsolata, monospace"
+        spacing={0}
+      >
         {cell?.jobQueuedAt && <ResponseLog time={cell.jobQueuedAt} title="Job queued" />}
         {cell?.jobStartedAt && <ResponseLog time={cell.jobStartedAt} title="Job started" />}
         {cell?.modelResponses?.map((response) => {
@@ -174,7 +169,7 @@ export default function OutputCell({
 
   if (mostRecentResponse?.respPayload && normalizedOutput?.type === "json") {
     return (
-      <CellWrapper>
+      <CellWrapper {...wrapperProps}>
         <SyntaxHighlighter
           customStyle={{ overflowX: "unset", width: "100%", flex: 1 }}
           language="json"
@@ -193,7 +188,7 @@ export default function OutputCell({
   const contentToDisplay = (normalizedOutput?.type === "text" && normalizedOutput.value) || "";
 
   return (
-    <CellWrapper>
+    <CellWrapper {...wrapperProps}>
       <Text whiteSpace="pre-wrap">{contentToDisplay}</Text>
     </CellWrapper>
   );

app/src/components/OutputsTable/OutputCell/PromptModal.tsx

@@ -5,30 +5,103 @@ import {
   ModalContent,
   ModalHeader,
   ModalOverlay,
+  VStack,
+  Text,
+  Box,
   type UseDisclosureReturn,
+  Link,
 } from "@chakra-ui/react";
-import { type RouterOutputs } from "~/utils/api";
+import { api, type RouterOutputs } from "~/utils/api";
 import { JSONTree } from "react-json-tree";
+import CopiableCode from "~/components/CopiableCode";
 
-export default function ExpandedModal(props: {
+const theme = {
+  scheme: "chalk",
+  author: "chris kempson (http://chriskempson.com)",
+  base00: "transparent",
+  base01: "#202020",
+  base02: "#303030",
+  base03: "#505050",
+  base04: "#b0b0b0",
+  base05: "#d0d0d0",
+  base06: "#e0e0e0",
+  base07: "#f5f5f5",
+  base08: "#fb9fb1",
+  base09: "#eda987",
+  base0A: "#ddb26f",
+  base0B: "#acc267",
+  base0C: "#12cfc0",
+  base0D: "#6fc2ef",
+  base0E: "#e1a3ee",
+  base0F: "#deaf8f",
+};
+
+export default function PromptModal(props: {
   cell: NonNullable<RouterOutputs["scenarioVariantCells"]["get"]>;
   disclosure: UseDisclosureReturn;
 }) {
+  const { data } = api.scenarioVariantCells.getTemplatedPromptMessage.useQuery(
+    {
+      cellId: props.cell.id,
+    },
+    {
+      enabled: props.disclosure.isOpen,
+    },
+  );
+
   return (
-    <Modal isOpen={props.disclosure.isOpen} onClose={props.disclosure.onClose} size="2xl">
+    <Modal isOpen={props.disclosure.isOpen} onClose={props.disclosure.onClose} size="xl">
       <ModalOverlay />
       <ModalContent>
-        <ModalHeader>Prompt</ModalHeader>
+        <ModalHeader>Prompt Details</ModalHeader>
         <ModalCloseButton />
         <ModalBody>
-          <JSONTree
-            data={props.cell.prompt}
-            invertTheme={true}
-            theme="chalk"
-            shouldExpandNodeInitially={() => true}
-            getItemString={() => ""}
-            hideRoot
-          />
+          <VStack py={4} w="">
+            <VStack w="full" alignItems="flex-start">
+              <Text fontWeight="bold">Full Prompt</Text>
+              <Box
+                w="full"
+                p={4}
+                alignItems="flex-start"
+                backgroundColor="blackAlpha.800"
+                borderRadius={4}
+              >
+                <JSONTree
+                  data={props.cell.prompt}
+                  theme={theme}
+                  shouldExpandNodeInitially={() => true}
+                  getItemString={() => ""}
+                  hideRoot
+                />
+              </Box>
+            </VStack>
+            {data?.templatedPrompt && (
+              <VStack w="full" mt={4} alignItems="flex-start">
+                <Text fontWeight="bold">Templated prompt message:</Text>
+                <CopiableCode
+                  w="full"
+                  // bgColor="gray.100"
+                  p={4}
+                  borderWidth={1}
+                  whiteSpace="pre-wrap"
+                  code={data.templatedPrompt}
+                />
+              </VStack>
+            )}
+            {data?.learnMoreUrl && (
+              <Link
+                href={data.learnMoreUrl}
+                isExternal
+                color="blue.500"
+                fontWeight="bold"
+                fontSize="sm"
+                mt={4}
+                alignSelf="flex-end"
+              >
+                Learn More
+              </Link>
+            )}
+          </VStack>
         </ModalBody>
       </ModalContent>
     </Modal>

app/src/components/OutputsTable/ScenarioEditor.tsx

@@ -16,7 +16,7 @@ import {
   VStack,
 } from "@chakra-ui/react";
 import { BsArrowsAngleExpand, BsX } from "react-icons/bs";
-import { cellPadding } from "../constants";
+import { cellPadding } from "./constants";
 import { FloatingLabelInput } from "./FloatingLabelInput";
 import { ScenarioEditorModal } from "./ScenarioEditorModal";
 
@@ -111,25 +111,23 @@ export default function ScenarioEditor({
         onDrop={onReorder}
         backgroundColor={isDragTarget ? "gray.100" : "transparent"}
       >
-        {variableLabels.length === 0 ? (
-          <Box color="gray.500">
-            {vars.data ? "No scenario variables configured" : "Loading..."}
-          </Box>
-        ) : (
+        {
           <VStack spacing={4} flex={1} py={2}>
             <HStack justifyContent="space-between" w="100%" align="center" spacing={0}>
               <Text flex={1}>Scenario</Text>
-              <Tooltip label="Expand" hasArrow>
-                <IconButton
-                  aria-label="Expand"
-                  icon={<Icon as={BsArrowsAngleExpand} boxSize={3} />}
-                  onClick={() => setScenarioEditorModalOpen(true)}
-                  size="xs"
-                  colorScheme="gray"
-                  color="gray.500"
-                  variant="ghost"
-                />
-              </Tooltip>
+              {variableLabels.length && (
+                <Tooltip label="Expand" hasArrow>
+                  <IconButton
+                    aria-label="Expand"
+                    icon={<Icon as={BsArrowsAngleExpand} boxSize={3} />}
+                    onClick={() => setScenarioEditorModalOpen(true)}
+                    size="xs"
+                    colorScheme="gray"
+                    color="gray.500"
+                    variant="ghost"
+                  />
+                </Tooltip>
+              )}
               {canModify && props.canHide && (
                 <Tooltip label="Delete" hasArrow>
                   <IconButton
@@ -150,31 +148,38 @@ export default function ScenarioEditor({
                 </Tooltip>
               )}
             </HStack>
-            {variableLabels.map((key) => {
-              const value = values[key] ?? "";
-              return (
-                <FloatingLabelInput
-                  key={key}
-                  label={key}
-                  isDisabled={!canModify}
-                  style={{ width: "100%" }}
-                  maxHeight={32}
-                  value={value}
-                  onChange={(e) => {
-                    setValues((prev) => ({ ...prev, [key]: e.target.value }));
-                  }}
-                  onKeyDown={(e) => {
-                    if (e.key === "Enter" && (e.metaKey || e.ctrlKey)) {
-                      e.preventDefault();
-                      e.currentTarget.blur();
-                      onSave();
-                    }
-                  }}
-                  onMouseEnter={() => setVariableInputHovered(true)}
-                  onMouseLeave={() => setVariableInputHovered(false)}
-                />
-              );
-            })}
+
+            {variableLabels.length === 0 ? (
+              <Box color="gray.500">
+                {vars.data ? "No scenario variables configured" : "Loading..."}
+              </Box>
+            ) : (
+              variableLabels.map((key) => {
+                const value = values[key] ?? "";
+                return (
+                  <FloatingLabelInput
+                    key={key}
+                    label={key}
+                    isDisabled={!canModify}
+                    style={{ width: "100%" }}
+                    maxHeight={32}
+                    value={value}
+                    onChange={(e) => {
+                      setValues((prev) => ({ ...prev, [key]: e.target.value }));
+                    }}
+                    onKeyDown={(e) => {
+                      if (e.key === "Enter" && (e.metaKey || e.ctrlKey)) {
+                        e.preventDefault();
+                        e.currentTarget.blur();
+                        onSave();
+                      }
+                    }}
+                    onMouseEnter={() => setVariableInputHovered(true)}
+                    onMouseLeave={() => setVariableInputHovered(false)}
+                  />
+                );
+              })
+            )}
             {hasChanged && (
               <HStack justify="right">
                 <Button
@@ -192,7 +197,7 @@ export default function ScenarioEditor({
               </HStack>
             )}
           </VStack>
-        )}
+        }
       </HStack>
       {scenarioEditorModalOpen && (
         <ScenarioEditorModal

app/src/components/OutputsTable/ScenarioEditorModal.tsx

@@ -65,11 +65,11 @@ export const ScenarioEditorModal = ({
     <Modal
       isOpen
       onClose={onClose}
-      size={{ base: "xl", sm: "2xl", md: "3xl", lg: "5xl", xl: "7xl" }}
+      size={{ base: "xl", sm: "2xl", md: "3xl", lg: "4xl", xl: "5xl" }}
     >
       <ModalOverlay />
       <ModalContent w={1200}>
-        <ModalHeader />
+        <ModalHeader>Edit Scenario</ModalHeader>
         <ModalCloseButton />
         <ModalBody maxW="unset">
           <VStack spacing={8}>

app/src/components/OutputsTable/ScenariosHeader.tsx

@@ -11,7 +11,7 @@ import {
   IconButton,
   Spinner,
 } from "@chakra-ui/react";
-import { cellPadding } from "../constants";
+import { cellPadding } from "./constants";
 import {
   useExperiment,
   useExperimentAccess,

app/src/components/OutputsTable/VariantEditor.tsx

@@ -110,7 +110,7 @@ export default function VariantEditor(props: { variant: PromptVariant }) {
     setIsChanged(false);
 
     await utils.promptVariants.list.invalidate();
-  }, [checkForChanges]);
+  }, [checkForChanges, replaceVariant.mutateAsync]);
 
   useEffect(() => {
     if (monaco) {

app/src/components/OutputsTable/VariantHeader/VariantHeader.tsx

@@ -1,11 +1,11 @@
 import { useState, type DragEvent } from "react";
-import { type PromptVariant } from "../OutputsTable/types";
+import { type PromptVariant } from "../types";
 import { api } from "~/utils/api";
 import { RiDraggable } from "react-icons/ri";
 import { useExperimentAccess, useHandledAsyncCallback } from "~/utils/hooks";
 import { HStack, Icon, Text, GridItem, type GridItemProps } from "@chakra-ui/react"; // Changed here
 import { cellPadding, headerMinHeight } from "../constants";
-import AutoResizeTextArea from "../AutoResizeTextArea";
+import AutoResizeTextArea from "../../AutoResizeTextArea";
 import VariantHeaderMenuButton from "./VariantHeaderMenuButton";
 
 export default function VariantHeader(
@@ -75,7 +75,7 @@ export default function VariantHeader(
       padding={0}
       sx={{
         position: "sticky",
-        top: "-2",
+        top: "0",
         // Ensure that the menu always appears above the sticky header of other variants
         zIndex: menuOpen ? "dropdown" : 10,
       }}

app/src/components/OutputsTable/VariantHeader/VariantHeaderMenuButton.tsx

@@ -1,6 +1,4 @@
-import { type PromptVariant } from "../OutputsTable/types";
-import { api } from "~/utils/api";
-import { useHandledAsyncCallback, useVisibleScenarioIds } from "~/utils/hooks";
+import { useState } from "react";
 import {
   Icon,
   Menu,
@@ -14,10 +12,13 @@ import {
 } from "@chakra-ui/react";
 import { BsFillTrashFill, BsGear, BsStars } from "react-icons/bs";
 import { FaRegClone } from "react-icons/fa";
-import { useState } from "react";
-import { RefinePromptModal } from "../RefinePromptModal/RefinePromptModal";
 import { RiExchangeFundsFill } from "react-icons/ri";
-import { ChangeModelModal } from "../ChangeModelModal/ChangeModelModal";
+
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback, useVisibleScenarioIds } from "~/utils/hooks";
+import { type PromptVariant } from "../types";
+import { RefinePromptModal } from "../../RefinePromptModal/RefinePromptModal";
+import { ChangeModelModal } from "../../ChangeModelModal/ChangeModelModal";
 
 export default function VariantHeaderMenuButton({
   variant,

app/src/components/OutputsTable/VariantStats.tsx

@@ -1,6 +1,6 @@
 import { HStack, Icon, Text, useToken } from "@chakra-ui/react";
 import { type PromptVariant } from "./types";
-import { cellPadding } from "../constants";
+import { cellPadding } from "./constants";
 import { api } from "~/utils/api";
 import chroma from "chroma-js";
 import { BsCurrencyDollar } from "react-icons/bs";

app/src/components/OutputsTable/index.tsx

@@ -3,13 +3,14 @@ import { api } from "~/utils/api";
 import AddVariantButton from "./AddVariantButton";
 import ScenarioRow from "./ScenarioRow";
 import VariantEditor from "./VariantEditor";
-import VariantHeader from "../VariantHeader/VariantHeader";
+import VariantHeader from "./VariantHeader/VariantHeader";
 import VariantStats from "./VariantStats";
 import { ScenariosHeader } from "./ScenariosHeader";
 import { borders } from "./styles";
 import { useScenarios } from "~/utils/hooks";
 import ScenarioPaginator from "./ScenarioPaginator";
 import { Fragment } from "react";
+import useScrolledPast from "./useHasScrolledPast";
 
 export default function OutputsTable({ experimentId }: { experimentId: string | undefined }) {
   const variants = api.promptVariants.list.useQuery(
@@ -18,6 +19,7 @@ export default function OutputsTable({ experimentId }: { experimentId: string |
   );
 
   const scenarios = useScenarios();
+  const shouldFlattenHeader = useScrolledPast(50);
 
   if (!variants.data || !scenarios.data) return null;
 
@@ -63,8 +65,8 @@ export default function OutputsTable({ experimentId }: { experimentId: string |
               variant={variant}
               canHide={variants.data.length > 1}
               rowStart={1}
-              borderTopLeftRadius={isFirst ? 8 : 0}
-              borderTopRightRadius={isLast ? 8 : 0}
+              borderTopLeftRadius={isFirst && !shouldFlattenHeader ? 8 : 0}
+              borderTopRightRadius={isLast && !shouldFlattenHeader ? 8 : 0}
               {...sharedProps}
             />
             <GridItem rowStart={2} {...sharedProps}>

app/src/components/OutputsTable/useHasScrolledPast.tsx

@@ -0,0 +1,34 @@
+import { useState, useEffect } from "react";
+
+const useScrolledPast = (scrollThreshold: number) => {
+  const [hasScrolledPast, setHasScrolledPast] = useState(true);
+
+  useEffect(() => {
+    const container = document.getElementById("output-container");
+
+    if (!container) {
+      console.warn('Element with id "outputs-container" not found.');
+      return;
+    }
+
+    const checkScroll = () => {
+      const { scrollTop } = container;
+
+      // Check if scrollTop is greater than or equal to scrollThreshold
+      setHasScrolledPast(scrollTop > scrollThreshold);
+    };
+
+    checkScroll();
+
+    container.addEventListener("scroll", checkScroll);
+
+    // Cleanup
+    return () => {
+      container.removeEventListener("scroll", checkScroll);
+    };
+  }, []);
+
+  return hasScrolledPast;
+};
+
+export default useScrolledPast;

app/src/components/experiments/ExperimentCard.tsx

@@ -14,21 +14,11 @@ import { formatTimePast } from "~/utils/dayjs";
 import Link from "next/link";
 import { useRouter } from "next/router";
 import { BsPlusSquare } from "react-icons/bs";
-import { api } from "~/utils/api";
+import { RouterOutputs, api } from "~/utils/api";
 import { useHandledAsyncCallback } from "~/utils/hooks";
 import { useAppStore } from "~/state/store";
 
-type ExperimentData = {
-  testScenarioCount: number;
-  promptVariantCount: number;
-  id: string;
-  label: string;
-  sortIndex: number;
-  createdAt: Date;
-  updatedAt: Date;
-};
-
-export const ExperimentCard = ({ exp }: { exp: ExperimentData }) => {
+export const ExperimentCard = ({ exp }: { exp: RouterOutputs["experiments"]["list"][0] }) => {
   return (
     <Card
       w="full"
@@ -45,7 +35,7 @@ export const ExperimentCard = ({ exp }: { exp: ExperimentData }) => {
         as={Link}
         w="full"
         h="full"
-        href={{ pathname: "/experiments/[id]", query: { id: exp.id } }}
+        href={{ pathname: "/experiments/[experimentSlug]", query: { experimentSlug: exp.slug } }}
         justify="space-between"
       >
         <HStack w="full" color="gray.700" justify="center">
@@ -89,8 +79,8 @@ export const NewExperimentCard = () => {
       projectId: selectedProjectId ?? "",
     });
     await router.push({
-      pathname: "/experiments/[id]",
-      query: { id: newExperiment.id },
+      pathname: "/experiments/[experimentSlug]",
+      query: { experimentSlug: newExperiment.slug },
     });
   }, [createMutation, router, selectedProjectId]);
 

app/src/components/experiments/ExperimentHeaderButtons/useOnForkButtonPressed.tsx

@@ -16,11 +16,14 @@ export const useOnForkButtonPressed = () => {
 
   const [onFork, isForking] = useHandledAsyncCallback(async () => {
     if (!experiment.data?.id || !selectedProjectId) return;
-    const forkedExperimentId = await forkMutation.mutateAsync({
+    const newExperiment = await forkMutation.mutateAsync({
       id: experiment.data.id,
       projectId: selectedProjectId,
     });
-    await router.push({ pathname: "/experiments/[id]", query: { id: forkedExperimentId } });
+    await router.push({
+      pathname: "/experiments/[experimentSlug]",
+      query: { experimentSlug: newExperiment.slug },
+    });
   }, [forkMutation, experiment.data?.id, router]);
 
   const onForkButtonPressed = useCallback(() => {

app/src/components/nav/ProjectMenu.tsx

@@ -15,7 +15,7 @@ import {
   Image,
   Box,
 } from "@chakra-ui/react";
-import React, { useEffect, useState } from "react";
+import { useEffect } from "react";
 import Link from "next/link";
 import { BsPlus, BsPersonCircle } from "react-icons/bs";
 import { type Project } from "@prisma/client";
@@ -67,7 +67,13 @@ export default function ProjectMenu() {
   );
 
   return (
-    <VStack w="full" alignItems="flex-start" spacing={0} py={1}>
+    <VStack
+      w="full"
+      alignItems="flex-start"
+      spacing={0}
+      py={1}
+      zIndex={popover.isOpen ? "dropdown" : undefined}
+    >
       <Popover
         placement="bottom"
         isOpen={popover.isOpen}
@@ -105,8 +111,8 @@ export default function ProjectMenu() {
         </PopoverTrigger>
         <PopoverContent
           _focusVisible={{ outline: "unset" }}
-          ml={-1}
-          w={224}
+          w={220}
+          ml={{ base: 2, md: 0 }}
           boxShadow="0 0 40px 4px rgba(0, 0, 0, 0.1);"
           fontSize="sm"
         >
@@ -176,7 +182,6 @@ const ProjectOption = ({
   onClose: () => void;
 }) => {
   const setSelectedProjectId = useAppStore((s) => s.setSelectedProjectId);
-  const [gearHovered, setGearHovered] = useState(false);
 
   return (
     <HStack
@@ -188,8 +193,8 @@ const ProjectOption = ({
       }}
       w="full"
       justifyContent="space-between"
-      _hover={gearHovered ? undefined : { bgColor: "gray.200", textDecoration: "none" }}
-      color={isActive ? "blue.400" : undefined}
+      _hover={{ bgColor: "gray.200", textDecoration: "none" }}
+      bgColor={isActive ? "gray.100" : undefined}
       py={2}
       px={4}
       borderRadius={4}

app/src/components/projectSettings/InviteMemberModal.tsx

@@ -0,0 +1,128 @@
+import {
+  Button,
+  FormControl,
+  FormLabel,
+  Input,
+  FormHelperText,
+  HStack,
+  Modal,
+  ModalBody,
+  ModalCloseButton,
+  ModalContent,
+  ModalFooter,
+  ModalHeader,
+  ModalOverlay,
+  Spinner,
+  Text,
+  VStack,
+  RadioGroup,
+  Radio,
+} from "@chakra-ui/react";
+import { useState, useEffect } from "react";
+
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback, useSelectedProject } from "~/utils/hooks";
+import { maybeReportError } from "~/utils/errorHandling/maybeReportError";
+import { type ProjectUserRole } from "@prisma/client";
+
+export const InviteMemberModal = ({
+  isOpen,
+  onClose,
+}: {
+  isOpen: boolean;
+  onClose: () => void;
+}) => {
+  const selectedProject = useSelectedProject().data;
+  const utils = api.useContext();
+
+  const [email, setEmail] = useState("");
+  const [role, setRole] = useState<ProjectUserRole>("MEMBER");
+
+  useEffect(() => {
+    setEmail("");
+    setRole("MEMBER");
+  }, [isOpen]);
+
+  const emailIsValid = !email || !email.match(/.+@.+\..+/);
+
+  const inviteMemberMutation = api.users.inviteToProject.useMutation();
+
+  const [inviteMember, isInviting] = useHandledAsyncCallback(async () => {
+    if (!selectedProject?.id || !role) return;
+    const resp = await inviteMemberMutation.mutateAsync({
+      projectId: selectedProject.id,
+      email,
+      role,
+    });
+    if (maybeReportError(resp)) return;
+    await utils.projects.get.invalidate();
+    onClose();
+  }, [inviteMemberMutation, email, role, selectedProject?.id, onClose]);
+
+  return (
+    <Modal isOpen={isOpen} onClose={onClose}>
+      <ModalOverlay />
+      <ModalContent w={1200}>
+        <ModalHeader>
+          <HStack>
+            <Text>Invite Member</Text>
+          </HStack>
+        </ModalHeader>
+        <ModalCloseButton />
+        <ModalBody>
+          <VStack spacing={8} alignItems="flex-start">
+            <Text>
+              Invite a new member to <b>{selectedProject?.name}</b>.
+            </Text>
+
+            <RadioGroup
+              value={role}
+              onChange={(e) => setRole(e as ProjectUserRole)}
+              colorScheme="orange"
+            >
+              <VStack w="full" alignItems="flex-start">
+                <Radio value="MEMBER">
+                  <Text fontSize="sm">MEMBER</Text>
+                </Radio>
+                <Radio value="ADMIN">
+                  <Text fontSize="sm">ADMIN</Text>
+                </Radio>
+              </VStack>
+            </RadioGroup>
+            <FormControl>
+              <FormLabel>Email</FormLabel>
+              <Input
+                type="email"
+                value={email}
+                onChange={(e) => setEmail(e.target.value)}
+                onKeyDown={(e) => {
+                  if (e.key === "Enter" && (e.metaKey || e.ctrlKey || e.shiftKey)) {
+                    e.preventDefault();
+                    e.currentTarget.blur();
+                    inviteMember();
+                  }
+                }}
+              />
+              <FormHelperText>Enter the email of the person you want to invite.</FormHelperText>
+            </FormControl>
+          </VStack>
+        </ModalBody>
+        <ModalFooter mt={4}>
+          <HStack>
+            <Button colorScheme="gray" onClick={onClose} minW={24}>
+              <Text>Cancel</Text>
+            </Button>
+            <Button
+              colorScheme="orange"
+              onClick={inviteMember}
+              minW={24}
+              isDisabled={emailIsValid || isInviting}
+            >
+              {isInviting ? <Spinner boxSize={4} /> : <Text>Send Invitation</Text>}
+            </Button>
+          </HStack>
+        </ModalFooter>
+      </ModalContent>
+    </Modal>
+  );
+};

app/src/components/projectSettings/MemberTable.tsx

@@ -0,0 +1,145 @@
+import { useMemo, useState } from "react";
+import {
+  Table,
+  Thead,
+  Tr,
+  Th,
+  Tbody,
+  Td,
+  IconButton,
+  useDisclosure,
+  Text,
+  Button,
+} from "@chakra-ui/react";
+import { useSession } from "next-auth/react";
+import { BsTrash } from "react-icons/bs";
+import { type User } from "@prisma/client";
+
+import { useHandledAsyncCallback, useSelectedProject } from "~/utils/hooks";
+import { InviteMemberModal } from "./InviteMemberModal";
+import { RemoveMemberDialog } from "./RemoveMemberDialog";
+import { api } from "~/utils/api";
+import { maybeReportError } from "~/utils/errorHandling/maybeReportError";
+
+const MemberTable = () => {
+  const selectedProject = useSelectedProject().data;
+  const session = useSession().data;
+
+  const utils = api.useContext();
+
+  const [memberToRemove, setMemberToRemove] = useState<User | null>(null);
+  const inviteMemberModal = useDisclosure();
+
+  const cancelInvitationMutation = api.users.cancelProjectInvitation.useMutation();
+
+  const [cancelInvitation, isCancelling] = useHandledAsyncCallback(
+    async (invitationToken: string) => {
+      if (!selectedProject?.id) return;
+      const resp = await cancelInvitationMutation.mutateAsync({
+        invitationToken,
+      });
+      if (maybeReportError(resp)) return;
+      await utils.projects.get.invalidate();
+    },
+    [selectedProject?.id, cancelInvitationMutation],
+  );
+
+  const sortedMembers = useMemo(() => {
+    if (!selectedProject?.projectUsers) return [];
+    return selectedProject.projectUsers.sort((a, b) => {
+      if (a.role === b.role) return a.createdAt < b.createdAt ? -1 : 1;
+      // Take advantage of fact that ADMIN is alphabetically before MEMBER
+      return a.role < b.role ? -1 : 1;
+    });
+  }, [selectedProject?.projectUsers]);
+
+  return (
+    <>
+      <Table fontSize={{ base: "sm", md: "md" }}>
+        <Thead
+          sx={{
+            th: {
+              base: { px: 0 },
+              md: { px: 6 },
+            },
+          }}
+        >
+          <Tr>
+            <Th>Name</Th>
+            <Th display={{ base: "none", md: "table-cell" }}>Email</Th>
+            <Th>Role</Th>
+            {selectedProject?.role === "ADMIN" && <Th />}
+          </Tr>
+        </Thead>
+        <Tbody
+          sx={{
+            td: {
+              base: { px: 0 },
+              md: { px: 6 },
+            },
+          }}
+        >
+          {selectedProject &&
+            sortedMembers.map((member) => {
+              return (
+                <Tr key={member.id}>
+                  <Td>
+                    <Text fontWeight="bold">{member.user.name}</Text>
+                  </Td>
+                  <Td display={{ base: "none", md: "table-cell" }} h="full">
+                    {member.user.email}
+                  </Td>
+                  <Td fontSize={{ base: "xs", md: "sm" }}>{member.role}</Td>
+                  {selectedProject.role === "ADMIN" && (
+                    <Td textAlign="end">
+                      {member.user.id !== session?.user?.id &&
+                        member.user.id !== selectedProject.personalProjectUserId && (
+                          <IconButton
+                            aria-label="Remove member"
+                            colorScheme="red"
+                            icon={<BsTrash />}
+                            onClick={() => setMemberToRemove(member.user)}
+                          />
+                        )}
+                    </Td>
+                  )}
+                </Tr>
+              );
+            })}
+          {selectedProject?.projectUserInvitations?.map((invitation) => {
+            return (
+              <Tr key={invitation.id}>
+                <Td>
+                  <Text as="i">Invitation pending</Text>
+                </Td>
+                <Td>{invitation.email}</Td>
+                <Td fontSize="sm">{invitation.role}</Td>
+                {selectedProject.role === "ADMIN" && (
+                  <Td textAlign="end">
+                    <Button
+                      size="sm"
+                      colorScheme="red"
+                      variant="ghost"
+                      onClick={() => cancelInvitation(invitation.invitationToken)}
+                      isLoading={isCancelling}
+                    >
+                      Cancel
+                    </Button>
+                  </Td>
+                )}
+              </Tr>
+            );
+          })}
+        </Tbody>
+      </Table>
+      <InviteMemberModal isOpen={inviteMemberModal.isOpen} onClose={inviteMemberModal.onClose} />
+      <RemoveMemberDialog
+        member={memberToRemove}
+        isOpen={!!memberToRemove}
+        onClose={() => setMemberToRemove(null)}
+      />
+    </>
+  );
+};
+
+export default MemberTable;

app/src/components/projectSettings/RemoveMemberDialog.tsx

@@ -0,0 +1,71 @@
+import {
+  Button,
+  AlertDialog,
+  AlertDialogBody,
+  AlertDialogFooter,
+  AlertDialogHeader,
+  AlertDialogContent,
+  AlertDialogOverlay,
+  Text,
+  VStack,
+  Spinner,
+} from "@chakra-ui/react";
+import { type User } from "@prisma/client";
+
+import { useRouter } from "next/router";
+import { useRef } from "react";
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback, useSelectedProject } from "~/utils/hooks";
+
+export const RemoveMemberDialog = ({
+  isOpen,
+  onClose,
+  member,
+}: {
+  isOpen: boolean;
+  onClose: () => void;
+  member: User | null;
+}) => {
+  const selectedProject = useSelectedProject();
+  const removeUserMutation = api.users.removeUserFromProject.useMutation();
+  const utils = api.useContext();
+  const router = useRouter();
+
+  const cancelRef = useRef<HTMLButtonElement>(null);
+
+  const [onRemoveConfirm, isRemoving] = useHandledAsyncCallback(async () => {
+    if (!selectedProject.data?.id || !member?.id) return;
+    await removeUserMutation.mutateAsync({ projectId: selectedProject.data.id, userId: member.id });
+    await utils.projects.get.invalidate();
+    onClose();
+  }, [removeUserMutation, selectedProject, router]);
+
+  return (
+    <AlertDialog isOpen={isOpen} leastDestructiveRef={cancelRef} onClose={onClose}>
+      <AlertDialogOverlay>
+        <AlertDialogContent>
+          <AlertDialogHeader fontSize="lg" fontWeight="bold">
+            Remove Member
+          </AlertDialogHeader>
+
+          <AlertDialogBody>
+            <VStack spacing={4} alignItems="flex-start">
+              <Text>
+                Are you sure you want to remove <b>{member?.name}</b> from the project?
+              </Text>
+            </VStack>
+          </AlertDialogBody>
+
+          <AlertDialogFooter>
+            <Button ref={cancelRef} onClick={onClose}>
+              Cancel
+            </Button>
+            <Button colorScheme="red" onClick={onRemoveConfirm} ml={3} w={20}>
+              {isRemoving ? <Spinner /> : "Remove"}
+            </Button>
+          </AlertDialogFooter>
+        </AlertDialogContent>
+      </AlertDialogOverlay>
+    </AlertDialog>
+  );
+};

app/src/components/requestLogs/LogFilters/AddFilterButton.tsx

@@ -10,7 +10,14 @@ const AddFilterButton = () => {
     <HStack
       as={Button}
       variant="ghost"
-      onClick={() => addFilter({ field: defaultFilterableFields[0], comparator: comparators[0] })}
+      onClick={() =>
+        addFilter({
+          id: Date.now().toString(),
+          field: defaultFilterableFields[0],
+          comparator: comparators[0],
+          value: "",
+        })
+      }
       spacing={0}
       fontSize="sm"
     >

app/src/components/requestLogs/LogFilters/LogFilter.tsx

@@ -8,39 +8,34 @@ import { debounce } from "lodash-es";
 import SelectFieldDropdown from "./SelectFieldDropdown";
 import SelectComparatorDropdown from "./SelectComparatorDropdown";
 
-const LogFilter = ({ filter, index }: { filter: LogFilter; index: number }) => {
+const LogFilter = ({ filter }: { filter: LogFilter }) => {
   const updateFilter = useAppStore((s) => s.logFilters.updateFilter);
   const deleteFilter = useAppStore((s) => s.logFilters.deleteFilter);
 
-  const [editedValue, setEditedValue] = useState("");
+  const [editedValue, setEditedValue] = useState(filter.value);
 
   const debouncedUpdateFilter = useCallback(
-    debounce(
-      (index: number, filter: LogFilter) => {
-        console.log("updating filter!!!");
-        updateFilter(index, filter);
-      },
-      200,
-      { leading: true },
-    ),
+    debounce((filter: LogFilter) => updateFilter(filter), 500, {
+      leading: true,
+    }),
     [updateFilter],
   );
 
   return (
     <HStack>
-      <SelectFieldDropdown filter={filter} index={index} />
-      <SelectComparatorDropdown filter={filter} index={index} />
+      <SelectFieldDropdown filter={filter} />
+      <SelectComparatorDropdown filter={filter} />
       <Input
         value={editedValue}
         onChange={(e) => {
           setEditedValue(e.target.value);
-          debouncedUpdateFilter(index, { ...filter, value: e.target.value });
+          debouncedUpdateFilter({ ...filter, value: e.target.value });
         }}
       />
       <IconButton
         aria-label="Delete Filter"
         icon={<BsTrash />}
-        onClick={() => deleteFilter(index)}
+        onClick={() => deleteFilter(filter.id)}
       />
     </HStack>
   );

app/src/components/requestLogs/LogFilters/LogFilters.tsx

@@ -19,8 +19,8 @@ const LogFilters = () => {
       <Text fontWeight="bold" color="gray.500">
         Filters
       </Text>
-      {filters.map((filter, index) => (
-        <LogFilter key={index} filter={filter} index={index} />
+      {filters.map((filter) => (
+        <LogFilter key={filter.id} filter={filter} />
       ))}
       <AddFilterButton />
     </VStack>

app/src/components/requestLogs/LogFilters/SelectComparatorDropdown.tsx

@@ -2,7 +2,7 @@ import { comparators, type LogFilter } from "~/state/logFiltersSlice";
 import { useAppStore } from "~/state/store";
 import InputDropdown from "~/components/InputDropdown";
 
-const SelectComparatorDropdown = ({ filter, index }: { filter: LogFilter; index: number }) => {
+const SelectComparatorDropdown = ({ filter }: { filter: LogFilter }) => {
   const updateFilter = useAppStore((s) => s.logFilters.updateFilter);
 
   const { comparator } = filter;
@@ -11,7 +11,7 @@ const SelectComparatorDropdown = ({ filter, index }: { filter: LogFilter; index:
     <InputDropdown
       options={comparators}
       selectedOption={comparator}
-      onSelect={(option) => updateFilter(index, { ...filter, comparator: option })}
+      onSelect={(option) => updateFilter({ ...filter, comparator: option })}
     />
   );
 };

app/src/components/requestLogs/LogFilters/SelectFieldDropdown.tsx

@@ -3,7 +3,7 @@ import { useAppStore } from "~/state/store";
 import { useTagNames } from "~/utils/hooks";
 import InputDropdown from "~/components/InputDropdown";
 
-const SelectFieldDropdown = ({ filter, index }: { filter: LogFilter; index: number }) => {
+const SelectFieldDropdown = ({ filter }: { filter: LogFilter }) => {
   const tagNames = useTagNames().data;
 
   const updateFilter = useAppStore((s) => s.logFilters.updateFilter);
@@ -14,7 +14,7 @@ const SelectFieldDropdown = ({ filter, index }: { filter: LogFilter; index: numb
     <InputDropdown
       options={[...defaultFilterableFields, ...(tagNames || [])]}
       selectedOption={field}
-      onSelect={(option) => updateFilter(index, { ...filter, field: option })}
+      onSelect={(option) => updateFilter({ ...filter, field: option })}
     />
   );
 };

app/src/env.mjs

@@ -21,6 +21,19 @@ export const env = createEnv({
     ANTHROPIC_API_KEY: z.string().default("placeholder"),
     SENTRY_AUTH_TOKEN: z.string().optional(),
     OPENPIPE_API_KEY: z.string().optional(),
+    SENDER_EMAIL: z.string().default("placeholder"),
+    SMTP_HOST: z.string().default("placeholder"),
+    SMTP_PORT: z.string().default("placeholder"),
+    SMTP_LOGIN: z.string().default("placeholder"),
+    SMTP_PASSWORD: z.string().default("placeholder"),
+    WORKER_CONCURRENCY: z
+      .string()
+      .default("10")
+      .transform((val) => parseInt(val)),
+    WORKER_MAX_POOL_SIZE: z
+      .string()
+      .default("10")
+      .transform((val) => parseInt(val)),
   },
 
   /**
@@ -58,6 +71,13 @@ export const env = createEnv({
     SENTRY_AUTH_TOKEN: process.env.SENTRY_AUTH_TOKEN,
     OPENPIPE_API_KEY: process.env.OPENPIPE_API_KEY,
     NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS: process.env.NEXT_PUBLIC_FF_SHOW_LOGGED_CALLS,
+    SENDER_EMAIL: process.env.SENDER_EMAIL,
+    SMTP_HOST: process.env.SMTP_HOST,
+    SMTP_PORT: process.env.SMTP_PORT,
+    SMTP_LOGIN: process.env.SMTP_LOGIN,
+    SMTP_PASSWORD: process.env.SMTP_PASSWORD,
+    WORKER_CONCURRENCY: process.env.WORKER_CONCURRENCY,
+    WORKER_MAX_POOL_SIZE: process.env.WORKER_MAX_POOL_SIZE,
   },
   /**
    * Run `build` or `dev` with `SKIP_ENV_VALIDATION` to skip env validation.

app/src/modelProviders/frontendModelProviders.ts

@@ -1,6 +1,7 @@
 import openaiChatCompletionFrontend from "./openai-ChatCompletion/frontend";
 import replicateLlama2Frontend from "./replicate-llama2/frontend";
 import anthropicFrontend from "./anthropic-completion/frontend";
+import openpipeFrontend from "./openpipe-chat/frontend";
 import { type SupportedProvider, type FrontendModelProvider } from "./types";
 
 // Keep attributes here that need to be accessible from the frontend. We can't
@@ -10,6 +11,7 @@ const frontendModelProviders: Record<SupportedProvider, FrontendModelProvider<an
   "openai/ChatCompletion": openaiChatCompletionFrontend,
   "replicate/llama2": replicateLlama2Frontend,
   "anthropic/completion": anthropicFrontend,
+  "openpipe/Chat": openpipeFrontend,
 };
 
 export default frontendModelProviders;

app/src/modelProviders/modelProviders.ts

@@ -1,12 +1,14 @@
 import openaiChatCompletion from "./openai-ChatCompletion";
 import replicateLlama2 from "./replicate-llama2";
 import anthropicCompletion from "./anthropic-completion";
+import openpipeChatCompletion from "./openpipe-chat";
 import { type SupportedProvider, type ModelProvider } from "./types";
 
 const modelProviders: Record<SupportedProvider, ModelProvider<any, any, any>> = {
   "openai/ChatCompletion": openaiChatCompletion,
   "replicate/llama2": replicateLlama2,
   "anthropic/completion": anthropicCompletion,
+  "openpipe/Chat": openpipeChatCompletion,
 };
 
 export default modelProviders;

app/src/modelProviders/openai-ChatCompletion/getCompletion.ts

@@ -16,7 +16,16 @@ export async function getCompletion(
   try {
     if (onStream) {
       const resp = await openai.chat.completions.create(
-        { ...input, stream: true },
+        {
+          ...input,
+          stream: true,
+          openpipe: {
+            tags: {
+              prompt_id: "getCompletion",
+              stream: "true",
+            },
+          },
+        },
         {
           maxRetries: 0,
         },
@@ -34,7 +43,16 @@ export async function getCompletion(
       }
     } else {
       const resp = await openai.chat.completions.create(
-        { ...input, stream: false },
+        {
+          ...input,
+          stream: false,
+          openpipe: {
+            tags: {
+              prompt_id: "getCompletion",
+              stream: "false",
+            },
+          },
+        },
         {
           maxRetries: 0,
         },

app/src/modelProviders/openai-ChatCompletion/refinementActions.ts

@@ -12,7 +12,6 @@ export const refinementActions: Record<string, RefinementAction> = {
   
       definePrompt("openai/ChatCompletion", {
           model: "gpt-4",
-          stream: true,
           messages: [
               {
               role: "system",
@@ -29,7 +28,6 @@ export const refinementActions: Record<string, RefinementAction> = {
   
       definePrompt("openai/ChatCompletion", {
           model: "gpt-4",
-          stream: true,
           messages: [
               {
               role: "system",
@@ -126,7 +124,6 @@ export const refinementActions: Record<string, RefinementAction> = {
   
       definePrompt("openai/ChatCompletion", {
         model: "gpt-4",
-        stream: true,
         messages: [
           {
             role: "system",
@@ -143,7 +140,6 @@ export const refinementActions: Record<string, RefinementAction> = {
   
       definePrompt("openai/ChatCompletion", {
         model: "gpt-4",
-        stream: true,
         messages: [
           {
             role: "system",
@@ -237,7 +233,6 @@ export const refinementActions: Record<string, RefinementAction> = {
   
       definePrompt("openai/ChatCompletion", {
         model: "gpt-3.5-turbo",
-        stream: true,
         messages: [
           {
             role: "system",

app/src/modelProviders/openpipe-chat/frontend.ts

@@ -0,0 +1,98 @@
+import { type OpenpipeChatOutput, type SupportedModel } from ".";
+import { type FrontendModelProvider } from "../types";
+import { refinementActions } from "./refinementActions";
+import {
+  templateOpenOrcaPrompt,
+  templateAlpacaInstructPrompt,
+  //   templateSystemUserAssistantPrompt,
+  templateInstructionInputResponsePrompt,
+  templateAiroborosPrompt,
+  templateGryphePrompt,
+  templateVicunaPrompt,
+} from "./templatePrompt";
+
+const frontendModelProvider: FrontendModelProvider<SupportedModel, OpenpipeChatOutput> = {
+  name: "OpenAI ChatCompletion",
+
+  models: {
+    "Open-Orca/OpenOrcaxOpenChat-Preview2-13B": {
+      name: "OpenOrcaxOpenChat-Preview2-13B",
+      contextWindow: 4096,
+      pricePerSecond: 0.0003,
+      speed: "medium",
+      provider: "openpipe/Chat",
+      learnMoreUrl: "https://huggingface.co/Open-Orca/OpenOrcaxOpenChat-Preview2-13B",
+      templatePrompt: templateOpenOrcaPrompt,
+    },
+    "Open-Orca/OpenOrca-Platypus2-13B": {
+      name: "OpenOrca-Platypus2-13B",
+      contextWindow: 4096,
+      pricePerSecond: 0.0003,
+      speed: "medium",
+      provider: "openpipe/Chat",
+      learnMoreUrl: "https://huggingface.co/Open-Orca/OpenOrca-Platypus2-13B",
+      templatePrompt: templateAlpacaInstructPrompt,
+      defaultStopTokens: ["</s>"],
+    },
+    // "stabilityai/StableBeluga-13B": {
+    //   name: "StableBeluga-13B",
+    //   contextWindow: 4096,
+    //   pricePerSecond: 0.0003,
+    //   speed: "medium",
+    //   provider: "openpipe/Chat",
+    //   learnMoreUrl: "https://huggingface.co/stabilityai/StableBeluga-13B",
+    //   templatePrompt: templateSystemUserAssistantPrompt,
+    // },
+    "NousResearch/Nous-Hermes-Llama2-13b": {
+      name: "Nous-Hermes-Llama2-13b",
+      contextWindow: 4096,
+      pricePerSecond: 0.0003,
+      speed: "medium",
+      provider: "openpipe/Chat",
+      learnMoreUrl: "https://huggingface.co/NousResearch/Nous-Hermes-Llama2-13b",
+      templatePrompt: templateInstructionInputResponsePrompt,
+    },
+    "jondurbin/airoboros-l2-13b-gpt4-2.0": {
+      name: "airoboros-l2-13b-gpt4-2.0",
+      contextWindow: 4096,
+      pricePerSecond: 0.0003,
+      speed: "medium",
+      provider: "openpipe/Chat",
+      learnMoreUrl: "https://huggingface.co/jondurbin/airoboros-l2-13b-gpt4-2.0",
+      templatePrompt: templateAiroborosPrompt,
+    },
+    "lmsys/vicuna-13b-v1.5": {
+      name: "vicuna-13b-v1.5",
+      contextWindow: 4096,
+      pricePerSecond: 0.0003,
+      speed: "medium",
+      provider: "openpipe/Chat",
+      learnMoreUrl: "https://huggingface.co/lmsys/vicuna-13b-v1.5",
+      templatePrompt: templateVicunaPrompt,
+    },
+    "Gryphe/MythoMax-L2-13b": {
+      name: "MythoMax-L2-13b",
+      contextWindow: 4096,
+      pricePerSecond: 0.0003,
+      speed: "medium",
+      provider: "openpipe/Chat",
+      learnMoreUrl: "https://huggingface.co/Gryphe/MythoMax-L2-13b",
+      templatePrompt: templateGryphePrompt,
+    },
+    "NousResearch/Nous-Hermes-llama-2-7b": {
+      name: "Nous-Hermes-llama-2-7b",
+      contextWindow: 4096,
+      pricePerSecond: 0.0003,
+      speed: "medium",
+      provider: "openpipe/Chat",
+      learnMoreUrl: "https://huggingface.co/NousResearch/Nous-Hermes-llama-2-7b",
+      templatePrompt: templateInstructionInputResponsePrompt,
+    },
+  },
+
+  refinementActions,
+
+  normalizeOutput: (output) => ({ type: "text", value: output }),
+};
+
+export default frontendModelProvider;

app/src/modelProviders/openpipe-chat/getCompletion.ts

@@ -0,0 +1,121 @@
+/* eslint-disable @typescript-eslint/no-unsafe-call */
+import { isArray, isString } from "lodash-es";
+import OpenAI, { APIError } from "openai";
+
+import { type CompletionResponse } from "../types";
+import { type OpenpipeChatInput, type OpenpipeChatOutput } from ".";
+import frontendModelProvider from "./frontend";
+
+const modelEndpoints: Record<OpenpipeChatInput["model"], string> = {
+  "Open-Orca/OpenOrcaxOpenChat-Preview2-13B": "https://5ef82gjxk8kdys-8000.proxy.runpod.net/v1",
+  "Open-Orca/OpenOrca-Platypus2-13B": "https://lt5qlel6qcji8t-8000.proxy.runpod.net/v1",
+  // "stabilityai/StableBeluga-13B": "https://vcorl8mxni2ou1-8000.proxy.runpod.net/v1",
+  "NousResearch/Nous-Hermes-Llama2-13b": "https://ncv8pw3u0vb8j2-8000.proxy.runpod.net/v1",
+  "jondurbin/airoboros-l2-13b-gpt4-2.0": "https://9nrbx7oph4btou-8000.proxy.runpod.net/v1",
+  "lmsys/vicuna-13b-v1.5": "https://h88hkt3ux73rb7-8000.proxy.runpod.net/v1",
+  "Gryphe/MythoMax-L2-13b": "https://3l5jvhnxdgky3v-8000.proxy.runpod.net/v1",
+  "NousResearch/Nous-Hermes-llama-2-7b": "https://ua1bpc6kv3dgge-8000.proxy.runpod.net/v1",
+};
+
+export async function getCompletion(
+  input: OpenpipeChatInput,
+  onStream: ((partialOutput: OpenpipeChatOutput) => void) | null,
+): Promise<CompletionResponse<OpenpipeChatOutput>> {
+  const { model, messages, ...rest } = input;
+
+  const templatedPrompt = frontendModelProvider.models[model].templatePrompt?.(messages);
+
+  if (!templatedPrompt) {
+    return {
+      type: "error",
+      message: "Failed to generate prompt",
+      autoRetry: false,
+    };
+  }
+
+  const openai = new OpenAI({
+    baseURL: modelEndpoints[model],
+  });
+  const start = Date.now();
+  let finalCompletion: OpenpipeChatOutput = "";
+
+  const completionParams = {
+    model,
+    prompt: templatedPrompt,
+    ...rest,
+  };
+
+  if (!completionParams.stop && frontendModelProvider.models[model].defaultStopTokens) {
+    completionParams.stop = frontendModelProvider.models[model].defaultStopTokens;
+  }
+
+  try {
+    if (onStream) {
+      const resp = await openai.completions.create(
+        { ...completionParams, stream: true },
+        {
+          maxRetries: 0,
+        },
+      );
+
+      for await (const part of resp) {
+        finalCompletion += part.choices[0]?.text;
+        onStream(finalCompletion);
+      }
+      if (!finalCompletion) {
+        return {
+          type: "error",
+          message: "Streaming failed to return a completion",
+          autoRetry: false,
+        };
+      }
+    } else {
+      const resp = await openai.completions.create(
+        { ...completionParams, stream: false },
+        {
+          maxRetries: 0,
+        },
+      );
+      finalCompletion = resp.choices[0]?.text || "";
+      if (!finalCompletion) {
+        return {
+          type: "error",
+          message: "Failed to return a completion",
+          autoRetry: false,
+        };
+      }
+    }
+    const timeToComplete = Date.now() - start;
+
+    return {
+      type: "success",
+      statusCode: 200,
+      value: finalCompletion,
+      timeToComplete,
+    };
+  } catch (error: unknown) {
+    if (error instanceof APIError) {
+      // The types from the sdk are wrong
+      const rawMessage = error.message as string | string[];
+      // If the message is not a string, stringify it
+      const message = isString(rawMessage)
+        ? rawMessage
+        : isArray(rawMessage)
+        ? rawMessage.map((m) => m.toString()).join("\n")
+        : (rawMessage as any).toString();
+      return {
+        type: "error",
+        message,
+        autoRetry: error.status === 429 || error.status === 503,
+        statusCode: error.status,
+      };
+    } else {
+      console.error(error);
+      return {
+        type: "error",
+        message: (error as Error).message,
+        autoRetry: true,
+      };
+    }
+  }
+}

app/src/modelProviders/openpipe-chat/index.ts

@@ -0,0 +1,54 @@
+import { type JSONSchema4 } from "json-schema";
+import { type ModelProvider } from "../types";
+import inputSchema from "./input.schema.json";
+import { getCompletion } from "./getCompletion";
+import frontendModelProvider from "./frontend";
+
+const supportedModels = [
+  "Open-Orca/OpenOrcaxOpenChat-Preview2-13B",
+  "Open-Orca/OpenOrca-Platypus2-13B",
+  // "stabilityai/StableBeluga-13B",
+  "NousResearch/Nous-Hermes-Llama2-13b",
+  "jondurbin/airoboros-l2-13b-gpt4-2.0",
+  "lmsys/vicuna-13b-v1.5",
+  "Gryphe/MythoMax-L2-13b",
+  "NousResearch/Nous-Hermes-llama-2-7b",
+] as const;
+
+export type SupportedModel = (typeof supportedModels)[number];
+
+export type OpenpipeChatInput = {
+  model: SupportedModel;
+  messages: {
+    role: "system" | "user" | "assistant";
+    content: string;
+  }[];
+  temperature?: number;
+  top_p?: number;
+  stop?: string[] | string;
+  max_tokens?: number;
+  presence_penalty?: number;
+  frequency_penalty?: number;
+};
+
+export type OpenpipeChatOutput = string;
+
+export type OpenpipeChatModelProvider = ModelProvider<
+  SupportedModel,
+  OpenpipeChatInput,
+  OpenpipeChatOutput
+>;
+
+const modelProvider: OpenpipeChatModelProvider = {
+  getModel: (input) => input.model,
+  inputSchema: inputSchema as JSONSchema4,
+  canStream: true,
+  getCompletion,
+  getUsage: (input, output) => {
+    // TODO: Implement this
+    return null;
+  },
+  ...frontendModelProvider,
+};
+
+export default modelProvider;

app/src/modelProviders/openpipe-chat/input.schema.json

@@ -0,0 +1,96 @@
+{
+  "type": "object",
+  "properties": {
+    "model": {
+      "description": "ID of the model to use.",
+      "example": "Open-Orca/OpenOrcaxOpenChat-Preview2-13B",
+      "type": "string",
+      "enum": [
+        "Open-Orca/OpenOrcaxOpenChat-Preview2-13B",
+        "Open-Orca/OpenOrca-Platypus2-13B",
+        "NousResearch/Nous-Hermes-Llama2-13b",
+        "jondurbin/airoboros-l2-13b-gpt4-2.0",
+        "lmsys/vicuna-13b-v1.5",
+        "Gryphe/MythoMax-L2-13b",
+        "NousResearch/Nous-Hermes-llama-2-7b"
+      ]
+    },
+    "messages": {
+      "description": "A list of messages comprising the conversation so far.",
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "type": "object",
+        "properties": {
+          "role": {
+            "type": "string",
+            "enum": ["system", "user", "assistant"],
+            "description": "The role of the messages author. One of `system`, `user`, or `assistant`."
+          },
+          "content": {
+            "type": "string",
+            "description": "The contents of the message. `content` is required for all messages."
+          }
+        },
+        "required": ["role", "content"]
+      }
+    },
+    "temperature": {
+      "type": "number",
+      "minimum": 0,
+      "maximum": 2,
+      "default": 1,
+      "example": 1,
+      "nullable": true,
+      "description": "What sampling temperature to use, between 0 and 2. Higher values like 0.8 will make the output more random, while lower values like 0.2 will make it more focused and deterministic.\n\nWe generally recommend altering this or `top_p` but not both.\n"
+    },
+    "top_p": {
+      "type": "number",
+      "minimum": 0,
+      "maximum": 1,
+      "default": 1,
+      "example": 1,
+      "nullable": true,
+      "description": "An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.\n\nWe generally recommend altering this or `temperature` but not both.\n"
+    },
+    "stop": {
+      "description": "Up to 4 sequences where the API will stop generating further tokens.\n",
+      "default": null,
+      "oneOf": [
+        {
+          "type": "string",
+          "nullable": true
+        },
+        {
+          "type": "array",
+          "minItems": 1,
+          "maxItems": 4,
+          "items": {
+            "type": "string"
+          }
+        }
+      ]
+    },
+    "max_tokens": {
+      "description": "The maximum number of [tokens](/tokenizer) to generate in the chat completion.\n\nThe total length of input tokens and generated tokens is limited by the model's context length. [Example Python code](https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb) for counting tokens.\n",
+      "type": "integer"
+    },
+    "presence_penalty": {
+      "type": "number",
+      "default": 0,
+      "minimum": -2,
+      "maximum": 2,
+      "nullable": true,
+      "description": "Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.\n\n[See more information about frequency and presence penalties.](/docs/api-reference/parameter-details)\n"
+    },
+    "frequency_penalty": {
+      "type": "number",
+      "default": 0,
+      "minimum": -2,
+      "maximum": 2,
+      "nullable": true,
+      "description": "Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.\n\n[See more information about frequency and presence penalties.](/docs/api-reference/parameter-details)\n"
+    }
+  },
+  "required": ["model", "messages"]
+}

app/src/modelProviders/openpipe-chat/refinementActions.ts

@@ -0,0 +1,3 @@
+import { type RefinementAction } from "../types";
+
+export const refinementActions: Record<string, RefinementAction> = {};

app/src/modelProviders/openpipe-chat/templatePrompt.ts

@@ -0,0 +1,274 @@
+import { type OpenpipeChatInput } from ".";
+
+// User: Hello<|end_of_turn|>Assistant: Hi<|end_of_turn|>User: How are you today?<|end_of_turn|>Assistant:
+export const templateOpenOrcaPrompt = (messages: OpenpipeChatInput["messages"]) => {
+  const splitter = "<|end_of_turn|>";
+
+  const formattedMessages = messages.map((message) => {
+    if (message.role === "system" || message.role === "user") {
+      return "User: " + message.content;
+    } else {
+      return "Assistant: " + message.content;
+    }
+  });
+
+  let prompt = formattedMessages.join(splitter);
+
+  // Ensure that the prompt ends with an assistant message
+  const lastUserIndex = prompt.lastIndexOf("User:");
+  const lastAssistantIndex = prompt.lastIndexOf("Assistant:");
+  if (lastUserIndex > lastAssistantIndex) {
+    prompt += splitter + "Assistant:";
+  }
+
+  return prompt;
+};
+
+// ### Instruction:
+
+// <prompt> (without the <>)
+
+// ### Response: (leave two newlines for model to respond)
+export const templateAlpacaInstructPrompt = (messages: OpenpipeChatInput["messages"]) => {
+  const splitter = "\n\n";
+
+  const userTag = "### Instruction:\n\n";
+  const assistantTag = "### Response:\n\n";
+
+  const formattedMessages = messages.map((message) => {
+    if (message.role === "system" || message.role === "user") {
+      return userTag + message.content;
+    } else {
+      return assistantTag + message.content;
+    }
+  });
+
+  let prompt = formattedMessages.join(splitter);
+
+  // Ensure that the prompt ends with an assistant message
+  const lastUserIndex = prompt.lastIndexOf(userTag);
+  const lastAssistantIndex = prompt.lastIndexOf(assistantTag);
+  if (lastUserIndex > lastAssistantIndex) {
+    prompt += splitter + assistantTag;
+  }
+
+  return prompt;
+};
+
+// ### System:
+// This is a system prompt, please behave and help the user.
+
+// ### User:
+// Your prompt here
+
+// ### Assistant
+// The output of Stable Beluga 13B
+export const templateSystemUserAssistantPrompt = (messages: OpenpipeChatInput["messages"]) => {
+  const splitter = "\n\n";
+
+  const systemTag = "### System:\n";
+  const userTag = "### User:\n";
+  const assistantTag = "### Assistant\n";
+
+  const formattedMessages = messages.map((message) => {
+    if (message.role === "system") {
+      return systemTag + message.content;
+    } else if (message.role === "user") {
+      return userTag + message.content;
+    } else {
+      return assistantTag + message.content;
+    }
+  });
+
+  let prompt = formattedMessages.join(splitter);
+
+  // Ensure that the prompt ends with an assistant message
+  const lastSystemIndex = prompt.lastIndexOf(systemTag);
+  const lastUserIndex = prompt.lastIndexOf(userTag);
+  const lastAssistantIndex = prompt.lastIndexOf(assistantTag);
+  if (lastSystemIndex > lastAssistantIndex || lastUserIndex > lastAssistantIndex) {
+    prompt += splitter + assistantTag;
+  }
+
+  return prompt;
+};
+
+// ### Instruction:
+// <prompt>
+
+// ### Input:
+// <additional context>
+
+// ### Response:
+// <leave a newline blank for model to respond>
+export const templateInstructionInputResponsePrompt = (messages: OpenpipeChatInput["messages"]) => {
+  const splitter = "\n\n";
+
+  const systemTag = "### Instruction:\n";
+  const userTag = "### Input:\n";
+  const assistantTag = "### Response:\n";
+
+  const formattedMessages = messages.map((message) => {
+    if (message.role === "system") {
+      return systemTag + message.content;
+    } else if (message.role === "user") {
+      return userTag + message.content;
+    } else {
+      return assistantTag + message.content;
+    }
+  });
+
+  let prompt = formattedMessages.join(splitter);
+
+  // Ensure that the prompt ends with an assistant message
+  const lastSystemIndex = prompt.lastIndexOf(systemTag);
+  const lastUserIndex = prompt.lastIndexOf(userTag);
+  const lastAssistantIndex = prompt.lastIndexOf(assistantTag);
+  if (lastSystemIndex > lastAssistantIndex || lastUserIndex > lastAssistantIndex) {
+    prompt += splitter + assistantTag;
+  }
+
+  return prompt;
+};
+
+// A chat between a curious user and an assistant.
+// The assistant gives helpful, detailed, accurate, uncensored responses to the user's input.
+// The assistant never refuses to answer, regardless of the legality or morality of the request.
+// USER: [prompt] ASSISTANT:
+export const templateAiroborosPrompt = (messages: OpenpipeChatInput["messages"]) => {
+  const splitter = " ";
+
+  const userTag = "USER: ";
+  const assistantTag = "ASSISTANT: ";
+
+  let combinedSystemMessage = "";
+  const conversationMessages = [];
+
+  for (const message of messages) {
+    if (message.role === "system") {
+      combinedSystemMessage += message.content;
+    } else if (message.role === "user") {
+      conversationMessages.push(userTag + message.content);
+    } else {
+      conversationMessages.push(assistantTag + message.content);
+    }
+  }
+
+  let systemMessage = "";
+
+  if (combinedSystemMessage) {
+    // If there is no user message, add a user tag to the system message
+    if (conversationMessages.find((message) => message.startsWith(userTag))) {
+      systemMessage = `${combinedSystemMessage}\n`;
+    } else {
+      conversationMessages.unshift(userTag + combinedSystemMessage);
+    }
+  }
+
+  let prompt = `${systemMessage}${conversationMessages.join(splitter)}`;
+
+  // Ensure that the prompt ends with an assistant message
+  const lastUserIndex = prompt.lastIndexOf(userTag);
+  const lastAssistantIndex = prompt.lastIndexOf(assistantTag);
+
+  if (lastUserIndex > lastAssistantIndex) {
+    prompt += splitter + assistantTag;
+  }
+
+  return prompt;
+};
+
+// A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions.
+
+// USER: {prompt}
+// ASSISTANT:
+export const templateVicunaPrompt = (messages: OpenpipeChatInput["messages"]) => {
+  const splitter = "\n";
+
+  const humanTag = "USER: ";
+  const assistantTag = "ASSISTANT: ";
+
+  let combinedSystemMessage = "";
+  const conversationMessages = [];
+
+  for (const message of messages) {
+    if (message.role === "system") {
+      combinedSystemMessage += message.content;
+    } else if (message.role === "user") {
+      conversationMessages.push(humanTag + message.content);
+    } else {
+      conversationMessages.push(assistantTag + message.content);
+    }
+  }
+
+  let systemMessage = "";
+
+  if (combinedSystemMessage) {
+    // If there is no user message, add a user tag to the system message
+    if (conversationMessages.find((message) => message.startsWith(humanTag))) {
+      systemMessage = `${combinedSystemMessage}\n\n`;
+    } else {
+      conversationMessages.unshift(humanTag + combinedSystemMessage);
+    }
+  }
+
+  let prompt = `${systemMessage}${conversationMessages.join(splitter)}`;
+
+  // Ensure that the prompt ends with an assistant message
+  const lastHumanIndex = prompt.lastIndexOf(humanTag);
+  const lastAssistantIndex = prompt.lastIndexOf(assistantTag);
+  if (lastHumanIndex > lastAssistantIndex) {
+    prompt += splitter + assistantTag;
+  }
+
+  return prompt.trim();
+};
+
+// <System prompt/Character Card>
+
+// ### Instruction:
+// Your instruction or question here.
+// For roleplay purposes, I suggest the following - Write <CHAR NAME>'s next reply in a chat between <YOUR NAME> and <CHAR NAME>. Write a single reply only.
+
+// ### Response:
+export const templateGryphePrompt = (messages: OpenpipeChatInput["messages"]) => {
+  const splitter = "\n\n";
+
+  const instructionTag = "### Instruction:\n";
+  const responseTag = "### Response:\n";
+
+  let combinedSystemMessage = "";
+  const conversationMessages = [];
+
+  for (const message of messages) {
+    if (message.role === "system") {
+      combinedSystemMessage += message.content;
+    } else if (message.role === "user") {
+      conversationMessages.push(instructionTag + message.content);
+    } else {
+      conversationMessages.push(responseTag + message.content);
+    }
+  }
+
+  let systemMessage = "";
+
+  if (combinedSystemMessage) {
+    // If there is no user message, add a user tag to the system message
+    if (conversationMessages.find((message) => message.startsWith(instructionTag))) {
+      systemMessage = `${combinedSystemMessage}\n\n`;
+    } else {
+      conversationMessages.unshift(instructionTag + combinedSystemMessage);
+    }
+  }
+
+  let prompt = `${systemMessage}${conversationMessages.join(splitter)}`;
+
+  // Ensure that the prompt ends with an assistant message
+  const lastInstructionIndex = prompt.lastIndexOf(instructionTag);
+  const lastAssistantIndex = prompt.lastIndexOf(responseTag);
+  if (lastInstructionIndex > lastAssistantIndex) {
+    prompt += splitter + responseTag;
+  }
+
+  return prompt;
+};

app/src/modelProviders/replicate-llama2/getCompletion.ts

@@ -8,7 +8,7 @@ const replicate = new Replicate({
 });
 
 const modelIds: Record<ReplicateLlama2Input["model"], string> = {
-  "7b-chat": "4f0b260b6a13eb53a6b1891f089d57c08f41003ae79458be5011303d81a394dc",
+  "7b-chat": "7b0bfc9aff140d5b75bacbed23e91fd3c34b01a1e958d32132de6e0a19796e2c",
   "13b-chat": "2a7f981751ec7fdf87b5b91ad4db53683a98082e9ff7bfd12c8cd5ea85980a52",
   "70b-chat": "2c1608e18606fad2812020dc541930f2d0495ce32eee50074220b87300bc16e1",
 };

app/src/modelProviders/types.ts

@@ -2,11 +2,13 @@ import { type JSONSchema4 } from "json-schema";
 import { type IconType } from "react-icons";
 import { type JsonValue } from "type-fest";
 import { z } from "zod";
+import { type OpenpipeChatInput } from "./openpipe-chat";
 
 export const ZodSupportedProvider = z.union([
   z.literal("openai/ChatCompletion"),
   z.literal("replicate/llama2"),
   z.literal("anthropic/completion"),
+  z.literal("openpipe/Chat"),
 ]);
 
 export type SupportedProvider = z.infer<typeof ZodSupportedProvider>;
@@ -22,6 +24,8 @@ export type Model = {
   description?: string;
   learnMoreUrl?: string;
   apiDocsUrl?: string;
+  templatePrompt?: (initialPrompt: OpenpipeChatInput["messages"]) => string;
+  defaultStopTokens?: string[];
 };
 
 export type ProviderModel = { provider: z.infer<typeof ZodSupportedProvider>; model: string };

app/src/pages/admin/jobs/index.tsx

@@ -0,0 +1,54 @@
+import { Card, Table, Tbody, Td, Th, Thead, Tr } from "@chakra-ui/react";
+import dayjs from "dayjs";
+import { isDate, isObject, isString } from "lodash-es";
+import AppShell from "~/components/nav/AppShell";
+import { type RouterOutputs, api } from "~/utils/api";
+
+const fieldsToShow: (keyof RouterOutputs["adminJobs"]["list"][0])[] = [
+  "id",
+  "queue_name",
+  "payload",
+  "priority",
+  "attempts",
+  "last_error",
+  "created_at",
+  "key",
+  "locked_at",
+  "run_at",
+];
+
+export default function Jobs() {
+  const jobs = api.adminJobs.list.useQuery({});
+
+  return (
+    <AppShell title="Admin Jobs">
+      <Card m={4} overflowX="auto">
+        <Table>
+          <Thead>
+            <Tr>
+              {fieldsToShow.map((field) => (
+                <Th key={field}>{field}</Th>
+              ))}
+            </Tr>
+          </Thead>
+          <Tbody>
+            {jobs.data?.map((job) => (
+              <Tr key={job.id}>
+                {fieldsToShow.map((field) => {
+                  // Check if object
+                  let value = job[field];
+                  if (isDate(value)) {
+                    value = dayjs(value).format("YYYY-MM-DD HH:mm:ss");
+                  } else if (isObject(value) && !isString(value)) {
+                    value = JSON.stringify(value);
+                  } // check if date
+                  return <Td key={field}>{value}</Td>;
+                })}
+              </Tr>
+            ))}
+          </Tbody>
+        </Table>
+      </Card>
+    </AppShell>
+  );
+}

app/src/pages/experiments/[experimentSlug].tsx

@@ -26,26 +26,6 @@ import Head from "next/head";
 import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
 import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContents";
 
-// TODO: import less to fix deployment with server side props
-// export const getServerSideProps = async (context: GetServerSidePropsContext<{ id: string }>) => {
-//   const experimentId = context.params?.id as string;
-
-//   const helpers = createServerSideHelpers({
-//     router: appRouter,
-//     ctx: createInnerTRPCContext({ session: null }),
-//     transformer: superjson, // optional - adds superjson serialization
-//   });
-
-//   // prefetch query
-//   await helpers.experiments.stats.prefetch({ id: experimentId });
-
-//   return {
-//     props: {
-//       trpcState: helpers.dehydrate(),
-//     },
-//   };
-// };
-
 export default function Experiment() {
   const router = useRouter();
   const utils = api.useContext();
@@ -53,9 +33,9 @@ export default function Experiment() {
 
   const experiment = useExperiment();
   const experimentStats = api.experiments.stats.useQuery(
-    { id: router.query.id as string },
+    { id: experiment.data?.id as string },
     {
-      enabled: !!router.query.id,
+      enabled: !!experiment.data?.id,
     },
   );
   const stats = experimentStats.data;
@@ -144,8 +124,8 @@ export default function Experiment() {
             <ExperimentHeaderButtons />
           </PageHeaderContainer>
           <ExperimentSettingsDrawer />
-          <Box w="100%" overflowX="auto" flex={1}>
-            <OutputsTable experimentId={router.query.id as string | undefined} />
+          <Box w="100%" overflowX="auto" flex={1} id="output-container">
+            <OutputsTable experimentId={experiment.data?.id} />
           </Box>
         </VStack>
       </AppShell>

app/src/pages/invitations/[invitationToken].tsx

@@ -0,0 +1,110 @@
+import { Center, Text, VStack, HStack, Button, Card } from "@chakra-ui/react";
+
+import { useRouter } from "next/router";
+import AppShell from "~/components/nav/AppShell";
+import { api } from "~/utils/api";
+import { useHandledAsyncCallback } from "~/utils/hooks";
+import { useAppStore } from "~/state/store";
+import { useSyncVariantEditor } from "~/state/sync";
+import { maybeReportError } from "~/utils/errorHandling/maybeReportError";
+
+export default function Invitation() {
+  const router = useRouter();
+  const utils = api.useContext();
+  useSyncVariantEditor();
+
+  const setSelectedProjectId = useAppStore((state) => state.setSelectedProjectId);
+
+  const invitationToken = router.query.invitationToken as string | undefined;
+
+  const invitation = api.users.getProjectInvitation.useQuery(
+    { invitationToken: invitationToken as string },
+    { enabled: !!invitationToken },
+  );
+
+  const cancelMutation = api.users.cancelProjectInvitation.useMutation();
+  const [declineInvitation, isDeclining] = useHandledAsyncCallback(async () => {
+    if (invitationToken) {
+      await cancelMutation.mutateAsync({
+        invitationToken,
+      });
+      await router.replace("/");
+    }
+  }, [cancelMutation, invitationToken]);
+
+  const acceptMutation = api.users.acceptProjectInvitation.useMutation();
+  const [acceptInvitation, isAccepting] = useHandledAsyncCallback(async () => {
+    if (invitationToken) {
+      const resp = await acceptMutation.mutateAsync({
+        invitationToken,
+      });
+      if (!maybeReportError(resp) && resp) {
+        await utils.projects.list.invalidate();
+        setSelectedProjectId(resp.payload);
+      }
+      await router.replace("/");
+    }
+  }, [acceptMutation, invitationToken]);
+
+  if (invitation.isLoading) {
+    return (
+      <AppShell requireAuth title="Loading...">
+        <Center h="full">
+          <Text>Loading...</Text>
+        </Center>
+      </AppShell>
+    );
+  }
+
+  if (!invitationToken || !invitation.data) {
+    return (
+      <AppShell requireAuth title="Invalid invitation token">
+        <Center h="full">
+          <Text>
+            The invitation you've received is invalid or expired. Please ask your project admin for
+            a new token.
+          </Text>
+        </Center>
+      </AppShell>
+    );
+  }
+
+  return (
+    <>
+      <AppShell requireAuth title="Invitation">
+        <Center h="full">
+          <Card>
+            <VStack
+              spacing={8}
+              w="full"
+              maxW="2xl"
+              p={16}
+              borderWidth={1}
+              borderRadius={8}
+              bgColor="white"
+            >
+              <Text fontSize="lg" fontWeight="bold">
+                You're invited! 🎉
+              </Text>
+              <Text textAlign="center">
+                You've been invited to join <b>{invitation.data.project.name}</b> by{" "}
+                <b>
+                  {invitation.data.sender.name} ({invitation.data.sender.email})
+                </b>
+                .
+              </Text>
+              <HStack spacing={4}>
+                <Button colorScheme="gray" isLoading={isDeclining} onClick={declineInvitation}>
+                  Decline
+                </Button>
+                <Button colorScheme="orange" isLoading={isAccepting} onClick={acceptInvitation}>
+                  Accept
+                </Button>
+              </HStack>
+            </VStack>
+          </Card>
+        </Center>
+      </AppShell>
+    </>
+  );
+}

app/src/pages/project/settings/index.tsx

@@ -9,9 +9,11 @@ import {
   Divider,
   Icon,
   useDisclosure,
+  Box,
+  Tooltip,
 } from "@chakra-ui/react";
 import { useEffect, useState } from "react";
-import { BsTrash } from "react-icons/bs";
+import { BsPlus, BsTrash } from "react-icons/bs";
 
 import AppShell from "~/components/nav/AppShell";
 import PageHeaderContainer from "~/components/nav/PageHeaderContainer";
@@ -21,6 +23,8 @@ import ProjectBreadcrumbContents from "~/components/nav/ProjectBreadcrumbContent
 import CopiableCode from "~/components/CopiableCode";
 import { DeleteProjectDialog } from "~/components/projectSettings/DeleteProjectDialog";
 import AutoResizeTextArea from "~/components/AutoResizeTextArea";
+import MemberTable from "~/components/projectSettings/MemberTable";
+import { InviteMemberModal } from "~/components/projectSettings/InviteMemberModal";
 
 export default function Settings() {
   const utils = api.useContext();
@@ -50,12 +54,13 @@ export default function Settings() {
     setName(selectedProject?.name);
   }, [selectedProject?.name]);
 
-  const deleteProjectOpen = useDisclosure();
+  const inviteMemberModal = useDisclosure();
+  const deleteProjectDialog = useDisclosure();
 
   return (
     <>
-      <AppShell>
-        <PageHeaderContainer>
+      <AppShell requireAuth>
+        <PageHeaderContainer px={{ base: 4, md: 8 }}>
           <Breadcrumb>
             <BreadcrumbItem>
               <ProjectBreadcrumbContents />
@@ -65,7 +70,7 @@ export default function Settings() {
             </BreadcrumbItem>
           </Breadcrumb>
         </PageHeaderContainer>
-        <VStack px={8} py={4} alignItems="flex-start" spacing={4}>
+        <VStack px={{ base: 4, md: 8 }} py={4} alignItems="flex-start" spacing={4}>
           <VStack spacing={0} alignItems="flex-start">
             <Text fontSize="2xl" fontWeight="bold">
               Project Settings
@@ -109,6 +114,37 @@ export default function Settings() {
               </Button>
             </VStack>
             <Divider backgroundColor="gray.300" />
+            <VStack w="full" alignItems="flex-start">
+              <Subtitle>Project Members</Subtitle>
+
+              <Text fontSize="sm">
+                Add members to your project to allow them to view and edit your project's data.
+              </Text>
+              <Box mt={4} w="full">
+                <MemberTable />
+              </Box>
+              <Tooltip
+                isDisabled={selectedProject?.role === "ADMIN"}
+                label="Only admins can invite new members"
+                hasArrow
+              >
+                <Button
+                  variant="outline"
+                  colorScheme="orange"
+                  borderRadius={4}
+                  onClick={inviteMemberModal.onOpen}
+                  mt={2}
+                  _disabled={{
+                    opacity: 0.6,
+                  }}
+                  isDisabled={selectedProject?.role !== "ADMIN"}
+                >
+                  <Icon as={BsPlus} boxSize={5} />
+                  <Text>Invite New Member</Text>
+                </Button>
+              </Tooltip>
+            </VStack>
+            <Divider backgroundColor="gray.300" />
             <VStack alignItems="flex-start">
               <Subtitle>Project API Key</Subtitle>
               <Text fontSize="sm">
@@ -141,7 +177,7 @@ export default function Settings() {
                   borderRadius={4}
                   mt={2}
                   height="auto"
-                  onClick={deleteProjectOpen.onOpen}
+                  onClick={deleteProjectDialog.onOpen}
                 >
                   <Icon as={BsTrash} />
                   <Text overflowWrap="break-word" whiteSpace="normal" py={2}>
@@ -153,7 +189,11 @@ export default function Settings() {
           </VStack>
         </VStack>
       </AppShell>
-      <DeleteProjectDialog isOpen={deleteProjectOpen.isOpen} onClose={deleteProjectOpen.onClose} />
+      <InviteMemberModal isOpen={inviteMemberModal.isOpen} onClose={inviteMemberModal.onClose} />
+      <DeleteProjectDialog
+        isOpen={deleteProjectDialog.isOpen}
+        onClose={deleteProjectDialog.onClose}
+      />
     </>
   );
 }

app/src/server/api/autogenerate/autogenerateDatasetEntries.ts

@@ -89,6 +89,11 @@ export const autogenerateDatasetEntries = async (
 
       function_call: { name: "add_list_of_data" },
       temperature: 0.5,
+      openpipe: {
+        tags: {
+          prompt_id: "autogenerateDatasetEntries",
+        },
+      },
     });
 
   const completionCallbacks = batchSizes.map((batchSize) =>

app/src/server/api/autogenerate/autogenerateScenarioValues.ts

@@ -98,6 +98,11 @@ export const autogenerateScenarioValues = async (
 
       function_call: { name: "add_scenario" },
       temperature: 0.5,
+      openpipe: {
+        tags: {
+          prompt_id: "autogenerateScenarioValues",
+        },
+      },
     });
 
     const parsed = JSON.parse(

app/src/server/api/external/v1Api.router.ts

@@ -66,7 +66,7 @@ export const v1ApiRouter = createOpenApiRouter({
 
       if (!existingResponse) return { respPayload: null };
 
-      await prisma.loggedCall.create({
+      const newCall = await prisma.loggedCall.create({
         data: {
           projectId: ctx.key.projectId,
           requestedAt: new Date(input.requestedAt),
@@ -75,11 +75,7 @@ export const v1ApiRouter = createOpenApiRouter({
         },
       });
 
-      await createTags(
-        existingResponse.originalLoggedCall.projectId,
-        existingResponse.originalLoggedCallId,
-        input.tags,
-      );
+      await createTags(newCall.projectId, newCall.id, input.tags);
       return {
         respPayload: existingResponse.respPayload,
       };
@@ -111,7 +107,7 @@ export const v1ApiRouter = createOpenApiRouter({
           .default({}),
       }),
     )
-    .output(z.object({ status: z.literal("ok") }))
+    .output(z.object({ status: z.union([z.literal("ok"), z.literal("error")]) }))
     .mutation(async ({ input, ctx }) => {
       const reqPayload = await reqValidator.spa(input.reqPayload);
       const respPayload = await respValidator.spa(input.respPayload);
@@ -212,6 +208,7 @@ export const v1ApiRouter = createOpenApiRouter({
           createdAt: true,
           cacheHit: true,
           tags: true,
+          id: true,
           modelResponse: {
             select: {
               id: true,
@@ -237,7 +234,7 @@ async function createTags(projectId: string, loggedCallId: string, tags: Record<
   const tagsToCreate = Object.entries(tags).map(([name, value]) => ({
     projectId,
     loggedCallId,
-    name: name.replaceAll(/[^a-zA-Z0-9_$]/g, "_"),
+    name: name.replaceAll(/[^a-zA-Z0-9_$.]/g, "_"),
     value,
   }));
   await prisma.loggedCallTag.createMany({

app/src/server/api/root.router.ts

@@ -11,6 +11,8 @@ import { datasetEntries } from "./routers/datasetEntries.router";
 import { projectsRouter } from "./routers/projects.router";
 import { dashboardRouter } from "./routers/dashboard.router";
 import { loggedCallsRouter } from "./routers/loggedCalls.router";
+import { usersRouter } from "./routers/users.router";
+import { adminJobsRouter } from "./routers/adminJobs.router";
 
 /**
  * This is the primary router for your server.
@@ -30,6 +32,8 @@ export const appRouter = createTRPCRouter({
   projects: projectsRouter,
   dashboard: dashboardRouter,
   loggedCalls: loggedCallsRouter,
+  users: usersRouter,
+  adminJobs: adminJobsRouter,
 });
 
 // export type definition of API

app/src/server/api/routers/adminJobs.router.ts

@@ -0,0 +1,18 @@
+import { z } from "zod";
+
+import { createTRPCRouter, protectedProcedure } from "~/server/api/trpc";
+import { kysely } from "~/server/db";
+import { requireIsAdmin } from "~/utils/accessControl";
+
+export const adminJobsRouter = createTRPCRouter({
+  list: protectedProcedure.input(z.object({})).query(async ({ ctx }) => {
+    await requireIsAdmin(ctx);
+
+    return await kysely
+      .selectFrom("graphile_worker.jobs")
+      .limit(100)
+      .selectAll()
+      .orderBy("created_at", "desc")
+      .execute();
+  }),
+});

app/src/server/api/routers/experiments.router.ts

@@ -85,15 +85,16 @@ export const experimentsRouter = createTRPCRouter({
       return experimentsWithCounts;
     }),
 
-  get: publicProcedure.input(z.object({ id: z.string() })).query(async ({ input, ctx }) => {
-    await requireCanViewExperiment(input.id, ctx);
+  get: publicProcedure.input(z.object({ slug: z.string() })).query(async ({ input, ctx }) => {
     const experiment = await prisma.experiment.findFirstOrThrow({
-      where: { id: input.id },
+      where: { slug: input.slug },
       include: {
         project: true,
       },
     });
 
+    await requireCanViewExperiment(experiment.id, ctx);
+
     const canModify = ctx.session?.user.id
       ? await canModifyExperiment(experiment.id, ctx.session?.user.id)
       : false;
@@ -177,6 +178,7 @@ export const experimentsRouter = createTRPCRouter({
         existingToNewVariantIds.set(variant.id, newVariantId);
         variantsToCreate.push({
           ...variant,
+          uiId: uuidv4(),
           id: newVariantId,
           experimentId: newExperimentId,
         });
@@ -190,6 +192,7 @@ export const experimentsRouter = createTRPCRouter({
         scenariosToCreate.push({
           ...scenario,
           id: newScenarioId,
+          uiId: uuidv4(),
           experimentId: newExperimentId,
           variableValues: scenario.variableValues as Prisma.InputJsonValue,
         });
@@ -290,7 +293,10 @@ export const experimentsRouter = createTRPCRouter({
         }),
       ]);
 
-      return newExperimentId;
+      const newExperiment = await prisma.experiment.findUniqueOrThrow({
+        where: { id: newExperimentId },
+      });
+      return newExperiment;
     }),
 
   create: protectedProcedure
@@ -335,7 +341,6 @@ export const experimentsRouter = createTRPCRouter({
           
           definePrompt("openai/ChatCompletion", {
             model: "gpt-3.5-turbo-0613",
-            stream: true,
             messages: [
               {
                 role: "system",

app/src/server/api/routers/loggedCalls.router.ts

@@ -1,5 +1,5 @@
 import { z } from "zod";
-import { type Expression, type SqlBool, sql } from "kysely";
+import { type Expression, type SqlBool, sql, type RawBuilder } from "kysely";
 import { jsonArrayFrom } from "kysely/helpers/postgres";
 
 import { createTRPCRouter, protectedProcedure } from "~/server/api/trpc";
@@ -8,15 +8,22 @@ import { comparators, defaultFilterableFields } from "~/state/logFiltersSlice";
 import { requireCanViewProject } from "~/utils/accessControl";
 
 // create comparator type based off of comparators
-const comparatorToSqlValue = (comparator: (typeof comparators)[number], value: string) => {
-  switch (comparator) {
-    case "=":
-      return `= '${value}'`;
-    case "!=":
-      return `!= '${value}'`;
-    case "CONTAINS":
-      return `like '%${value}%'`;
-  }
+const comparatorToSqlExpression = (comparator: (typeof comparators)[number], value: string) => {
+  return (reference: RawBuilder<unknown>): Expression<SqlBool> => {
+    switch (comparator) {
+      case "=":
+        return sql`${reference} = ${value}`;
+      case "!=":
+        // Handle NULL values
+        return sql`${reference} IS DISTINCT FROM ${value}`;
+      case "CONTAINS":
+        return sql`${reference} LIKE ${"%" + value + "%"}`;
+      case "NOT_CONTAINS":
+        return sql`(${reference} NOT LIKE ${"%" + value + "%"} OR ${reference} IS NULL)`;
+      default:
+        throw new Error("Unknown comparator");
+    }
+  };
 };
 
 export const loggedCallsRouter = createTRPCRouter({
@@ -30,7 +37,7 @@ export const loggedCallsRouter = createTRPCRouter({
           z.object({
             field: z.string(),
             comparator: z.enum(comparators),
-            value: z.string().optional(),
+            value: z.string(),
           }),
         ),
       }),
@@ -48,40 +55,19 @@ export const loggedCallsRouter = createTRPCRouter({
 
           for (const filter of input.filters) {
             if (!filter.value) continue;
+            const filterExpression = comparatorToSqlExpression(filter.comparator, filter.value);
+
             if (filter.field === "Request") {
-              wheres.push(
-                sql.raw(
-                  `lcmr."reqPayload"::text ${comparatorToSqlValue(
-                    filter.comparator,
-                    filter.value,
-                  )}`,
-                ),
-              );
+              wheres.push(filterExpression(sql.raw(`lcmr."reqPayload"::text`)));
             }
             if (filter.field === "Response") {
-              wheres.push(
-                sql.raw(
-                  `lcmr."respPayload"::text ${comparatorToSqlValue(
-                    filter.comparator,
-                    filter.value,
-                  )}`,
-                ),
-              );
+              wheres.push(filterExpression(sql.raw(`lcmr."respPayload"::text`)));
             }
             if (filter.field === "Model") {
-              wheres.push(
-                sql.raw(`lc."model" ${comparatorToSqlValue(filter.comparator, filter.value)}`),
-              );
+              wheres.push(filterExpression(sql.raw(`lc."model"`)));
             }
             if (filter.field === "Status Code") {
-              wheres.push(
-                sql.raw(
-                  `lcmr."statusCode"::text ${comparatorToSqlValue(
-                    filter.comparator,
-                    filter.value,
-                  )}`,
-                ),
-              );
+              wheres.push(filterExpression(sql.raw(`lcmr."statusCode"::text`)));
             }
           }
 
@@ -101,15 +87,15 @@ export const loggedCallsRouter = createTRPCRouter({
         const filter = tagFilters[i];
         if (!filter?.value) continue;
         const tableAlias = `lct${i}`;
+        const filterExpression = comparatorToSqlExpression(filter.comparator, filter.value);
+
         updatedBaseQuery = updatedBaseQuery
           .leftJoin(`LoggedCallTag as ${tableAlias}`, (join) =>
             join
               .onRef("lc.id", "=", `${tableAlias}.loggedCallId`)
               .on(`${tableAlias}.name`, "=", filter.field),
           )
-          .where(
-            sql.raw(`${tableAlias}.value ${comparatorToSqlValue(filter.comparator, filter.value)}`),
-          ) as unknown as typeof baseQuery;
+          .where(filterExpression(sql.raw(`${tableAlias}.value`))) as unknown as typeof baseQuery;
       }
 
       const rawCalls = await updatedBaseQuery

app/src/server/api/routers/projects.router.ts

@@ -51,6 +51,12 @@ export const projectsRouter = createTRPCRouter({
         include: {
           apiKeys: true,
           personalProjectUser: true,
+          projectUsers: {
+            include: {
+              user: true,
+            },
+          },
+          projectUserInvitations: true,
         },
       }),
       prisma.projectUser.findFirst({
@@ -58,7 +64,7 @@ export const projectsRouter = createTRPCRouter({
           userId: ctx.session.user.id,
           projectId: input.id,
           role: {
-            in: ["ADMIN", "MEMBER"],
+            in: ["ADMIN", "MEMBER", "VIEWER"],
           },
         },
       }),

app/src/server/api/routers/scenarioVariantCells.router.ts

@@ -1,4 +1,6 @@
+import { TRPCError } from "@trpc/server";
 import { z } from "zod";
+import modelProviders from "~/modelProviders/modelProviders";
 import { createTRPCRouter, protectedProcedure, publicProcedure } from "~/server/api/trpc";
 import { prisma } from "~/server/db";
 import { queueQueryModel } from "~/server/tasks/queryModel.task";
@@ -59,7 +61,7 @@ export const scenarioVariantCellsRouter = createTRPCRouter({
         evalsComplete,
       };
     }),
-  forceRefetch: protectedProcedure
+  hardRefetch: protectedProcedure
     .input(
       z.object({
         scenarioId: z.string(),
@@ -83,7 +85,10 @@ export const scenarioVariantCellsRouter = createTRPCRouter({
       });
 
       if (!cell) {
-        await generateNewCell(input.variantId, input.scenarioId, { stream: true });
+        await generateNewCell(input.variantId, input.scenarioId, {
+          stream: true,
+          hardRefetch: true,
+        });
         return;
       }
 
@@ -94,6 +99,48 @@ export const scenarioVariantCellsRouter = createTRPCRouter({
         },
       });
 
-      await queueQueryModel(cell.id, true);
+      await queueQueryModel(cell.id, { stream: true, hardRefetch: true });
+    }),
+  getTemplatedPromptMessage: publicProcedure
+    .input(
+      z.object({
+        cellId: z.string(),
+      }),
+    )
+    .query(async ({ input }) => {
+      const cell = await prisma.scenarioVariantCell.findUnique({
+        where: { id: input.cellId },
+        include: {
+          promptVariant: true,
+          modelResponses: true,
+        },
+      });
+
+      if (!cell) {
+        throw new TRPCError({
+          code: "NOT_FOUND",
+        });
+      }
+
+      const promptMessages = (cell.prompt as { messages: [] })["messages"];
+
+      if (!promptMessages) return null;
+
+      const { modelProvider, model } = cell.promptVariant;
+
+      const provider = modelProviders[modelProvider as keyof typeof modelProviders];
+
+      if (!provider) return null;
+
+      const modelObj = provider.models[model as keyof typeof provider.models];
+
+      const templatePrompt = modelObj?.templatePrompt;
+
+      if (!templatePrompt) return null;
+
+      return {
+        templatedPrompt: templatePrompt(promptMessages),
+        learnMoreUrl: modelObj.learnMoreUrl,
+      };
     }),
 });

app/src/server/api/routers/users.router.ts

@@ -0,0 +1,232 @@
+import { v4 as uuidv4 } from "uuid";
+import { z } from "zod";
+
+import { createTRPCRouter, protectedProcedure } from "~/server/api/trpc";
+import { prisma } from "~/server/db";
+import { error, success } from "~/utils/errorHandling/standardResponses";
+import { requireIsProjectAdmin, requireNothing } from "~/utils/accessControl";
+import { TRPCError } from "@trpc/server";
+import { sendProjectInvitation } from "~/server/emails/sendProjectInvitation";
+
+export const usersRouter = createTRPCRouter({
+  inviteToProject: protectedProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        email: z.string().email(),
+        role: z.enum(["ADMIN", "MEMBER", "VIEWER"]),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      await requireIsProjectAdmin(input.projectId, ctx);
+
+      const user = await prisma.user.findUnique({
+        where: {
+          email: input.email,
+        },
+      });
+
+      if (user) {
+        const existingMembership = await prisma.projectUser.findUnique({
+          where: {
+            projectId_userId: {
+              projectId: input.projectId,
+              userId: user.id,
+            },
+          },
+        });
+
+        if (existingMembership) {
+          return error(`A user with ${input.email} is already a member of this project`);
+        }
+      }
+
+      const invitation = await prisma.userInvitation.upsert({
+        where: {
+          projectId_email: {
+            projectId: input.projectId,
+            email: input.email,
+          },
+        },
+        update: {
+          role: input.role,
+        },
+        create: {
+          projectId: input.projectId,
+          email: input.email,
+          role: input.role,
+          invitationToken: uuidv4(),
+          senderId: ctx.session.user.id,
+        },
+        include: {
+          project: {
+            select: {
+              name: true,
+            },
+          },
+        },
+      });
+
+      try {
+        await sendProjectInvitation({
+          invitationToken: invitation.invitationToken,
+          recipientEmail: input.email,
+          invitationSenderName: ctx.session.user.name || "",
+          invitationSenderEmail: ctx.session.user.email || "",
+          projectName: invitation.project.name,
+        });
+      } catch (e) {
+        // If we fail to send the email, we should delete the invitation
+        await prisma.userInvitation.delete({
+          where: {
+            invitationToken: invitation.invitationToken,
+          },
+        });
+        return error("Failed to send email");
+      }
+
+      return success();
+    }),
+  getProjectInvitation: protectedProcedure
+    .input(
+      z.object({
+        invitationToken: z.string(),
+      }),
+    )
+    .query(async ({ input, ctx }) => {
+      requireNothing(ctx);
+
+      const invitation = await prisma.userInvitation.findUnique({
+        where: {
+          invitationToken: input.invitationToken,
+        },
+        include: {
+          project: {
+            select: {
+              name: true,
+            },
+          },
+          sender: {
+            select: {
+              name: true,
+              email: true,
+            },
+          },
+        },
+      });
+
+      if (!invitation) {
+        throw new TRPCError({ code: "NOT_FOUND" });
+      }
+
+      return invitation;
+    }),
+  acceptProjectInvitation: protectedProcedure
+    .input(
+      z.object({
+        invitationToken: z.string(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      requireNothing(ctx);
+
+      const invitation = await prisma.userInvitation.findUnique({
+        where: {
+          invitationToken: input.invitationToken,
+        },
+      });
+
+      if (!invitation) {
+        throw new TRPCError({ code: "NOT_FOUND" });
+      }
+
+      await prisma.projectUser.create({
+        data: {
+          projectId: invitation.projectId,
+          userId: ctx.session.user.id,
+          role: invitation.role,
+        },
+      });
+
+      await prisma.userInvitation.delete({
+        where: {
+          invitationToken: input.invitationToken,
+        },
+      });
+
+      return success(invitation.projectId);
+    }),
+  cancelProjectInvitation: protectedProcedure
+    .input(
+      z.object({
+        invitationToken: z.string(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      requireNothing(ctx);
+
+      const invitation = await prisma.userInvitation.findUnique({
+        where: {
+          invitationToken: input.invitationToken,
+        },
+      });
+
+      if (!invitation) {
+        throw new TRPCError({ code: "NOT_FOUND" });
+      }
+
+      await prisma.userInvitation.delete({
+        where: {
+          invitationToken: input.invitationToken,
+        },
+      });
+
+      return success();
+    }),
+  editProjectUserRole: protectedProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        userId: z.string(),
+        role: z.enum(["ADMIN", "MEMBER", "VIEWER"]),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      await requireIsProjectAdmin(input.projectId, ctx);
+
+      await prisma.projectUser.update({
+        where: {
+          projectId_userId: {
+            projectId: input.projectId,
+            userId: input.userId,
+          },
+        },
+        data: {
+          role: input.role,
+        },
+      });
+
+      return success();
+    }),
+  removeUserFromProject: protectedProcedure
+    .input(
+      z.object({
+        projectId: z.string(),
+        userId: z.string(),
+      }),
+    )
+    .mutation(async ({ input, ctx }) => {
+      await requireIsProjectAdmin(input.projectId, ctx);
+
+      await prisma.projectUser.delete({
+        where: {
+          projectId_userId: {
+            projectId: input.projectId,
+            userId: input.userId,
+          },
+        },
+      });
+
+      return success();
+    }),
+});

app/src/server/db.ts

@@ -1,27 +1,6 @@
-import {
-  type Experiment,
-  type PromptVariant,
-  type TestScenario,
-  type TemplateVariable,
-  type ScenarioVariantCell,
-  type ModelResponse,
-  type Evaluation,
-  type OutputEvaluation,
-  type Dataset,
-  type DatasetEntry,
-  type Project,
-  type ProjectUser,
-  type WorldChampEntrant,
-  type LoggedCall,
-  type LoggedCallModelResponse,
-  type LoggedCallTag,
-  type ApiKey,
-  type Account,
-  type Session,
-  type User,
-  type VerificationToken,
-  PrismaClient,
-} from "@prisma/client";
+import { type DB } from "./db.types";
+
+import { PrismaClient } from "@prisma/client";
 import { Kysely, PostgresDialect } from "kysely";
 // TODO: Revert to normal import when our tsconfig.json is fixed
 // import { Pool } from "pg";
@@ -32,30 +11,6 @@ const Pool = (UntypedPool.default ? UntypedPool.default : UntypedPool) as typeof
 
 import { env } from "~/env.mjs";
 
-interface DB {
-  Experiment: Experiment;
-  PromptVariant: PromptVariant;
-  TestScenario: TestScenario;
-  TemplateVariable: TemplateVariable;
-  ScenarioVariantCell: ScenarioVariantCell;
-  ModelResponse: ModelResponse;
-  Evaluation: Evaluation;
-  OutputEvaluation: OutputEvaluation;
-  Dataset: Dataset;
-  DatasetEntry: DatasetEntry;
-  Project: Project;
-  ProjectUser: ProjectUser;
-  WorldChampEntrant: WorldChampEntrant;
-  LoggedCall: LoggedCall;
-  LoggedCallModelResponse: LoggedCallModelResponse;
-  LoggedCallTag: LoggedCallTag;
-  ApiKey: ApiKey;
-  Account: Account;
-  Session: Session;
-  User: User;
-  VerificationToken: VerificationToken;
-}
-
 const globalForPrisma = globalThis as unknown as {
   prisma: PrismaClient | undefined;
 };

app/src/server/db.types.ts

@@ -0,0 +1,336 @@
+import type { ColumnType } from "kysely";
+
+export type Generated<T> = T extends ColumnType<infer S, infer I, infer U>
+  ? ColumnType<S, I | undefined, U>
+  : ColumnType<T, T | undefined, T>;
+
+export type Int8 = ColumnType<string, string | number | bigint, string | number | bigint>;
+
+export type Json = ColumnType<JsonValue, string, string>;
+
+export type JsonArray = JsonValue[];
+
+export type JsonObject = {
+  [K in string]?: JsonValue;
+};
+
+export type JsonPrimitive = boolean | null | number | string;
+
+export type JsonValue = JsonArray | JsonObject | JsonPrimitive;
+
+export type Numeric = ColumnType<string, string | number, string | number>;
+
+export type Timestamp = ColumnType<Date, Date | string, Date | string>;
+
+export interface _PrismaMigrations {
+  id: string;
+  checksum: string;
+  finished_at: Timestamp | null;
+  migration_name: string;
+  logs: string | null;
+  rolled_back_at: Timestamp | null;
+  started_at: Generated<Timestamp>;
+  applied_steps_count: Generated<number>;
+}
+
+export interface Account {
+  id: string;
+  userId: string;
+  type: string;
+  provider: string;
+  providerAccountId: string;
+  refresh_token: string | null;
+  refresh_token_expires_in: number | null;
+  access_token: string | null;
+  expires_at: number | null;
+  token_type: string | null;
+  scope: string | null;
+  id_token: string | null;
+  session_state: string | null;
+}
+
+export interface ApiKey {
+  id: string;
+  name: string;
+  apiKey: string;
+  projectId: string;
+  createdAt: Generated<Timestamp>;
+  updatedAt: Timestamp;
+}
+
+export interface Dataset {
+  id: string;
+  name: string;
+  projectId: string;
+  createdAt: Generated<Timestamp>;
+  updatedAt: Timestamp;
+}
+
+export interface DatasetEntry {
+  id: string;
+  input: string;
+  output: string | null;
+  datasetId: string;
+  createdAt: Generated<Timestamp>;
+  updatedAt: Timestamp;
+}
+
+export interface Evaluation {
+  id: string;
+  label: string;
+  value: string;
+  evalType: "CONTAINS" | "DOES_NOT_CONTAIN" | "GPT4_EVAL";
+  experimentId: string;
+  createdAt: Generated<Timestamp>;
+  updatedAt: Timestamp;
+}
+
+export interface Experiment {
+  id: string;
+  label: string;
+  sortIndex: Generated<number>;
+  createdAt: Generated<Timestamp>;
+  updatedAt: Timestamp;
+  projectId: string;
+}
+
+export interface GraphileWorkerJobQueues {
+  queue_name: string;
+  job_count: number;
+  locked_at: Timestamp | null;
+  locked_by: string | null;
+}
+
+export interface GraphileWorkerJobs {
+  id: Generated<Int8>;
+  queue_name: string | null;
+  task_identifier: string;
+  payload: Generated<Json>;
+  priority: Generated<number>;
+  run_at: Generated<Timestamp>;
+  attempts: Generated<number>;
+  max_attempts: Generated<number>;
+  last_error: string | null;
+  created_at: Generated<Timestamp>;
+  updated_at: Generated<Timestamp>;
+  key: string | null;
+  locked_at: Timestamp | null;
+  locked_by: string | null;
+  revision: Generated<number>;
+  flags: Json | null;
+}
+
+export interface GraphileWorkerKnownCrontabs {
+  identifier: string;
+  known_since: Timestamp;
+  last_execution: Timestamp | null;
+}
+
+export interface GraphileWorkerMigrations {
+  id: number;
+  ts: Generated<Timestamp>;
+}
+
+export interface LoggedCall {
+  id: string;
+  requestedAt: Timestamp;
+  cacheHit: boolean;
+  modelResponseId: string | null;
+  projectId: string;
+  createdAt: Generated<Timestamp>;
+  updatedAt: Timestamp;
+  model: string | null;
+}
+
+export interface LoggedCallModelResponse {
+  id: string;
+  reqPayload: Json;
+  statusCode: number | null;
+  respPayload: Json | null;
+  errorMessage: string | null;
+  requestedAt: Timestamp;
+  receivedAt: Timestamp;
+  cacheKey: string | null;
+  durationMs: number | null;
+  inputTokens: number | null;
+  outputTokens: number | null;
+  finishReason: string | null;
+  completionId: string | null;
+  cost: Numeric | null;
+  originalLoggedCallId: string;
+  createdAt: Generated<Timestamp>;
+  updatedAt: Timestamp;
+}
+
+export interface LoggedCallTag {
+  id: string;
+  name: string;
+  value: string | null;
+  loggedCallId: string;
+  projectId: string;
+}
+
+export interface ModelResponse {
+  id: string;
+  cacheKey: string;
+  respPayload: Json | null;
+  inputTokens: number | null;
+  outputTokens: number | null;
+  createdAt: Generated<Timestamp>;
+  updatedAt: Timestamp;
+  scenarioVariantCellId: string;
+  cost: number | null;
+  requestedAt: Timestamp | null;
+  receivedAt: Timestamp | null;
+  statusCode: number | null;
+  errorMessage: string | null;
+  retryTime: Timestamp | null;
+  outdated: Generated<boolean>;
+}
+
+export interface OutputEvaluation {
+  id: string;
+  result: number;
+  details: string | null;
+  modelResponseId: string;
+  evaluationId: string;
+  createdAt: Generated<Timestamp>;
+  updatedAt: Timestamp;
+}
+
+export interface Project {
+  id: string;
+  createdAt: Generated<Timestamp>;
+  updatedAt: Timestamp;
+  personalProjectUserId: string | null;
+  name: Generated<string>;
+}
+
+export interface ProjectUser {
+  id: string;
+  role: "ADMIN" | "MEMBER" | "VIEWER";
+  projectId: string;
+  userId: string;
+  createdAt: Generated<Timestamp>;
+  updatedAt: Timestamp;
+}
+
+export interface PromptVariant {
+  id: string;
+  label: string;
+  uiId: string;
+  visible: Generated<boolean>;
+  sortIndex: Generated<number>;
+  experimentId: string;
+  createdAt: Generated<Timestamp>;
+  updatedAt: Timestamp;
+  promptConstructor: string;
+  model: string;
+  promptConstructorVersion: number;
+  modelProvider: string;
+}
+
+export interface ScenarioVariantCell {
+  id: string;
+  errorMessage: string | null;
+  promptVariantId: string;
+  testScenarioId: string;
+  createdAt: Generated<Timestamp>;
+  updatedAt: Timestamp;
+  retrievalStatus: Generated<"COMPLETE" | "ERROR" | "IN_PROGRESS" | "PENDING">;
+  prompt: Json | null;
+  jobQueuedAt: Timestamp | null;
+  jobStartedAt: Timestamp | null;
+}
+
+export interface Session {
+  id: string;
+  sessionToken: string;
+  userId: string;
+  expires: Timestamp;
+}
+
+export interface TemplateVariable {
+  id: string;
+  label: string;
+  experimentId: string;
+  createdAt: Generated<Timestamp>;
+  updatedAt: Timestamp;
+}
+
+export interface TestScenario {
+  id: string;
+  variableValues: Json;
+  uiId: string;
+  visible: Generated<boolean>;
+  sortIndex: Generated<number>;
+  experimentId: string;
+  createdAt: Generated<Timestamp>;
+  updatedAt: Timestamp;
+}
+
+export interface User {
+  id: string;
+  name: string | null;
+  email: string | null;
+  emailVerified: Timestamp | null;
+  image: string | null;
+  createdAt: Generated<Timestamp>;
+  updatedAt: Generated<Timestamp>;
+  role: Generated<"ADMIN" | "USER">;
+}
+
+export interface UserInvitation {
+  id: string;
+  projectId: string;
+  email: string;
+  role: "ADMIN" | "MEMBER" | "VIEWER";
+  invitationToken: string;
+  senderId: string;
+  createdAt: Generated<Timestamp>;
+  updatedAt: Timestamp;
+}
+
+export interface VerificationToken {
+  identifier: string;
+  token: string;
+  expires: Timestamp;
+}
+
+export interface WorldChampEntrant {
+  id: string;
+  userId: string;
+  approved: Generated<boolean>;
+  createdAt: Generated<Timestamp>;
+  updatedAt: Timestamp;
+}
+
+export interface DB {
+  _prisma_migrations: _PrismaMigrations;
+  Account: Account;
+  ApiKey: ApiKey;
+  Dataset: Dataset;
+  DatasetEntry: DatasetEntry;
+  Evaluation: Evaluation;
+  Experiment: Experiment;
+  "graphile_worker.job_queues": GraphileWorkerJobQueues;
+  "graphile_worker.jobs": GraphileWorkerJobs;
+  "graphile_worker.known_crontabs": GraphileWorkerKnownCrontabs;
+  "graphile_worker.migrations": GraphileWorkerMigrations;
+  LoggedCall: LoggedCall;
+  LoggedCallModelResponse: LoggedCallModelResponse;
+  LoggedCallTag: LoggedCallTag;
+  ModelResponse: ModelResponse;
+  OutputEvaluation: OutputEvaluation;
+  Project: Project;
+  ProjectUser: ProjectUser;
+  PromptVariant: PromptVariant;
+  ScenarioVariantCell: ScenarioVariantCell;
+  Session: Session;
+  TemplateVariable: TemplateVariable;
+  TestScenario: TestScenario;
+  User: User;
+  UserInvitation: UserInvitation;
+  VerificationToken: VerificationToken;
+  WorldChampEntrant: WorldChampEntrant;
+}

app/src/server/emails/sendEmail.ts

@@ -0,0 +1,31 @@
+import { marked } from "marked";
+import nodemailer from "nodemailer";
+import { env } from "~/env.mjs";
+
+const transporter = nodemailer.createTransport({
+  // All the SMTP_ env vars come from https://app.brevo.com/settings/keys/smtp
+  // @ts-expect-error nodemailer types are wrong
+  host: env.SMTP_HOST,
+  port: env.SMTP_PORT,
+  auth: {
+    user: env.SMTP_LOGIN,
+    pass: env.SMTP_PASSWORD,
+  },
+});
+
+export const sendEmail = async (options: { to: string; subject: string; body: string }) => {
+  const bodyHtml = await marked.parseInline(options.body, { mangle: false });
+
+  try {
+    await transporter.sendMail({
+      from: env.SENDER_EMAIL,
+      to: options.to,
+      subject: options.subject,
+      html: bodyHtml,
+      text: options.body,
+    });
+  } catch (error) {
+    console.error("error sending email", error);
+    throw error;
+  }
+};

app/src/server/emails/sendProjectInvitation.ts

@@ -0,0 +1,29 @@
+import { env } from "~/env.mjs";
+import { sendEmail } from "./sendEmail";
+
+export const sendProjectInvitation = async ({
+  invitationToken,
+  recipientEmail,
+  invitationSenderName,
+  invitationSenderEmail,
+  projectName,
+}: {
+  invitationToken: string;
+  recipientEmail: string;
+  invitationSenderName: string;
+  invitationSenderEmail: string;
+  projectName: string;
+}) => {
+  const invitationLink = `${env.NEXT_PUBLIC_HOST}/invitations/${invitationToken}`;
+
+  const emailBody = `
+    <p>You have been invited to join ${projectName} by ${invitationSenderName} (${invitationSenderEmail}).</p>
+    <p>Click <a href="${invitationLink}">here</a> to accept the invitation.</p>
+    `;
+
+  await sendEmail({
+    to: recipientEmail,
+    subject: "You've been invited to join a project",
+    body: emailBody,
+  });
+};

app/src/server/scripts/openai-test.ts

@@ -1,19 +0,0 @@
-import "dotenv/config";
-import { openai } from "../utils/openai";
-
-const resp = await openai.chat.completions.create({
-  model: "gpt-3.5-turbo-0613",
-  stream: true,
-  messages: [
-    {
-      role: "user",
-      content: "count to 20",
-    },
-  ],
-});
-
-for await (const part of resp) {
-  console.log("part", part);
-}
-
-console.log("final resp", resp);

app/src/server/tasks/defineTask.ts

@@ -1,15 +1,26 @@
-// Import necessary dependencies
-import { quickAddJob, type Helpers, type Task } from "graphile-worker";
+import { type Helpers, type Task, makeWorkerUtils, TaskSpec } from "graphile-worker";
 import { env } from "~/env.mjs";
 
-// Define the defineTask function
+let workerUtilsPromise: ReturnType<typeof makeWorkerUtils> | null = null;
+
+function workerUtils() {
+  if (!workerUtilsPromise) {
+    workerUtilsPromise = makeWorkerUtils({
+      connectionString: env.DATABASE_URL,
+    });
+  }
+  return workerUtilsPromise;
+}
+
 function defineTask<TPayload>(
   taskIdentifier: string,
   taskHandler: (payload: TPayload, helpers: Helpers) => Promise<void>,
 ) {
-  const enqueue = async (payload: TPayload, runAt?: Date) => {
+  const enqueue = async (payload: TPayload, spec?: TaskSpec) => {
     console.log("Enqueuing task", taskIdentifier, payload);
-    await quickAddJob({ connectionString: env.DATABASE_URL }, taskIdentifier, payload, { runAt });
+
+    const utils = await workerUtils();
+    return await utils.addJob(taskIdentifier, payload, spec);
   };
 
   const handler = (payload: TPayload, helpers: Helpers) => {

app/src/server/tasks/queryModel.task.ts

@@ -25,7 +25,6 @@ function calculateDelay(numPreviousTries: number): number {
 }
 
 export const queryModel = defineTask<QueryModelJob>("queryModel", async (task) => {
-  console.log("RUNNING TASK", task);
   const { cellId, stream, numPreviousTries } = task;
   const cell = await prisma.scenarioVariantCell.findUnique({
     where: { id: cellId },
@@ -153,7 +152,7 @@ export const queryModel = defineTask<QueryModelJob>("queryModel", async (task) =
           stream,
           numPreviousTries: numPreviousTries + 1,
         },
-        retryTime,
+        { runAt: retryTime, jobKey: cellId, priority: 3 },
       );
       await prisma.scenarioVariantCell.update({
         where: { id: cellId },
@@ -172,7 +171,13 @@ export const queryModel = defineTask<QueryModelJob>("queryModel", async (task) =
   }
 });
 
-export const queueQueryModel = async (cellId: string, stream: boolean) => {
+export const queueQueryModel = async (
+  cellId: string,
+  options: { stream?: boolean; hardRefetch?: boolean } = {},
+) => {
+  // Hard refetches are higher priority than streamed queries, which are higher priority than non-streamed queries.
+  const jobPriority = options.hardRefetch ? 0 : options.stream ? 1 : 2;
+
   await Promise.all([
     prisma.scenarioVariantCell.update({
       where: {
@@ -184,6 +189,13 @@ export const queueQueryModel = async (cellId: string, stream: boolean) => {
         jobQueuedAt: new Date(),
       },
     }),
-    queryModel.enqueue({ cellId, stream, numPreviousTries: 0 }),
+
+    queryModel.enqueue(
+      { cellId, stream: options.stream ?? false, numPreviousTries: 0 },
+
+      // Streamed queries are higher priority than non-streamed queries. Lower
+      // numbers are higher priority in graphile-worker.
+      { jobKey: cellId, priority: jobPriority },
+    ),
   ]);
 };

app/src/server/tasks/runNewEval.task.ts

@@ -13,5 +13,6 @@ export const runNewEval = defineTask<RunNewEvalJob>("runNewEval", async (task) =
 });
 
 export const queueRunNewEval = async (experimentId: string) => {
-  await runNewEval.enqueue({ experimentId });
+  // Evals are lower priority than completions
+  await runNewEval.enqueue({ experimentId }, { priority: 4 });
 };

app/src/server/tasks/test-tasks.ts

@@ -0,0 +1,47 @@
+import "dotenv/config";
+
+import defineTask from "./defineTask";
+import { type TaskList, run } from "graphile-worker";
+import { env } from "~/env.mjs";
+
+import "../../../sentry.server.config";
+
+export type TestTask = { i: number };
+
+// When a new eval is created, we want to run it on all existing outputs, but return the new eval first
+export const testTask = defineTask<TestTask>("testTask", (task) => {
+  console.log("ran task ", task.i);
+
+  void new Promise((_resolve, reject) => setTimeout(reject, 500));
+  return Promise.resolve();
+});
+
+const registeredTasks = [testTask];
+
+const taskList = registeredTasks.reduce((acc, task) => {
+  acc[task.task.identifier] = task.task.handler;
+  return acc;
+}, {} as TaskList);
+
+// process.on("unhandledRejection", (reason, promise) => {
+//   console.log("Unhandled Rejection at:", reason?.stack || reason);
+// });
+
+// Run a worker to execute jobs:
+const runner = await run({
+  connectionString: env.DATABASE_URL,
+  concurrency: 10,
+  // Install signal handlers for graceful shutdown on SIGINT, SIGTERM, etc
+  noHandleSignals: false,
+  pollInterval: 1000,
+  taskList,
+});
+
+console.log("Worker successfully started");
+
+for (let i = 0; i < 10; i++) {
+  await testTask.enqueue({ i });
+  await new Promise((resolve) => setTimeout(resolve, 1000));
+}
+
+await runner.promise;

app/src/server/tasks/worker.ts

@@ -1,5 +1,6 @@
 import { type TaskList, run } from "graphile-worker";
 import "dotenv/config";
+import "../../../sentry.server.config";
 
 import { env } from "~/env.mjs";
 import { queryModel } from "./queryModel.task";
@@ -17,7 +18,8 @@ const taskList = registeredTasks.reduce((acc, task) => {
 // Run a worker to execute jobs:
 const runner = await run({
   connectionString: env.DATABASE_URL,
-  concurrency: 10,
+  concurrency: env.WORKER_CONCURRENCY,
+  maxPoolSize: env.WORKER_MAX_POOL_SIZE,
   // Install signal handlers for graceful shutdown on SIGINT, SIGTERM, etc
   noHandleSignals: false,
   pollInterval: 1000,

app/src/server/utils/deriveNewContructFn.ts

@@ -109,6 +109,12 @@ const requestUpdatedPromptFunction = async (
         function_call: {
           name: "update_prompt_constructor_function",
         },
+        openpipe: {
+          tags: {
+            prompt_id: "deriveNewConstructFn",
+            model_translation: (!!newModel).toString(),
+          },
+        },
       });
       const argString = completion.choices[0]?.message?.function_call?.arguments || "{}";
 

app/src/server/utils/generateNewCell.ts

@@ -9,10 +9,8 @@ import parsePromptConstructor from "~/promptConstructor/parse";
 export const generateNewCell = async (
   variantId: string,
   scenarioId: string,
-  options?: { stream?: boolean },
+  options: { stream?: boolean; hardRefetch?: boolean } = {},
 ): Promise<void> => {
-  const stream = options?.stream ?? false;
-
   const variant = await prisma.promptVariant.findUnique({
     where: {
       id: variantId,
@@ -121,6 +119,6 @@ export const generateNewCell = async (
       }),
     );
   } else {
-    await queueQueryModel(cell.id, stream);
+    await queueQueryModel(cell.id, options);
   }
 };

app/src/server/utils/runOneEval.ts

@@ -53,6 +53,11 @@ export const runGpt4Eval = async (
         },
       },
     ],
+    openpipe: {
+      tags: {
+        prompt_id: "runOneEval",
+      },
+    },
   });
 
   try {

app/src/state/logFiltersSlice.ts

@@ -1,20 +1,21 @@
 import { type SliceCreator } from "./store";
 
-export const comparators = ["=", "!=", "CONTAINS"] as const;
+export const comparators = ["=", "!=", "CONTAINS", "NOT_CONTAINS"] as const;
 
 export const defaultFilterableFields = ["Request", "Response", "Model", "Status Code"] as const;
 
 export interface LogFilter {
+  id: string;
   field: string;
   comparator: (typeof comparators)[number];
-  value?: string;
+  value: string;
 }
 
 export type LogFiltersSlice = {
   filters: LogFilter[];
   addFilter: (filter: LogFilter) => void;
-  updateFilter: (index: number, filter: LogFilter) => void;
-  deleteFilter: (index: number) => void;
+  updateFilter: (filter: LogFilter) => void;
+  deleteFilter: (id: string) => void;
   clearSelectedLogIds: () => void;
 };
 
@@ -24,12 +25,14 @@ export const createLogFiltersSlice: SliceCreator<LogFiltersSlice> = (set, get) =
     set((state) => {
       state.logFilters.filters.push(filter);
     }),
-  updateFilter: (index: number, filter: LogFilter) =>
+  updateFilter: (filter: LogFilter) =>
     set((state) => {
+      const index = state.logFilters.filters.findIndex((f) => f.id === filter.id);
       state.logFilters.filters[index] = filter;
     }),
-  deleteFilter: (index: number) =>
+  deleteFilter: (id: string) =>
     set((state) => {
+      const index = state.logFilters.filters.findIndex((f) => f.id === id);
       state.logFilters.filters.splice(index, 1);
     }),
   clearSelectedLogIds: () =>

app/src/theme/ChakraThemeProvider.tsx

@@ -18,7 +18,7 @@ const { definePartsStyle, defineMultiStyleConfig } = createMultiStyleConfigHelpe
 
 const modalTheme = defineMultiStyleConfig({
   baseStyle: definePartsStyle({
-    dialog: { borderRadius: "sm" },
+    dialog: { borderRadius: "md", mx: 4 },
   }),
 });
 

app/src/utils/accessControl.ts

@@ -17,6 +17,8 @@ export const requireNothing = (ctx: TRPCContext) => {
 };
 
 export const requireIsProjectAdmin = async (projectId: string, ctx: TRPCContext) => {
+  ctx.markAccessControlRun();
+
   const userId = ctx.session?.user.id;
   if (!userId) {
     throw new TRPCError({ code: "UNAUTHORIZED" });
@@ -33,11 +35,11 @@ export const requireIsProjectAdmin = async (projectId: string, ctx: TRPCContext)
   if (!isAdmin) {
     throw new TRPCError({ code: "UNAUTHORIZED" });
   }
-
-  ctx.markAccessControlRun();
 };
 
 export const requireCanViewProject = async (projectId: string, ctx: TRPCContext) => {
+  ctx.markAccessControlRun();
+
   const userId = ctx.session?.user.id;
   if (!userId) {
     throw new TRPCError({ code: "UNAUTHORIZED" });
@@ -53,11 +55,11 @@ export const requireCanViewProject = async (projectId: string, ctx: TRPCContext)
   if (!canView) {
     throw new TRPCError({ code: "UNAUTHORIZED" });
   }
-
-  ctx.markAccessControlRun();
 };
 
 export const requireCanModifyProject = async (projectId: string, ctx: TRPCContext) => {
+  ctx.markAccessControlRun();
+
   const userId = ctx.session?.user.id;
   if (!userId) {
     throw new TRPCError({ code: "UNAUTHORIZED" });
@@ -74,11 +76,11 @@ export const requireCanModifyProject = async (projectId: string, ctx: TRPCContex
   if (!canModify) {
     throw new TRPCError({ code: "UNAUTHORIZED" });
   }
-
-  ctx.markAccessControlRun();
 };
 
 export const requireCanViewDataset = async (datasetId: string, ctx: TRPCContext) => {
+  ctx.markAccessControlRun();
+
   const dataset = await prisma.dataset.findFirst({
     where: {
       id: datasetId,
@@ -96,8 +98,6 @@ export const requireCanViewDataset = async (datasetId: string, ctx: TRPCContext)
   if (!dataset) {
     throw new TRPCError({ code: "UNAUTHORIZED" });
   }
-
-  ctx.markAccessControlRun();
 };
 
 export const requireCanModifyDataset = async (datasetId: string, ctx: TRPCContext) => {
@@ -105,13 +105,10 @@ export const requireCanModifyDataset = async (datasetId: string, ctx: TRPCContex
   await requireCanViewDataset(datasetId, ctx);
 };
 
-export const requireCanViewExperiment = async (experimentId: string, ctx: TRPCContext) => {
-  await prisma.experiment.findFirst({
-    where: { id: experimentId },
-  });
-
+export const requireCanViewExperiment = (experimentId: string, ctx: TRPCContext): Promise<void> => {
   // Right now all experiments are publicly viewable, so this is a no-op.
   ctx.markAccessControlRun();
+  return Promise.resolve();
 };
 
 export const canModifyExperiment = async (experimentId: string, userId: string) => {
@@ -136,6 +133,8 @@ export const canModifyExperiment = async (experimentId: string, userId: string)
 };
 
 export const requireCanModifyExperiment = async (experimentId: string, ctx: TRPCContext) => {
+  ctx.markAccessControlRun();
+
   const userId = ctx.session?.user.id;
   if (!userId) {
     throw new TRPCError({ code: "UNAUTHORIZED" });
@@ -144,6 +143,17 @@ export const requireCanModifyExperiment = async (experimentId: string, ctx: TRPC
   if (!(await canModifyExperiment(experimentId, userId))) {
     throw new TRPCError({ code: "UNAUTHORIZED" });
   }
-
-  ctx.markAccessControlRun();
+};
+
+export const requireIsAdmin = async (ctx: TRPCContext) => {
+  ctx.markAccessControlRun();
+
+  const userId = ctx.session?.user.id;
+  if (!userId) {
+    throw new TRPCError({ code: "UNAUTHORIZED" });
+  }
+
+  if (!(await isAdmin(userId))) {
+    throw new TRPCError({ code: "UNAUTHORIZED" });
+  }
 };

app/src/utils/hooks.ts

@@ -15,8 +15,8 @@ export const useExperiments = () => {
 export const useExperiment = () => {
   const router = useRouter();
   const experiment = api.experiments.get.useQuery(
-    { id: router.query.id as string },
-    { enabled: !!router.query.id },
+    { slug: router.query.experimentSlug as string },
+    { enabled: !!router.query.experimentSlug },
   );
 
   return experiment;

app/test-docker.sh

@@ -1,9 +0,0 @@
-#! /bin/bash
-
-set -e
-
-cd "$(dirname "$0")/.."
-
-source app/.env
-
-docker build . --file app/Dockerfile

client-libs/openapi.json

@@ -141,9 +141,19 @@
                   "type": "object",
                   "properties": {
                     "status": {
-                      "type": "string",
-                      "enum": [
-                        "ok"
+                      "anyOf": [
+                        {
+                          "type": "string",
+                          "enum": [
+                            "ok"
+                          ]
+                        },
+                        {
+                          "type": "string",
+                          "enum": [
+                            "error"
+                          ]
+                        }
                       ]
                     }
                   },

client-libs/python/openpipe/api_client/models/__init__.py

@@ -13,7 +13,8 @@ from .local_testing_only_get_latest_logged_call_response_200_tags import (
 from .report_json_body import ReportJsonBody
 from .report_json_body_tags import ReportJsonBodyTags
 from .report_response_200 import ReportResponse200
-from .report_response_200_status import ReportResponse200Status
+from .report_response_200_status_type_0 import ReportResponse200StatusType0
+from .report_response_200_status_type_1 import ReportResponse200StatusType1
 
 __all__ = (
     "CheckCacheJsonBody",
@@ -25,5 +26,6 @@ __all__ = (
     "ReportJsonBody",
     "ReportJsonBodyTags",
     "ReportResponse200",
-    "ReportResponse200Status",
+    "ReportResponse200StatusType0",
+    "ReportResponse200StatusType1",
 )

client-libs/python/openpipe/api_client/models/report_response_200.py

@@ -1,8 +1,9 @@
-from typing import Any, Dict, Type, TypeVar
+from typing import Any, Dict, Type, TypeVar, Union
 
 from attrs import define
 
-from ..models.report_response_200_status import ReportResponse200Status
+from ..models.report_response_200_status_type_0 import ReportResponse200StatusType0
+from ..models.report_response_200_status_type_1 import ReportResponse200StatusType1
 
 T = TypeVar("T", bound="ReportResponse200")
 
@@ -11,13 +12,19 @@ T = TypeVar("T", bound="ReportResponse200")
 class ReportResponse200:
     """
     Attributes:
-        status (ReportResponse200Status):
+        status (Union[ReportResponse200StatusType0, ReportResponse200StatusType1]):
     """
 
-    status: ReportResponse200Status
+    status: Union[ReportResponse200StatusType0, ReportResponse200StatusType1]
 
     def to_dict(self) -> Dict[str, Any]:
-        status = self.status.value
+        status: str
+
+        if isinstance(self.status, ReportResponse200StatusType0):
+            status = self.status.value
+
+        else:
+            status = self.status.value
 
         field_dict: Dict[str, Any] = {}
         field_dict.update(
@@ -31,7 +38,23 @@ class ReportResponse200:
     @classmethod
     def from_dict(cls: Type[T], src_dict: Dict[str, Any]) -> T:
         d = src_dict.copy()
-        status = ReportResponse200Status(d.pop("status"))
+
+        def _parse_status(data: object) -> Union[ReportResponse200StatusType0, ReportResponse200StatusType1]:
+            try:
+                if not isinstance(data, str):
+                    raise TypeError()
+                status_type_0 = ReportResponse200StatusType0(data)
+
+                return status_type_0
+            except:  # noqa: E722
+                pass
+            if not isinstance(data, str):
+                raise TypeError()
+            status_type_1 = ReportResponse200StatusType1(data)
+
+            return status_type_1
+
+        status = _parse_status(d.pop("status"))
 
         report_response_200 = cls(
             status=status,

client-libs/python/openpipe/api_client/models/report_response_200_status_type_0.py

@@ -1,7 +1,7 @@
 from enum import Enum
 
 
-class ReportResponse200Status(str, Enum):
+class ReportResponse200StatusType0(str, Enum):
     OK = "ok"
 
     def __str__(self) -> str:

client-libs/python/openpipe/api_client/models/report_response_200_status_type_1.py

@@ -0,0 +1,8 @@
+from enum import Enum
+
+
+class ReportResponse200StatusType1(str, Enum):
+    ERROR = "error"
+
+    def __str__(self) -> str:
+        return str(self.value)

client-libs/python/openpipe/shared.py

@@ -24,10 +24,18 @@ def _get_tags(openpipe_options):
     return ReportJsonBodyTags.from_dict(tags)
 
 
-def _should_check_cache(openpipe_options):
+def _should_check_cache(openpipe_options, req_payload):
     if configured_client.token == "":
         return False
-    return openpipe_options.get("cache", False)
+
+    cache_requested = openpipe_options.get("cache", False)
+    streaming = req_payload.get("stream", False)
+    if cache_requested and streaming:
+        print(
+            "Caching is not yet supported for streaming requests. Ignoring cache flag. Vote for this feature at https://github.com/OpenPipe/OpenPipe/issues/159"
+        )
+        return False
+    return cache_requested
 
 
 def _process_cache_payload(
@@ -44,7 +52,7 @@ def maybe_check_cache(
     openpipe_options={},
     req_payload={},
 ):
-    if not _should_check_cache(openpipe_options):
+    if not _should_check_cache(openpipe_options, req_payload):
         return None
     try:
         payload = check_cache.sync(
@@ -68,7 +76,7 @@ async def maybe_check_cache_async(
     openpipe_options={},
     req_payload={},
 ):
-    if not _should_check_cache(openpipe_options):
+    if not _should_check_cache(openpipe_options, req_payload):
         return None
 
     try:

client-libs/typescript/package.json

@@ -13,15 +13,17 @@
   "author": "",
   "license": "Apache-2.0",
   "dependencies": {
+    "encoding": "^0.1.13",
     "form-data": "^4.0.0",
     "lodash-es": "^4.17.21",
-    "node-fetch": "^3.3.2",
+    "node-fetch": "^2.6.12",
     "openai-beta": "npm:openai@4.0.0-beta.7",
     "openai-legacy": "npm:openai@3.3.0"
   },
   "devDependencies": {
     "@types/lodash-es": "^4.17.8",
     "@types/node": "^20.4.8",
+    "@types/node-fetch": "^2.6.4",
     "dotenv": "^16.3.1",
     "tsx": "^3.12.7",
     "typescript": "^5.0.4",

client-libs/typescript/src/codegen/core/request.ts

@@ -2,301 +2,283 @@
 /* istanbul ignore file */
 /* tslint:disable */
 /* eslint-disable */
-import FormData from "form-data";
-import fetch, { Headers } from "node-fetch";
-import type { RequestInit, Response } from "node-fetch";
+import FormData from 'form-data';
+import fetch, { Headers } from 'node-fetch';
+import type { RequestInit, Response } from 'node-fetch';
+import type { AbortSignal } from 'node-fetch/externals';
 
-// @ts-expect-error TODO maybe I need an older node-fetch or something?
-import type { AbortSignal } from "node-fetch/externals";
+import { ApiError } from './ApiError';
+import type { ApiRequestOptions } from './ApiRequestOptions';
+import type { ApiResult } from './ApiResult';
+import { CancelablePromise } from './CancelablePromise';
+import type { OnCancel } from './CancelablePromise';
+import type { OpenAPIConfig } from './OpenAPI';
 
-import { ApiError } from "./ApiError";
-import type { ApiRequestOptions } from "./ApiRequestOptions";
-import type { ApiResult } from "./ApiResult";
-import { CancelablePromise } from "./CancelablePromise";
-import type { OnCancel } from "./CancelablePromise";
-import type { OpenAPIConfig } from "./OpenAPI";
-
-export const isDefined = <T>(
-  value: T | null | undefined
-): value is Exclude<T, null | undefined> => {
-  return value !== undefined && value !== null;
+export const isDefined = <T>(value: T | null | undefined): value is Exclude<T, null | undefined> => {
+    return value !== undefined && value !== null;
 };
 
 export const isString = (value: any): value is string => {
-  return typeof value === "string";
+    return typeof value === 'string';
 };
 
 export const isStringWithValue = (value: any): value is string => {
-  return isString(value) && value !== "";
+    return isString(value) && value !== '';
 };
 
 export const isBlob = (value: any): value is Blob => {
-  return (
-    typeof value === "object" &&
-    typeof value.type === "string" &&
-    typeof value.stream === "function" &&
-    typeof value.arrayBuffer === "function" &&
-    typeof value.constructor === "function" &&
-    typeof value.constructor.name === "string" &&
-    /^(Blob|File)$/.test(value.constructor.name) &&
-    /^(Blob|File)$/.test(value[Symbol.toStringTag])
-  );
+    return (
+        typeof value === 'object' &&
+        typeof value.type === 'string' &&
+        typeof value.stream === 'function' &&
+        typeof value.arrayBuffer === 'function' &&
+        typeof value.constructor === 'function' &&
+        typeof value.constructor.name === 'string' &&
+        /^(Blob|File)$/.test(value.constructor.name) &&
+        /^(Blob|File)$/.test(value[Symbol.toStringTag])
+    );
 };
 
 export const isFormData = (value: any): value is FormData => {
-  return value instanceof FormData;
+    return value instanceof FormData;
 };
 
 export const base64 = (str: string): string => {
-  try {
-    return btoa(str);
-  } catch (err) {
-    // @ts-ignore
-    return Buffer.from(str).toString("base64");
-  }
+    try {
+        return btoa(str);
+    } catch (err) {
+        // @ts-ignore
+        return Buffer.from(str).toString('base64');
+    }
 };
 
 export const getQueryString = (params: Record<string, any>): string => {
-  const qs: string[] = [];
+    const qs: string[] = [];
 
-  const append = (key: string, value: any) => {
-    qs.push(`${encodeURIComponent(key)}=${encodeURIComponent(String(value))}`);
-  };
+    const append = (key: string, value: any) => {
+        qs.push(`${encodeURIComponent(key)}=${encodeURIComponent(String(value))}`);
+    };
 
-  const process = (key: string, value: any) => {
-    if (isDefined(value)) {
-      if (Array.isArray(value)) {
-        value.forEach((v) => {
-          process(key, v);
-        });
-      } else if (typeof value === "object") {
-        Object.entries(value).forEach(([k, v]) => {
-          process(`${key}[${k}]`, v);
-        });
-      } else {
-        append(key, value);
-      }
+    const process = (key: string, value: any) => {
+        if (isDefined(value)) {
+            if (Array.isArray(value)) {
+                value.forEach(v => {
+                    process(key, v);
+                });
+            } else if (typeof value === 'object') {
+                Object.entries(value).forEach(([k, v]) => {
+                    process(`${key}[${k}]`, v);
+                });
+            } else {
+                append(key, value);
+            }
+        }
+    };
+
+    Object.entries(params).forEach(([key, value]) => {
+        process(key, value);
+    });
+
+    if (qs.length > 0) {
+        return `?${qs.join('&')}`;
     }
-  };
 
-  Object.entries(params).forEach(([key, value]) => {
-    process(key, value);
-  });
-
-  if (qs.length > 0) {
-    return `?${qs.join("&")}`;
-  }
-
-  return "";
+    return '';
 };
 
 const getUrl = (config: OpenAPIConfig, options: ApiRequestOptions): string => {
-  const encoder = config.ENCODE_PATH || encodeURI;
+    const encoder = config.ENCODE_PATH || encodeURI;
 
-  const path = options.url
-    .replace("{api-version}", config.VERSION)
-    .replace(/{(.*?)}/g, (substring: string, group: string) => {
-      if (options.path?.hasOwnProperty(group)) {
-        return encoder(String(options.path[group]));
-      }
-      return substring;
-    });
+    const path = options.url
+        .replace('{api-version}', config.VERSION)
+        .replace(/{(.*?)}/g, (substring: string, group: string) => {
+            if (options.path?.hasOwnProperty(group)) {
+                return encoder(String(options.path[group]));
+            }
+            return substring;
+        });
 
-  const url = `${config.BASE}${path}`;
-  if (options.query) {
-    return `${url}${getQueryString(options.query)}`;
-  }
-  return url;
+    const url = `${config.BASE}${path}`;
+    if (options.query) {
+        return `${url}${getQueryString(options.query)}`;
+    }
+    return url;
 };
 
 export const getFormData = (options: ApiRequestOptions): FormData | undefined => {
-  if (options.formData) {
-    const formData = new FormData();
+    if (options.formData) {
+        const formData = new FormData();
 
-    const process = (key: string, value: any) => {
-      if (isString(value) || isBlob(value)) {
-        formData.append(key, value);
-      } else {
-        formData.append(key, JSON.stringify(value));
-      }
-    };
+        const process = (key: string, value: any) => {
+            if (isString(value) || isBlob(value)) {
+                formData.append(key, value);
+            } else {
+                formData.append(key, JSON.stringify(value));
+            }
+        };
 
-    Object.entries(options.formData)
-      .filter(([_, value]) => isDefined(value))
-      .forEach(([key, value]) => {
-        if (Array.isArray(value)) {
-          value.forEach((v) => process(key, v));
-        } else {
-          process(key, value);
-        }
-      });
+        Object.entries(options.formData)
+            .filter(([_, value]) => isDefined(value))
+            .forEach(([key, value]) => {
+                if (Array.isArray(value)) {
+                    value.forEach(v => process(key, v));
+                } else {
+                    process(key, value);
+                }
+            });
 
-    return formData;
-  }
-  return undefined;
+        return formData;
+    }
+    return undefined;
 };
 
 type Resolver<T> = (options: ApiRequestOptions) => Promise<T>;
 
-export const resolve = async <T>(
-  options: ApiRequestOptions,
-  resolver?: T | Resolver<T>
-): Promise<T | undefined> => {
-  if (typeof resolver === "function") {
-    return (resolver as Resolver<T>)(options);
-  }
-  return resolver;
+export const resolve = async <T>(options: ApiRequestOptions, resolver?: T | Resolver<T>): Promise<T | undefined> => {
+    if (typeof resolver === 'function') {
+        return (resolver as Resolver<T>)(options);
+    }
+    return resolver;
 };
 
-export const getHeaders = async (
-  config: OpenAPIConfig,
-  options: ApiRequestOptions
-): Promise<Headers> => {
-  const token = await resolve(options, config.TOKEN);
-  const username = await resolve(options, config.USERNAME);
-  const password = await resolve(options, config.PASSWORD);
-  const additionalHeaders = await resolve(options, config.HEADERS);
+export const getHeaders = async (config: OpenAPIConfig, options: ApiRequestOptions): Promise<Headers> => {
+    const token = await resolve(options, config.TOKEN);
+    const username = await resolve(options, config.USERNAME);
+    const password = await resolve(options, config.PASSWORD);
+    const additionalHeaders = await resolve(options, config.HEADERS);
 
-  const headers = Object.entries({
-    Accept: "application/json",
-    ...additionalHeaders,
-    ...options.headers,
-  })
-    .filter(([_, value]) => isDefined(value))
-    .reduce(
-      (headers, [key, value]) => ({
-        ...headers,
-        [key]: String(value),
-      }),
-      {} as Record<string, string>
-    );
+    const headers = Object.entries({
+        Accept: 'application/json',
+        ...additionalHeaders,
+        ...options.headers,
+    })
+        .filter(([_, value]) => isDefined(value))
+        .reduce((headers, [key, value]) => ({
+            ...headers,
+            [key]: String(value),
+        }), {} as Record<string, string>);
 
-  if (isStringWithValue(token)) {
-    headers["Authorization"] = `Bearer ${token}`;
-  }
-
-  if (isStringWithValue(username) && isStringWithValue(password)) {
-    const credentials = base64(`${username}:${password}`);
-    headers["Authorization"] = `Basic ${credentials}`;
-  }
-
-  if (options.body) {
-    if (options.mediaType) {
-      headers["Content-Type"] = options.mediaType;
-    } else if (isBlob(options.body)) {
-      headers["Content-Type"] = "application/octet-stream";
-    } else if (isString(options.body)) {
-      headers["Content-Type"] = "text/plain";
-    } else if (!isFormData(options.body)) {
-      headers["Content-Type"] = "application/json";
+    if (isStringWithValue(token)) {
+        headers['Authorization'] = `Bearer ${token}`;
     }
-  }
 
-  return new Headers(headers);
+    if (isStringWithValue(username) && isStringWithValue(password)) {
+        const credentials = base64(`${username}:${password}`);
+        headers['Authorization'] = `Basic ${credentials}`;
+    }
+
+    if (options.body) {
+        if (options.mediaType) {
+            headers['Content-Type'] = options.mediaType;
+        } else if (isBlob(options.body)) {
+            headers['Content-Type'] = 'application/octet-stream';
+        } else if (isString(options.body)) {
+            headers['Content-Type'] = 'text/plain';
+        } else if (!isFormData(options.body)) {
+            headers['Content-Type'] = 'application/json';
+        }
+    }
+
+    return new Headers(headers);
 };
 
 export const getRequestBody = (options: ApiRequestOptions): any => {
-  if (options.body !== undefined) {
-    if (options.mediaType?.includes("/json")) {
-      return JSON.stringify(options.body);
-    } else if (isString(options.body) || isBlob(options.body) || isFormData(options.body)) {
-      return options.body as any;
-    } else {
-      return JSON.stringify(options.body);
+    if (options.body !== undefined) {
+        if (options.mediaType?.includes('/json')) {
+            return JSON.stringify(options.body)
+        } else if (isString(options.body) || isBlob(options.body) || isFormData(options.body)) {
+            return options.body as any;
+        } else {
+            return JSON.stringify(options.body);
+        }
     }
-  }
-  return undefined;
+    return undefined;
 };
 
 export const sendRequest = async (
-  options: ApiRequestOptions,
-  url: string,
-  body: any,
-  formData: FormData | undefined,
-  headers: Headers,
-  onCancel: OnCancel
+    options: ApiRequestOptions,
+    url: string,
+    body: any,
+    formData: FormData | undefined,
+    headers: Headers,
+    onCancel: OnCancel
 ): Promise<Response> => {
-  const controller = new AbortController();
+    const controller = new AbortController();
 
-  const request: RequestInit = {
-    headers,
-    method: options.method,
-    body: body ?? formData,
-    signal: controller.signal as AbortSignal,
-  };
+    const request: RequestInit = {
+        headers,
+        method: options.method,
+        body: body ?? formData,
+        signal: controller.signal as AbortSignal,
+    };
 
-  onCancel(() => controller.abort());
+    onCancel(() => controller.abort());
 
-  return await fetch(url, request);
+    return await fetch(url, request);
 };
 
-export const getResponseHeader = (
-  response: Response,
-  responseHeader?: string
-): string | undefined => {
-  if (responseHeader) {
-    const content = response.headers.get(responseHeader);
-    if (isString(content)) {
-      return content;
+export const getResponseHeader = (response: Response, responseHeader?: string): string | undefined => {
+    if (responseHeader) {
+        const content = response.headers.get(responseHeader);
+        if (isString(content)) {
+            return content;
+        }
     }
-  }
-  return undefined;
+    return undefined;
 };
 
 export const getResponseBody = async (response: Response): Promise<any> => {
-  if (response.status !== 204) {
-    try {
-      const contentType = response.headers.get("Content-Type");
-      if (contentType) {
-        const jsonTypes = ["application/json", "application/problem+json"];
-        const isJSON = jsonTypes.some((type) => contentType.toLowerCase().startsWith(type));
-        if (isJSON) {
-          return await response.json();
-        } else {
-          return await response.text();
+    if (response.status !== 204) {
+        try {
+            const contentType = response.headers.get('Content-Type');
+            if (contentType) {
+                const jsonTypes = ['application/json', 'application/problem+json']
+                const isJSON = jsonTypes.some(type => contentType.toLowerCase().startsWith(type));
+                if (isJSON) {
+                    return await response.json();
+                } else {
+                    return await response.text();
+                }
+            }
+        } catch (error) {
+            console.error(error);
         }
-      }
-    } catch (error) {
-      console.error(error);
     }
-  }
-  return undefined;
+    return undefined;
 };
 
 export const catchErrorCodes = (options: ApiRequestOptions, result: ApiResult): void => {
-  const errors: Record<number, string> = {
-    400: "Bad Request",
-    401: "Unauthorized",
-    403: "Forbidden",
-    404: "Not Found",
-    500: "Internal Server Error",
-    502: "Bad Gateway",
-    503: "Service Unavailable",
-    ...options.errors,
-  };
+    const errors: Record<number, string> = {
+        400: 'Bad Request',
+        401: 'Unauthorized',
+        403: 'Forbidden',
+        404: 'Not Found',
+        500: 'Internal Server Error',
+        502: 'Bad Gateway',
+        503: 'Service Unavailable',
+        ...options.errors,
+    }
 
-  const error = errors[result.status];
-  if (error) {
-    throw new ApiError(options, result, error);
-  }
+    const error = errors[result.status];
+    if (error) {
+        throw new ApiError(options, result, error);
+    }
 
-  if (!result.ok) {
-    const errorStatus = result.status ?? "unknown";
-    const errorStatusText = result.statusText ?? "unknown";
-    const errorBody = (() => {
-      try {
-        return JSON.stringify(result.body, null, 2);
-      } catch (e) {
-        return undefined;
-      }
-    })();
+    if (!result.ok) {
+        const errorStatus = result.status ?? 'unknown';
+        const errorStatusText = result.statusText ?? 'unknown';
+        const errorBody = (() => {
+            try {
+                return JSON.stringify(result.body, null, 2);
+            } catch (e) {
+                return undefined;
+            }
+        })();
 
-    throw new ApiError(
-      options,
-      result,
-      `Generic Error: status: ${errorStatus}; status text: ${errorStatusText}; body: ${errorBody}`
-    );
-  }
+        throw new ApiError(options, result,
+            `Generic Error: status: ${errorStatus}; status text: ${errorStatusText}; body: ${errorBody}`
+        );
+    }
 };
 
 /**
@@ -306,36 +288,33 @@ export const catchErrorCodes = (options: ApiRequestOptions, result: ApiResult):
  * @returns CancelablePromise<T>
  * @throws ApiError
  */
-export const request = <T>(
-  config: OpenAPIConfig,
-  options: ApiRequestOptions
-): CancelablePromise<T> => {
-  return new CancelablePromise(async (resolve, reject, onCancel) => {
-    try {
-      const url = getUrl(config, options);
-      const formData = getFormData(options);
-      const body = getRequestBody(options);
-      const headers = await getHeaders(config, options);
+export const request = <T>(config: OpenAPIConfig, options: ApiRequestOptions): CancelablePromise<T> => {
+    return new CancelablePromise(async (resolve, reject, onCancel) => {
+        try {
+            const url = getUrl(config, options);
+            const formData = getFormData(options);
+            const body = getRequestBody(options);
+            const headers = await getHeaders(config, options);
 
-      if (!onCancel.isCancelled) {
-        const response = await sendRequest(options, url, body, formData, headers, onCancel);
-        const responseBody = await getResponseBody(response);
-        const responseHeader = getResponseHeader(response, options.responseHeader);
+            if (!onCancel.isCancelled) {
+                const response = await sendRequest(options, url, body, formData, headers, onCancel);
+                const responseBody = await getResponseBody(response);
+                const responseHeader = getResponseHeader(response, options.responseHeader);
 
-        const result: ApiResult = {
-          url,
-          ok: response.ok,
-          status: response.status,
-          statusText: response.statusText,
-          body: responseHeader ?? responseBody,
-        };
+                const result: ApiResult = {
+                    url,
+                    ok: response.ok,
+                    status: response.status,
+                    statusText: response.statusText,
+                    body: responseHeader ?? responseBody,
+                };
 
-        catchErrorCodes(options, result);
+                catchErrorCodes(options, result);
 
-        resolve(result.body);
-      }
-    } catch (error) {
-      reject(error);
-    }
-  });
+                resolve(result.body);
+            }
+        } catch (error) {
+            reject(error);
+        }
+    });
 };

client-libs/typescript/src/codegen/services/DefaultService.ts

@@ -82,7 +82,7 @@ export class DefaultService {
             tags?: Record<string, string>;
         },
     ): CancelablePromise<{
-        status: 'ok';
+        status: ('ok' | 'error');
     }> {
         return this.httpRequest.request({
             method: 'POST',

client-libs/typescript/src/openai/index.test.ts

@@ -2,10 +2,13 @@ import dotenv from "dotenv";
 import { expect, test } from "vitest";
 import OpenAI from ".";
 import {
+  ChatCompletion,
   CompletionCreateParams,
   CreateChatCompletionRequestMessage,
 } from "openai-beta/resources/chat/completions";
 import { OPClient } from "../codegen";
+import mergeChunks from "./mergeChunks";
+import assert from "assert";
 
 dotenv.config({ path: "../.env" });
 
@@ -31,9 +34,7 @@ test("basic call", async () => {
   };
   const completion = await oaiClient.chat.completions.create({
     ...payload,
-    openpipe: {
-      tags: { promptId: "test" },
-    },
+    openpipe: { tags: { promptId: "test" } },
   });
   await completion.openpipe.reportingFinished;
   const lastLogged = await lastLoggedCall();
@@ -46,29 +47,32 @@ const randomString = (length: number) => {
   const characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
   return Array.from(
     { length },
-    () => characters[Math.floor(Math.random() * characters.length)]
+    () => characters[Math.floor(Math.random() * characters.length)],
   ).join("");
 };
 
-test.skip("streaming", async () => {
+test("streaming", async () => {
   const completion = await oaiClient.chat.completions.create({
     model: "gpt-3.5-turbo",
-    messages: [{ role: "system", content: "count to 4" }],
+    messages: [{ role: "system", content: "count to 3" }],
     stream: true,
   });
 
-  let merged = null;
+  let merged: ChatCompletion | null = null;
   for await (const chunk of completion) {
-    merged = merge_openai_chunks(merged, chunk);
+    merged = mergeChunks(merged, chunk);
   }
 
   const lastLogged = await lastLoggedCall();
-  expect(lastLogged?.modelResponse?.respPayload.choices[0].message.content).toBe(
-    merged.choices[0].message.content
-  );
+  await completion.openpipe.reportingFinished;
+
+  expect(merged).toMatchObject(lastLogged?.modelResponse?.respPayload);
+  expect(lastLogged?.modelResponse?.reqPayload.messages).toMatchObject([
+    { role: "system", content: "count to 3" },
+  ]);
 });
 
-test.skip("bad call streaming", async () => {
+test("bad call streaming", async () => {
   try {
     await oaiClient.chat.completions.create({
       model: "gpt-3.5-turbo-blaster",
@@ -76,26 +80,29 @@ test.skip("bad call streaming", async () => {
       stream: true,
     });
   } catch (e) {
+    await e.openpipe.reportingFinished;
     const lastLogged = await lastLoggedCall();
-    expect(lastLogged?.modelResponse?.errorMessage).toBe(
-      "The model `gpt-3.5-turbo-blaster` does not exist"
+    expect(lastLogged?.modelResponse?.errorMessage).toEqual(
+      "The model `gpt-3.5-turbo-blaster` does not exist",
     );
-    expect(lastLogged?.modelResponse?.statusCode).toBe(404);
+    expect(lastLogged?.modelResponse?.statusCode).toEqual(404);
   }
 });
 
 test("bad call", async () => {
   try {
     await oaiClient.chat.completions.create({
-      model: "gpt-3.5-turbo-booster",
+      model: "gpt-3.5-turbo-buster",
       messages: [{ role: "system", content: "count to 10" }],
     });
   } catch (e) {
+    assert("openpipe" in e);
+    await e.openpipe.reportingFinished;
     const lastLogged = await lastLoggedCall();
-    expect(lastLogged?.modelResponse?.errorMessage).toBe(
-      "The model `gpt-3.5-turbo-booster` does not exist"
+    expect(lastLogged?.modelResponse?.errorMessage).toEqual(
+      "The model `gpt-3.5-turbo-buster` does not exist",
     );
-    expect(lastLogged?.modelResponse?.statusCode).toBe(404);
+    expect(lastLogged?.modelResponse?.statusCode).toEqual(404);
   }
 });
 
@@ -109,12 +116,12 @@ test("caching", async () => {
     messages: [message],
     openpipe: { cache: true },
   });
-  expect(completion.openpipe.cacheStatus).toBe("MISS");
+  expect(completion.openpipe.cacheStatus).toEqual("MISS");
 
   await completion.openpipe.reportingFinished;
   const firstLogged = await lastLoggedCall();
-  expect(completion.choices[0].message.content).toBe(
-    firstLogged?.modelResponse?.respPayload.choices[0].message.content
+  expect(completion.choices[0].message.content).toEqual(
+    firstLogged?.modelResponse?.respPayload.choices[0].message.content,
   );
 
   const completion2 = await oaiClient.chat.completions.create({
@@ -122,5 +129,5 @@ test("caching", async () => {
     messages: [message],
     openpipe: { cache: true },
   });
-  expect(completion2.openpipe.cacheStatus).toBe("HIT");
+  expect(completion2.openpipe.cacheStatus).toEqual("HIT");
 });

client-libs/typescript/src/openai/index.ts

@@ -5,9 +5,9 @@ import {
   ChatCompletion,
   ChatCompletionChunk,
   CompletionCreateParams,
-  Completions,
 } from "openai-beta/resources/chat/completions";
 
+import { WrappedStream } from "./streaming";
 import { DefaultService, OPClient } from "../codegen";
 import { Stream } from "openai-beta/streaming";
 import { OpenPipeArgs, OpenPipeMeta, type OpenPipeConfig, getTags } from "../shared";
@@ -27,11 +27,11 @@ export default class OpenAI extends openai.OpenAI {
           BASE:
             openpipe?.baseUrl ?? readEnv("OPENPIPE_BASE_URL") ?? "https://app.openpipe.ai/api/v1",
           TOKEN: openPipeApiKey,
-        })
+        }),
       );
     } else {
       console.warn(
-        "You're using the OpenPipe client without an API key. No completion requests will be logged."
+        "You're using the OpenPipe client without an API key. No completion requests will be logged.",
       );
     }
   }
@@ -43,10 +43,10 @@ class WrappedChat extends openai.OpenAI.Chat {
     this.completions.opClient = client;
   }
 
-  completions: InstrumentedCompletions = new InstrumentedCompletions(this.client);
+  completions: WrappedCompletions = new WrappedCompletions(this.client);
 }
 
-class InstrumentedCompletions extends openai.OpenAI.Chat.Completions {
+class WrappedCompletions extends openai.OpenAI.Chat.Completions {
   opClient?: OPClient;
 
   constructor(client: openai.OpenAI, opClient?: OPClient) {
@@ -54,32 +54,35 @@ class InstrumentedCompletions extends openai.OpenAI.Chat.Completions {
     this.opClient = opClient;
   }
 
-  _report(args: Parameters<DefaultService["report"]>[0]) {
+  async _report(args: Parameters<DefaultService["report"]>[0]) {
     try {
-      return this.opClient ? this.opClient.default.report(args) : Promise.resolve();
+      this.opClient ? await this.opClient.default.report(args) : Promise.resolve();
     } catch (e) {
       console.error(e);
-      return Promise.resolve();
     }
   }
 
   create(
     body: CompletionCreateParams.CreateChatCompletionRequestNonStreaming & OpenPipeArgs,
-    options?: Core.RequestOptions
+    options?: Core.RequestOptions,
   ): Promise<Core.APIResponse<ChatCompletion & { openpipe: OpenPipeMeta }>>;
   create(
     body: CompletionCreateParams.CreateChatCompletionRequestStreaming & OpenPipeArgs,
-    options?: Core.RequestOptions
-  ): Promise<Core.APIResponse<Stream<ChatCompletionChunk>>>;
+    options?: Core.RequestOptions,
+  ): Promise<Core.APIResponse<WrappedStream>>;
   async create(
     { openpipe, ...body }: CompletionCreateParams & OpenPipeArgs,
-    options?: Core.RequestOptions
-  ): Promise<
-    Core.APIResponse<(ChatCompletion & { openpipe: OpenPipeMeta }) | Stream<ChatCompletionChunk>>
-  > {
-    console.log("LALALA REPORT", this.opClient);
+    options?: Core.RequestOptions,
+  ): Promise<Core.APIResponse<(ChatCompletion & { openpipe: OpenPipeMeta }) | WrappedStream>> {
     const requestedAt = Date.now();
-    const cacheRequested = openpipe?.cache ?? false;
+    let reportingFinished: OpenPipeMeta["reportingFinished"] = Promise.resolve();
+    let cacheRequested = openpipe?.cache ?? false;
+    if (cacheRequested && body.stream) {
+      console.warn(
+        `Caching is not yet supported for streaming requests. Ignoring cache flag. Vote for this feature at https://github.com/OpenPipe/OpenPipe/issues/159`,
+      );
+      cacheRequested = false;
+    }
 
     if (cacheRequested) {
       try {
@@ -92,12 +95,13 @@ class InstrumentedCompletions extends openai.OpenAI.Chat.Completions {
           .then((res) => res.respPayload);
 
         if (cached) {
+          const meta = {
+            cacheStatus: "HIT",
+            reportingFinished,
+          };
           return {
             ...cached,
-            openpipe: {
-              cacheStatus: "HIT",
-              reportingFinished: Promise.resolve(),
-            },
+            openpipe: meta,
           };
         }
       } catch (e) {
@@ -105,15 +109,23 @@ class InstrumentedCompletions extends openai.OpenAI.Chat.Completions {
       }
     }
 
-    let reportingFinished: OpenPipeMeta["reportingFinished"] = Promise.resolve();
-
     try {
       if (body.stream) {
         const stream = await super.create(body, options);
+        const wrappedStream = new WrappedStream(stream, (response) =>
+          this._report({
+            requestedAt,
+            receivedAt: Date.now(),
+            reqPayload: body,
+            respPayload: response,
+            statusCode: 200,
+            tags: getTags(openpipe),
+          }),
+        );
 
         // Do some logging of each chunk here
 
-        return stream;
+        return wrappedStream;
       } else {
         const response = await super.create(body, options);
 
@@ -147,6 +159,16 @@ class InstrumentedCompletions extends openai.OpenAI.Chat.Completions {
           tags: getTags(openpipe),
         });
       }
+      // make sure error is an object we can add properties to
+      if (typeof error === "object" && error !== null) {
+        error = {
+          ...error,
+          openpipe: {
+            cacheStatus: cacheRequested ? "MISS" : "SKIP",
+            reportingFinished,
+          },
+        };
+      }
 
       throw error;
     }

client-libs/typescript/src/openai/streaming.ts

@@ -0,0 +1,43 @@
+import { ChatCompletion, ChatCompletionChunk } from "openai-beta/resources/chat";
+import { Stream } from "openai-beta/streaming";
+import { OpenPipeMeta } from "../shared";
+import mergeChunks from "./mergeChunks";
+
+export class WrappedStream extends Stream<ChatCompletionChunk> {
+  openpipe: OpenPipeMeta;
+
+  private resolveReportingFinished: () => void = () => {};
+  private report: (response: unknown) => Promise<void>;
+
+  constructor(stream: Stream<ChatCompletionChunk>, report: (response: unknown) => Promise<void>) {
+    super(stream.response, stream.controller);
+    this.report = report;
+
+    const reportingFinished = new Promise<void>((resolve) => {
+      this.resolveReportingFinished = resolve;
+    });
+
+    this.openpipe = {
+      cacheStatus: "MISS",
+      reportingFinished,
+    };
+  }
+
+  async *[Symbol.asyncIterator](): AsyncIterator<ChatCompletionChunk, any, undefined> {
+    const iterator = super[Symbol.asyncIterator]();
+
+    let combinedResponse: ChatCompletion | null = null;
+    while (true) {
+      const result = await iterator.next();
+      if (result.done) break;
+      combinedResponse = mergeChunks(combinedResponse, result.value);
+
+      yield result.value;
+    }
+
+    await this.report(combinedResponse);
+
+    // Resolve the promise here
+    this.resolveReportingFinished();
+  }
+}

client-libs/typescript/src/shared.ts

@@ -1,4 +1,5 @@
 import pkg from "../package.json";
+import { DefaultService } from "./codegen";
 
 export type OpenPipeConfig = {
   apiKey?: string;
@@ -15,9 +16,11 @@ export type OpenPipeMeta = {
   // We report your call to OpenPipe asynchronously in the background. If you
   // need to wait until the report is sent to take further action, you can await
   // this promise.
-  reportingFinished: Promise<void | { status: "ok" }>;
+  reportingFinished: Promise<void>;
 };
 
+export type ReportFn = (...args: Parameters<DefaultService["report"]>) => Promise<void>;
+
 export const getTags = (args: OpenPipeArgs["openpipe"]): Record<string, string> => ({
   ...args?.tags,
   ...(args?.cache ? { $cache: args.cache?.toString() } : {}),